#! /usr/bin/python

# ...
try:
    from matplotlib import pyplot as plt
    PLOT=True
except ImportError:
    PLOT=False
# ...
import numpy                as np
from pigasus.gallery.poisson import *
import sys
import inspect
filename = inspect.getfile(inspect.currentframe()) # script filename (usually with path)

# ...
sin = np.sin ; cos = np.cos ; pi = np.pi ; exp = np.exp
# ...

#-----------------------------------
try:
    nx = int(sys.argv[1])
except:
    nx = 31

try:
    ny = int(sys.argv[2])
except:
    ny = 31

try:
    px = int(sys.argv[3])
except:
    px = 2

try:
    py = int(sys.argv[4])
except:
    py = 2

from igakit.cad_geometry import quart_circle as domain
geo = domain(n=[nx,ny],p=[px,py])
#-----------------------------------

# ...
# exact solution
# ...
R = 1.
r = 0.5
c = 1. # for neumann
#c = pi / (R**2-r**2) # for all dirichlet bc
u = lambda x,y : [ x * y * sin ( c * (R**2 - x**2 - y**2 )) ]
# ...

# ...
# rhs
# ...
f = lambda x,y : [4*c**2*x**3*y*sin(c*(R**2 - x**2 - y**2)) \
                  + 4*c**2*x*y**3*sin(c*(R**2 - x**2 - y**2)) \
                  + 12*c*x*y*cos(c*(R**2 - x**2 - y**2)) ]
# ...

# ...
# values of gradu.n at the boundary
# ...
gradu   = lambda x,y : [-2*c*x**2*y*cos(c*(R**2 - x**2 - y**2)) + y*sin(c*(R**2
                                                                           -
                                                                           x**2
                                                                           -
                                                                           y**2)) \
                       ,-2*c*x*y**2*cos(c*(R**2 - x**2 - y**2)) + x*sin(c*(R**2 - x**2 - y**2)) ]

def func_g (x,y) :
    du  = gradu (x, y)
    return [ du[0] , du[1] ]
# ...

# ...
# values of u at the boundary
# ...

bc_neumann={}

bc_neumann [0,0] = func_g
Dirichlet = [[1,2,3]]

#AllDirichlet = True
# ...

# ...
try:
    bc_dirichlet
except NameError:
    bc_dirichlet = None
else:
    pass

try:
    bc_neumann
except NameError:
    bc_neumann = None
else:
    pass

try:
    AllDirichlet
except NameError:
    AllDirichlet = None
else:
    pass

try:
    Dirichlet
except NameError:
    Dirichlet = None
else:
    pass

try:
    Metric
except NameError:
    Metric = None
else:
    pass
# ...

# ...
PDE = poisson(geometry=geo, bc_dirichlet=bc_dirichlet, bc_neumann=bc_neumann,
              AllDirichlet=AllDirichlet, Dirichlet=Dirichlet,metric=Metric)
# ...

# ...
PDE.assembly(f=f)
PDE.solve()
# ...

# ...
normU = PDE.norm(exact=u)
print "norm U   = ", normU
# ...

# ...
if PLOT:
    PDE.plot()  ; plt.colorbar(); plt.title('$u_h$')
    plt.savefig(filename.split('.py')[0]+'.png', format='png')
    plt.clf()
# ...

PDE.free()


"""
=========================================================
Gaussian Processes regression: basic introductory example
=========================================================

A simple one-dimensional regression example computed in two different ways:

1. A noise-free case
2. A noisy case with known noise-level per datapoint

In both cases, the kernel's parameters are estimated using the maximum
likelihood principle.

The figures illustrate the interpolating property of the Gaussian Process
model as well as its probabilistic nature in the form of a pointwise 95%
confidence interval.

Note that the parameter ``alpha`` is applied as a Tikhonov
regularization of the assumed covariance between the training points.
"""
print(__doc__)

# Author: Vincent Dubourg <vincent.dubourg@gmail.com>
#         Jake Vanderplas <vanderplas@astro.washington.edu>
#         Jan Hendrik Metzen <jhm@informatik.uni-bremen.de>s
# License: BSD 3 clause

import numpy as np
from matplotlib import pyplot as plt

from sklearn.gaussian_process import GaussianProcessRegressor
from sklearn.gaussian_process.kernels import RBF, ConstantKernel as C

np.random.seed(1)

def f(x):
    """The function to predict."""
    return x * np.sin(x)

# ----------------------------------------------------------------------
#  First the noiseless case
X = np.atleast_2d([1., 3., 5., 6., 7., 8.]).T

# Observations
y = f(X).ravel()

# Mesh the input space for evaluations of the real function, the prediction and
# its MSE
x = np.atleast_2d(np.linspace(0, 10, 1000)).T

# Instanciate a Gaussian Process model
kernel = C(1.0, (1e-3, 1e3)) * RBF(10, (1e-2, 1e2))
gp = GaussianProcessRegressor(kernel=kernel, n_restarts_optimizer=9)

# Fit to data using Maximum Likelihood Estimation of the parameters
gp.fit(X, y)

# Make the prediction on the meshed x-axis (ask for MSE as well)
y_pred, sigma = gp.predict(x, return_std=True)

# Plot the function, the prediction and the 95% confidence interval based on
# the MSE
fig = plt.figure()
plt.plot(x, f(x), 'r:', label=u'$f(x) = x\,\sin(x)$')
plt.plot(X, y, 'r.', markersize=10, label=u'Observations')
plt.plot(x, y_pred, 'b-', label=u'Prediction')
plt.fill(np.concatenate([x, x[::-1]]),
         np.concatenate([y_pred - 1.9600 * sigma,
                        (y_pred + 1.9600 * sigma)[::-1]]),
         alpha=.5, fc='b', ec='None', label='95% confidence interval')
plt.xlabel('$x$')
plt.ylabel('$f(x)$')
plt.ylim(-10, 20)
plt.legend(loc='upper left')

# ----------------------------------------------------------------------
# now the noisy case
X = np.linspace(0.1, 9.9, 20)
X = np.atleast_2d(X).T

# Observations and noise
y = f(X).ravel()
dy = 0.5 + 1.0 * np.random.random(y.shape)
noise = np.random.normal(0, dy)
y += noise

# Instanciate a Gaussian Process model
gp = GaussianProcessRegressor(kernel=kernel, alpha=(dy / y) ** 2,
                              n_restarts_optimizer=10)

# Fit to data using Maximum Likelihood Estimation of the parameters
gp.fit(X, y)

# Make the prediction on the meshed x-axis (ask for MSE as well)
y_pred, sigma = gp.predict(x, return_std=True)

# Plot the function, the prediction and the 95% confidence interval based on
# the MSE
fig = plt.figure()
plt.plot(x, f(x), 'r:', label=u'$f(x) = x\,\sin(x)$')
plt.errorbar(X.ravel(), y, dy, fmt='r.', markersize=10, label=u'Observations')
plt.plot(x, y_pred, 'b-', label=u'Prediction')
plt.fill(np.concatenate([x, x[::-1]]),
         np.concatenate([y_pred - 1.9600 * sigma,
                        (y_pred + 1.9600 * sigma)[::-1]]),
         alpha=.5, fc='b', ec='None', label='95% confidence interval')
plt.xlabel('$x$')
plt.ylabel('$f(x)$')
plt.ylim(-10, 20)
plt.legend(loc='upper left')

plt.show()


# This file is part of BurnMan - a thermoelastic and thermodynamic toolkit for the Earth and Planetary Sciences
# Copyright (C) 2012 - 2015 by the BurnMan team, released under the GNU
# GPL v2 or later.

"""
example_geotherms
-----------------

This example shows each of the geotherms currently possible with BurnMan.
These are:

1. Brown and Shankland, 1981 :cite:`Brown1981`
2. Anderson, 1982 :cite:`anderson1982earth`
3. Watson and Baxter, 2007 :cite:`Watson2007`
4. linear extrapolation
5. Read in from file from user
6. Adiabatic from potential temperature and choice of mineral

*Uses:*

* :func:`burnman.geotherm.brown_shankland`
* :func:`burnman.geotherm.anderson`
* input geotherm file *input_geotherm/example_geotherm.txt* (optional)
* :class:`burnman.composite.Composite` for adiabat

*Demonstrates:*

* the available geotherms

"""
from __future__ import absolute_import

import os
import sys
import numpy as np
import matplotlib.pyplot as plt
# hack to allow scripts to be placed in subdirectories next to burnman:
if not os.path.exists('burnman') and os.path.exists('../burnman'):
    sys.path.insert(1, os.path.abspath('..'))

import burnman
from burnman import minerals

if __name__ == "__main__":
    # we want to evaluate several geotherms at these values
    pressures = np.arange(9.0e9, 128e9, 3e9)
    seismic_model = burnman.seismic.PREM()
    depths = seismic_model.depth(pressures)
    # load two builtin geotherms and evaluate the temperatures at all pressures
    temperature1 = burnman.geotherm.brown_shankland(depths)
    temperature2 = burnman.geotherm.anderson(depths)

    # a geotherm is actually just a function that returns a list of temperatures given pressures in Pa
    # so we can just write our own function
    my_geotherm_function = lambda p: [1500 + (2500 - 1500) * x / 128e9 for x in p]
    temperature3 = my_geotherm_function(pressures)

    # what about a geotherm defined from datapoints given in a file (our
    # inline)?
    table = [[1e9, 1600], [30e9, 1700], [130e9, 2700]]
    # this could also be loaded from a file, just uncomment this
    # table = burnman.tools.read_table("input_geotherm/example_geotherm.txt")

    table_pressure = np.array(table)[:, 0]
    table_temperature = np.array(table)[:, 1]

    my_geotherm_interpolate = lambda p: [np.interp(x, table_pressure,
                                                    table_temperature) for x in p]

    temperature4 = my_geotherm_interpolate(pressures)

    # finally, we can also calculate a self consistent
    # geotherm for an assemblage of minerals
    # based on self compression of the composite rock.
    # First we need to define an assemblage
    amount_perovskite = 0.8
    fe_pv = 0.05
    fe_pc = 0.2
    pv = minerals.SLB_2011.mg_fe_perovskite()
    pc = minerals.SLB_2011.ferropericlase()
    pv.set_composition([1. - fe_pv, fe_pv, 0.])
    pc.set_composition([1. - fe_pc, fe_pc])
    example_rock = burnman.Composite(
        [pv, pc], [amount_perovskite, 1.0 - amount_perovskite])

    # next, define an anchor temperature at which we are starting.
    # Perhaps 1500 K for the upper mantle
    T0 = 1500.
    # then generate temperature values using the self consistent function.
    # This takes more time than the above methods
    temperature5 = burnman.geotherm.adiabatic(pressures, T0, example_rock)

    # you can also look at burnman/geotherm.py to see how the geotherms are
    # implemented

    plt.plot(pressures / 1e9, temperature1, '-r', label="Brown, Shankland")
    plt.plot(pressures / 1e9, temperature2, '-c', label="Anderson")
    plt.plot(pressures / 1e9, temperature3, '-b', label="handwritten linear")
    plt.plot(pressures / 1e9, temperature4,
             '-k', label="handwritten from table")
    plt.plot(pressures / 1e9, temperature5, '-m',
             label="Adiabat with pv (70%) and fp(30%)")

    plt.legend(loc='lower right')
    plt.xlim([8.5, 130])
    plt.xlabel('Pressure/GPa')
    plt.ylabel('Temperature')
    plt.savefig("output_figures/example_geotherm.png")
    plt.show()


#!/usr/bin/env python

import glob
import numpy as np
import matplotlib.pyplot as plt

import os
import sys

pathname = os.path.dirname(sys.argv[0])
if pathname:
    os.chdir(pathname)

n_dim = None
trains = []
for fname in glob.glob("tl*"):
    t = np.loadtxt(fname)
    trains.append(t)

tests = []
for fname in glob.glob("tt*"):
    t = np.loadtxt(fname)
    tests.append(t)

trial_results= []
for fname in glob.glob("rtl*"):
    t = np.loadtxt(fname)
    trial_results.append(t)

test_results= []
for fname in glob.glob("rtt*"):
    t = np.loadtxt(fname)
    test_results.append(t)

fig = plt.figure()
ax = fig.add_subplot(111, aspect="equal")
for d in trains:
    ax.plot(d[:,1] +d[:,7]*6, d[:,2] +d[:,8]*6, color="blue", lw=3, alpha=0.5)
for d in tests:
    ax.plot(d[:,1] +d[:,7]*6, d[:,2] +d[:,8]*6, color="red",  lw=3, alpha=0.5)
for d in trial_results:
    ax.plot(d[:,1] +d[:,7]*6, d[:,2] +d[:,8]*6, color=[0,0,.5],  lw=2)
for d in test_results:
    ax.plot(d[:,1] +d[:,7]*6, d[:,2] +d[:,8]*6, color=[.5,0,0],  lw=2)
plt.show()


import zmq
import numpy
import threading

from collections import namedtuple

Point2D = namedtuple('Point2D', ('x', 'y'))
Point3D = namedtuple('Point3D', ('x', 'y', 'z'))
Quaternion = namedtuple('Quaternion', ('x', 'y', 'z', 'w'))

torso_joints = ('hip_center', 'spine', 'shoulder_center', 'head')
left_arm_joints = ('shoulder_left', 'elbow_left', 'wrist_left', 'hand_left')
right_arm_joints = ('shoulder_right', 'elbow_right', 'wrist_right', 'hand_right')
left_leg_joints = ('hip_left', 'knee_left', 'ankle_left', 'foot_left')
right_leg_joints = ('hip_right', 'knee_right', 'ankle_right', 'foot_right')
skeleton_joints = torso_joints + left_arm_joints + right_arm_joints + left_leg_joints + right_leg_joints

class Skeleton(namedtuple('Skeleton', ('timestamp', 'user_id') + skeleton_joints)):
    joints = skeleton_joints

    @property
    def to_np(self):
        l = []

        for j in self.joints:
            p = getattr(self, j).position
            l.append((p.x, p.y, p.z))

        return numpy.array(l)

Joint = namedtuple('Joint', ('position', 'orientation', 'pixel_coordinate'))

class KinectSensor(object):
    def __init__(self, addr, port):
        self._lock = threading.Lock()
        self._skeleton = None

        context = zmq.Context()
        self.socket = context.socket(zmq.REQ)
        self.socket.connect('tcp://{}:{}'.format(addr, port))

        t = threading.Thread(target=self.get_skeleton)
        t.daemon = True
        t.start()

    @property
    def tracked_skeleton(self):
        with self._lock:
            return self._skeleton

    @tracked_skeleton.setter
    def tracked_skeleton(self, skeleton):
        with self._lock:
            self._skeleton = skeleton

    def get_skeleton(self):
        while True:
            self.socket.send('Hello')

            md = self.socket.recv_json()
            msg = self.socket.recv()

            skeleton_array = numpy.frombuffer(buffer(msg), dtype=md['dtype'])
            skeleton_array = skeleton_array.reshape(md['shape'])

            joints = []
            for i in range(len(skeleton_joints)):
                x, y, z, w = skeleton_array[i][0:4]
                position = Point3D(x / w, y / w, z / w)
                pixel_coord = Point2D(*skeleton_array[i][4:6])
                orientation = Quaternion(*skeleton_array[i][6:10])
                joints.append(Joint(position, orientation, pixel_coord))

            self.tracked_skeleton = Skeleton(md['timestamp'], md['user_index'], *joints)

def draw_position(skel, ax):
    xy, zy = [], []

    if not skel:
        return

    for j in skeleton_joints:
        p = getattr(skel, j).position
        xy.append((p.x, p.y))
        zy.append((p.z, p.y))

    ax.set_xlim(-2, 5)
    ax.set_ylim(-1.5, 1.5)
    ax.scatter(zip(*xy)[0], zip(*xy)[1], 30, 'b')
    ax.scatter(zip(*zy)[0], zip(*zy)[1], 30, 'r')

if __name__ == '__main__':
    import time

    import matplotlib.pyplot as plt
    plt.ion()
    fig = plt.figure()
    ax = fig.add_subplot(111)

    kinect_sensor = KinectSensor('193.50.110.210', 9999)

    import skelangle

    kinect_angle = skelangle.AngleFromSkel()

    try:
        while True:
            ax.clear()
            draw_position(kinect_sensor.tracked_skeleton, ax)
            plt.draw()
            time.sleep(0.1)
    except KeyboardInterrupt:
        plt.close('all')


#!/usr/bin/env python

import os, sys, subprocess
import argparse
import subprocess
import threading
import timeit
from multiprocessing import Queue, Lock
from configobj import ConfigObj
from numpy import loadtxt
from numpy.linalg import inv
import matplotlib.pyplot as plt
import moving
from cvguipy import trajstorage, cvgenetic, cvconfig

"""
Grouping Calibration By Genetic Algorithm.
This script uses genetic algorithm to search for the best configuration.

It does not monitor RAM usage, therefore, CPU thrashing might be happened when number of parents (selection size) is too large.
"""
# class for genetic algorithm
class GeneticCompare(object):
    def __init__(self, motalist, motplist, IDlist, cfg_list, lock):
        self.motalist = motalist
        self.motplist = motplist
        self.IDlist = IDlist
        self.cfg_list = cfg_list
        self.lock = lock

    # This is used for calculte fitness of individual in genetic algorithn.
    # It is modified to create sqlite and cfg file before tuning computeClearMOT.
    # NOTE errors show up when loading two same ID
    def computeMOT(self, i):

        # create sqlite and cfg file with id i
        cfg_name = config_files +str(i)+'.cfg'
        sql_name = sqlite_files +str(i)+'.sqlite'
        open(cfg_name,'w').close()
        config = ConfigObj(cfg_name)
        cfg_list.write_config(i ,config)
        command = ['cp', 'tracking_only.sqlite', sql_name]
        process = subprocess.Popen(command)
        process.wait()
        command = ['trajextract.py', args.inputVideo, '-o', args.homography, '-t', cfg_name, '-d', sql_name, '--gf']
        # suppress output of grouping extraction
        devnull = open(os.devnull, 'wb')
        process = subprocess.Popen(command, stdout = devnull)
        process.wait()

        obj = trajstorage.CVsqlite(sql_name)
        print "loading", i
        obj.loadObjects()
        motp, mota, mt, mme, fpt, gt = moving.computeClearMOT(cdb.annotations, obj.objects, args.matchDistance, firstFrame, lastFrame)
        if motp is None:
            motp = 0
        self.lock.acquire()
        self.IDlist.put(i)
        self.motplist.put(motp)
        self.motalist.put(mota)
        obj.close()
        if args.PrintMOTA:
            print("ID: mota:{} motp:{}".format(mota, motp))
        self.lock.release()

        return mota

if __name__ == '__main__' :
    parser = argparse.ArgumentParser(description="compare all sqlites that are created by cfg_combination.py to the Annotated version to find the ID of the best configuration")
    parser.add_argument('inputVideo', help= "input video filename")
    parser.add_argument('-r', '--configuration-file', dest='range_cfg', help= "the configuration-file contain the range of configuration")
    parser.add_argument('-t', '--traffintel-config', dest='traffintelConfig', help= "the TrafficIntelligence file to use for running the first extraction.")
    parser.add_argument('-m', '--mask-File', dest='maskFilename', help="Name of the mask-File for trajextract")
    parser.add_argument('-d', '--database-file', dest ='databaseFile', help ="Name of the databaseFile.")
    parser.add_argument('-o', '--homography-file', dest ='homography', help = "Name of the homography file.", required = True)
    parser.add_argument('-md', '--matching-distance', dest='matchDistance', help = "matchDistance", default = 10, type = float)
    parser.add_argument('-a', '--accuracy', dest = 'accuracy', help = "accuracy parameter for genetic algorithm", type = int)
    parser.add_argument('-p', '--population', dest = 'population', help = "population parameter for genetic algorithm", required = True, type = int)
    parser.add_argument('-np', '--num-of-parents', dest = 'num_of_parents', help = "Number of parents that are selected each generation", type = int)
    parser.add_argument('-mota', '--print-MOTA', dest='PrintMOTA', action = 'store_true', help = "Print MOTA for each ID.")
    args = parser.parse_args()

    os.mkdir('cfg_files')
    os.mkdir('sql_files')
    sqlite_files = "sql_files/Sqlite_ID_"
    config_files = "cfg_files/Cfg_ID_"

    # ------------------initialize annotated version if not existed ---------- #
    # inputVideo check
    if not os.path.exists(args.inputVideo):
        print("Input video {} does not exist! Exiting...".format(args.inputVideo))
        sys.exit(1)

    # configuration file check
    if args.range_cfg is None:
        config = ConfigObj('range.cfg')
    else:
        config = ConfigObj(args.range_cfg)

    # get configuration and put them to a List
    cfg_list = cvconfig.CVConfigList()
    thread_cfgtolist = threading.Thread(target = cvconfig.config_to_list, args = (cfg_list, config))
    thread_cfgtolist.start();
    # check if dbfile name is entered
    if args.databaseFile is None:
        print("Database-file is not entered, running trajextract and cvplayer.")
        if not os.path.exists(args.homography):
            print("Homography file does not exist! Exiting...")
            sys.exit(1)
        else:
            videofile=args.inputVideo
            if 'avi' in videofile:
                if args.maskFilename is not None:
                    command = ['trajextract.py',args.inputVideo,'-m', args.maskFilename,'-o', args.homography]
                else:
                    command = ['trajextract.py',args.inputVideo,'-o', args.homography]
                process = subprocess.Popen(command)
                process.wait()
                databaseFile = videofile.replace('avi','sqlite')
                command = ['cvplayer.py',args.inputVideo,'-d',databaseFile,'-o',args.homography]
                process = subprocess.Popen(command)
                process.wait()
            else:
                print("Input video {} is not 'avi' type. Exiting...".format(args.inputVideo))
                sys.exit(1)
    else:
        databaseFile = args.databaseFile
    thread_cfgtolist.join()
    # ------------------Done initialization for annotation-------------------- #

    # create first tracking only database template.
    print("creating the first tracking only database template.")
    if args.maskFilename is not None:
        command = map(str, ['trajextract.py',args.inputVideo, '-d', 'tracking_only.sqlite', '-t', args.traffintelConfig, '-o', args.homography, '-m', args.maskFilename, '--tf'])
    else:
        command = map(str, ['trajextract.py',args.inputVideo, '-d', sql_name, '-t', args.traffintelConfig, '-o', args.homography, '--tf'])
    process = subprocess.Popen(command)
    process.wait()
    # ----start using genetic algorithm to search for best configuration-------#
    start = timeit.default_timer()

    dbfile = databaseFile;
    homography = loadtxt(args.homography)

    cdb = trajstorage.CVsqlite(dbfile)
    cdb.open()
    cdb.getLatestAnnotation()
    cdb.createBoundingBoxTable(cdb.latestannotations, inv(homography))
    cdb.loadAnnotaion()
    for a in cdb.annotations:
        a.computeCentroidTrajectory(homography)
    print "Latest Annotaions in "+dbfile+": ", cdb.latestannotations

    cdb.frameNumbers = cdb.getFrameList()
    firstFrame = cdb.frameNumbers[0]
    lastFrame = cdb.frameNumbers[-1]

    foundmota = Queue()
    foundmotp = Queue()
    IDs = Queue()
    lock = Lock()

    Comp = GeneticCompare(foundmota, foundmotp, IDs, cfg_list, lock)
    if args.accuracy != None:
        GeneticCal = cvgenetic.CVGenetic(args.population, cfg_list, Comp.computeMOT, args.accuracy)
    else:
        GeneticCal = cvgenetic.CVGenetic(args.population, cfg_list, Comp.computeMOT)
    if args.num_of_parents != None:
        GeneticCal.run_thread(args.num_of_parents)
    else:
        GeneticCal.run_thread()

    # tranform queues to lists
    foundmota = cvgenetic.Queue_to_list(foundmota)
    foundmotp = cvgenetic.Queue_to_list(foundmotp)
    IDs = cvgenetic.Queue_to_list(IDs)

    for i in range(len(foundmotp)):
        foundmotp[i] /= args.matchDistance
    Best_mota = max(foundmota)
    Best_ID = IDs[foundmota.index(Best_mota)]
    print "Best multiple object tracking accuracy (MOTA)", Best_mota
    print "ID:", Best_ID
    stop = timeit.default_timer()
    print str(stop-start) + "s"

    total = []
    for i in range(len(foundmota)):
        total.append(foundmota[i]- 0.1 * foundmotp[i])
    Best_total = max(total)
    Best_total_ID = IDs[total.index(Best_total)]
    # ------------------------------Done searching----------------------------#
    # use matplot to plot a graph of all calculated IDs along with thier mota
    plt.figure(1)
    plt.plot(foundmota ,IDs ,'bo')
    plt.plot(foundmotp ,IDs ,'yo')
    plt.plot(Best_mota, Best_ID, 'ro')
    plt.axis([-1, 1, -1, cfg_list.get_total_combination()])
    plt.xlabel('mota')
    plt.ylabel('ID')
    plt.title(b'Best MOTA: '+str(Best_mota) +'\nwith ID: '+str(Best_ID))
    plotFile = os.path.splitext(dbfile)[0] + '_CalibrationResult_mota.png'
    plt.savefig(plotFile)

    plt.figure(2)
    plt.plot(total, IDs, 'bo')
    plt.plot(Best_total, Best_total_ID, 'ro')
    plt.xlabel('mota + motp')
    plt.ylabel('ID')
    plt.title(b'Best total: '+str(Best_total) +'\nwith ID: '+str(Best_total_ID))

    # save the plot
    plotFile = os.path.splitext(dbfile)[0] + '_CalibrationResult_motp.png'
    plt.savefig(plotFile)

    plt.show()

    cdb.close()


"""
========================================
Ammonia inversion transition TKIN fitter
========================================

Ammonia inversion transition TKIN fitter translated from Erik Rosolowsky's
http://svn.ok.ubc.ca/svn/signals/nh3fit/

.. moduleauthor:: Adam Ginsburg <adam.g.ginsburg@gmail.com>

Module API
^^^^^^^^^^

"""
import numpy as np
from pyspeckit.mpfit import mpfit
from pyspeckit.spectrum.parinfo import ParinfoList,Parinfo
import fitter
import matplotlib.cbook as mpcb
import copy
import model

line_names = ['oneone','twotwo','threethree','fourfour']

freq_dict = {
    'oneone':     23.694506e9,
    'twotwo':     23.722633335e9,
    'threethree': 23.8701296e9,
    'fourfour':   24.1394169e9,
    }
aval_dict = {
    'oneone':     1.712e-7,  #64*!pi**4/(3*h*c**3)*nu11**3*mu0**2*(1/2.)
    'twotwo':     2.291e-7,  #64*!pi**4/(3*h*c**3)*nu22**3*mu0**2*(2/3.)
    'threethree': 2.625e-7,  #64*!pi**4/(3*h*c**3)*nu33**3*mu0**2*(3/4.)
    'fourfour':   3.167e-7,  #64*!pi**4/(3*h*c**3)*nu44**3*mu0**2*(4/5.)
    }
ortho_dict = {
    'oneone':     False,
    'twotwo':     False,
    'threethree': True,
    'fourfour':   False,
    }
n_ortho = np.arange(0,28,3) # 0..3..27
n_para = np.array([x for x in range(28) if x % 3 != 0])

voff_lines_dict = {
    'oneone': [19.8513, 19.3159, 7.88669, 7.46967, 7.35132, 0.460409, 0.322042,
        -0.0751680, -0.213003, 0.311034, 0.192266, -0.132382, -0.250923, -7.23349,
        -7.37280, -7.81526, -19.4117, -19.5500],
    'twotwo':[26.5263, 26.0111, 25.9505, 16.3917, 16.3793, 15.8642, 0.562503,
        0.528408, 0.523745, 0.0132820, -0.00379100, -0.0132820, -0.501831,
        -0.531340, -0.589080, -15.8547, -16.3698, -16.3822, -25.9505, -26.0111,
        -26.5263],
    'threethree':[29.195098, 29.044147, 28.941877, 28.911408, 21.234827,
        21.214619, 21.136387, 21.087456, 1.005122, 0.806082, 0.778062,
        0.628569, 0.016754, -0.005589, -0.013401, -0.639734, -0.744554,
        -1.031924, -21.125222, -21.203441, -21.223649, -21.076291, -28.908067,
        -28.938523, -29.040794, -29.191744],
    'fourfour':[  0.        , -30.49783692,  30.49783692,   0., 24.25907811,
        -24.25907811,   0.        ]
                      }

tau_wts_dict = {
    'oneone': [0.0740740, 0.148148, 0.0925930, 0.166667, 0.0185190, 0.0370370,
        0.0185190, 0.0185190, 0.0925930, 0.0333330, 0.300000, 0.466667,
        0.0333330, 0.0925930, 0.0185190, 0.166667, 0.0740740, 0.148148],
    'twotwo': [0.00418600, 0.0376740, 0.0209300, 0.0372090, 0.0260470,
        0.00186000, 0.0209300, 0.0116280, 0.0106310, 0.267442, 0.499668,
        0.146512, 0.0116280, 0.0106310, 0.0209300, 0.00186000, 0.0260470,
        0.0372090, 0.0209300, 0.0376740, 0.00418600],
    'threethree': [0.012263, 0.008409, 0.003434, 0.005494, 0.006652, 0.008852,
        0.004967, 0.011589, 0.019228, 0.010387, 0.010820, 0.009482, 0.293302,
        0.459109, 0.177372, 0.009482, 0.010820, 0.019228, 0.004967, 0.008852,
        0.006652, 0.011589, 0.005494, 0.003434, 0.008409, 0.012263],
    'fourfour': [0.2431, 0.0162, 0.0162, 0.3008, 0.0163, 0.0163, 0.3911]}

def ammonia(xarr, tkin=20, tex=None, ntot=1e14, width=1,
        xoff_v=0.0, fortho=0.0, tau=None, fillingfraction=None, return_tau=False,
        thin=False, verbose=False, return_components=False, debug=False ):
    """
    Generate a model Ammonia spectrum based on input temperatures, column, and
    gaussian parameters

    ntot can be specified as a column density (e.g., 10^15) or a log-column-density (e.g., 15)

    tex can be specified or can be assumed LTE if unspecified, if tex>tkin, or if "thin"
        is specified

    "thin" uses a different parametetrization and requires only the optical depth, width, offset,
        and tkin to be specified.  In the 'thin' approximation, tex is not used in computation of
        the partition function - LTE is implicitly assumed

    If tau is specified, ntot is NOT fit but is set to a fixed value
    fillingfraction is an arbitrary scaling factor to apply to the model
    fortho is the ortho/(ortho+para) fraction.  The default is to assume all ortho.
    xoff_v is the velocity offset in km/s

    tau refers to the optical depth of the 1-1 line.  The optical depths of the
    other lines are fixed relative to tau_oneone

    (not implemented) if tau is specified, ntot is ignored
    """

    # Convert X-units to frequency in GHz
    xarr = xarr.as_unit('GHz')

    if tex is not None:
        if tex > tkin: # cannot have Tex > Tkin
            tex = tkin
        elif thin: # tex is not used in this case
            tex = tkin
    else:
        tex = tkin

    if thin:
        ntot = 1e15
    elif 5 < ntot < 25:
        # allow ntot to be specified as a logarithm.  This is
        # safe because ntot < 1e10 gives a spectrum of all zeros, and the
        # plausible range of columns is not outside the specified range
        ntot = 10**ntot
    elif (25 < ntot < 1e5) or (ntot < 5):
        # these are totally invalid for log/non-log
        return 0

    # fillingfraction is an arbitrary scaling for the data
    # The model will be (normal model) * fillingfraction
    if fillingfraction is None:
        fillingfraction = 1.0

    ckms = 2.99792458e5
    ccms = ckms*1e5
    g1 = 1
    g2 = 1
    h = 6.6260693e-27
    kb = 1.3806505e-16
    mu0 = 1.476e-18               # Dipole Moment in cgs (1.476 Debeye)

    # Generate Partition Functions
    nlevs = 51
    jv=np.arange(nlevs)
    ortho = jv % 3 == 0
    para = True-ortho
    Jpara = jv[para]
    Jortho = jv[ortho]
    Brot = 298117.06e6
    Crot = 186726.36e6

    runspec = np.zeros(len(xarr))

    tau_dict = {}
    para_count = 0
    ortho_count = 1 # ignore 0-0

    if tau is not None and thin:
        """
        Use optical depth in the 1-1 line as a free parameter
        The optical depths of the other lines are then set by the kinetic temperature
        Tex is still a free parameter in the final spectrum calculation at the bottom
        (technically, I think this process assumes LTE; Tex should come into play in
        these equations, not just the final one)
        """
        dT0 = 41.5                    # Energy diff between (2,2) and (1,1) in K
        trot = tkin/(1+tkin/dT0*np.log(1+0.6*np.exp(-15.7/tkin)))
        tau_dict['oneone']     = tau
        tau_dict['twotwo']     = tau*(23.722/23.694)**2*4/3.*5/3.*np.exp(-41.5/trot)
        tau_dict['threethree'] = tau*(23.8701279/23.694)**2*3/2.*14./3.*np.exp(-101.1/trot)
        tau_dict['fourfour']   = tau*(24.1394169/23.694)**2*8/5.*9/3.*np.exp(-177.34/trot)
    else:
        """
        Column density is the free parameter.  It is used in conjunction with
        the full partition function to compute the optical depth in each band
        Given the complexity of these equations, it would be worth my while to
        comment each step carefully.
        """
        Zpara = (2*Jpara+1)*np.exp(-h*(Brot*Jpara*(Jpara+1)+
            (Crot-Brot)*Jpara**2)/(kb*tkin))
        Zortho = 2*(2*Jortho+1)*np.exp(-h*(Brot*Jortho*(Jortho+1)+
            (Crot-Brot)*Jortho**2)/(kb*tkin))
        for linename in line_names:
            if ortho_dict[linename]:
                orthoparafrac = fortho
                Z = Zortho
                count = ortho_count
                ortho_count += 1
            else:
                orthoparafrac = 1.0-fortho
                Z = Zpara
                count = para_count # need to treat partition function separately
                para_count += 1
            tau_dict[linename] = (ntot * orthoparafrac * Z[count]/(Z.sum()) / ( 1
                + np.exp(-h*freq_dict[linename]/(kb*tkin) )) * ccms**2 /
                (8*np.pi*freq_dict[linename]**2) * aval_dict[linename]*
                (1-np.exp(-h*freq_dict[linename]/(kb*tex))) /
                (width/ckms*freq_dict[linename]*np.sqrt(2*np.pi)) )

    # allow tau(11) to be specified instead of ntot
    # in the thin case, this is not needed: ntot plays no role
    # this process allows you to specify tau without using the approximate equations specified
    # above.  It should remove ntot from the calculations anyway...
    if tau is not None and not thin:
        tau11_temp = tau_dict['oneone']
        # re-scale all optical depths so that tau is as specified, but the relative taus
        # are sest by the kinetic temperature and partition functions
        for linename,t in tau_dict.iteritems():
            tau_dict[linename] = t * tau/tau11_temp

    components =[]
    for linename in line_names:
        voff_lines = np.array(voff_lines_dict[linename])
        tau_wts = np.array(tau_wts_dict[linename])

        lines = (1-voff_lines/ckms)*freq_dict[linename]/1e9
        tau_wts = tau_wts / (tau_wts).sum()
        nuwidth = np.abs(width/ckms*lines)
        nuoff = xoff_v/ckms*lines

        # tau array
        tauprof = np.zeros(len(xarr))
        for kk,no in enumerate(nuoff):
            tauprof += (tau_dict[linename] * tau_wts[kk] *
                    np.exp(-(xarr+no-lines[kk])**2 / (2.0*nuwidth[kk]**2)) *
                    fillingfraction)
            components.append( tauprof )

        T0 = (h*xarr*1e9/kb) # "temperature" of wavelength
        if tau is not None and thin:
            #runspec = tauprof+runspec
            # is there ever a case where you want to ignore the optical depth function? I think no
            runspec = (T0/(np.exp(T0/tex)-1)-T0/(np.exp(T0/2.73)-1))*(1-np.exp(-tauprof))+runspec
        else:
            runspec = (T0/(np.exp(T0/tex)-1)-T0/(np.exp(T0/2.73)-1))*(1-np.exp(-tauprof))+runspec
        if runspec.min() < 0:
            raise ValueError("Model dropped below zero.  That is not possible normally.  Here are the input values: "+
                    ("tex: %f " % tex) +
                    ("tkin: %f " % tkin) +
                    ("ntot: %f " % ntot) +
                    ("width: %f " % width) +
                    ("xoff_v: %f " % xoff_v) +
                    ("fortho: %f " % fortho)
                    )

    if verbose or debug:
        print "tkin: %g  tex: %g  ntot: %g  width: %g  xoff_v: %g  fortho: %g  fillingfraction: %g" % (tkin,tex,ntot,width,xoff_v,fortho,fillingfraction)

    if return_components:
        return (T0/(np.exp(T0/tex)-1)-T0/(np.exp(T0/2.73)-1))*(1-np.exp(-1*np.array(components)))

    if return_tau:
        return tau_dict

    return runspec

class ammonia_model(model.SpectralModel):

    def __init__(self,npeaks=1,npars=6,multisingle='multi',**kwargs):
        self.npeaks = npeaks
        self.npars = npars
        self._default_parnames = ['tkin','tex','ntot','width','xoff_v','fortho']
        self.parnames = copy.copy(self._default_parnames)

        # all fitters must have declared modelfuncs, which should take the fitted pars...
        self.modelfunc = ammonia
        self.n_modelfunc = self.n_ammonia

        # for fitting ammonia simultaneously with a flat background
        self.onepeakammonia = fitter.vheightmodel(ammonia)
        #self.onepeakammoniafit = self._fourparfitter(self.onepeakammonia)

        if multisingle in ('multi','single'):
            self.multisingle = multisingle
        else:
            raise Exception("multisingle must be multi or single")

        self.default_parinfo = None
        self.default_parinfo, kwargs = self._make_parinfo(**kwargs)

        # enforce ammonia-specific parameter limits
        for par in self.default_parinfo:
            if 'tex' in par.parname.lower():
                par.limited = (True,par.limited[1])
                par.limits = (max(par.limits[0],2.73), par.limits[1])
            if 'tkin' in par.parname.lower():
                par.limited = (True,par.limited[1])
                par.limits = (max(par.limits[0],2.73), par.limits[1])
            if 'width' in par.parname.lower():
                par.limited = (True,par.limited[1])
                par.limits = (max(par.limits[0],0), par.limits[1])
            if 'fortho' in par.parname.lower():
                par.limited = (True,True)
                if par.limits[1] != 0:
                    par.limits = (max(par.limits[0],0), min(par.limits[1],1))
                else:
                    par.limits = (max(par.limits[0],0), 1)
            if 'ntot' in par.parname.lower():
                par.limited = (True,par.limited[1])
                par.limits = (max(par.limits[0],0), par.limits[1])

        self.parinfo = copy.copy(self.default_parinfo)

        self.modelfunc_kwargs = kwargs
        # lower case? self.modelfunc_kwargs.update({'parnames':self.parinfo.parnames})

    def __call__(self,*args,**kwargs):
        #if 'use_lmfit' in kwargs: kwargs.pop('use_lmfit')
        use_lmfit = kwargs.pop('use_lmfit') if 'use_lmfit' in kwargs else self.use_lmfit
        if use_lmfit:
            return self.lmfitter(*args,**kwargs)
        if self.multisingle == 'single':
            return self.onepeakammoniafit(*args,**kwargs)
        elif self.multisingle == 'multi':
            return self.multinh3fit(*args,**kwargs)

    def n_ammonia(self, pars=None, parnames=None, **kwargs):
        """
        Returns a function that sums over N ammonia line profiles, where N is the length of
        tkin,tex,ntot,width,xoff_v,fortho *OR* N = len(pars) / 6

        The background "height" is assumed to be zero (you must "baseline" your
        spectrum before fitting)

        *pars* [ list ]
            a list with len(pars) = (6-nfixed)n, assuming
            tkin,tex,ntot,width,xoff_v,fortho repeated

        *parnames* [ list ]
            len(parnames) must = len(pars).  parnames determine how the ammonia
            function parses the arguments
        """
        if hasattr(pars,'values'):
            # important to treat as Dictionary, since lmfit params & parinfo both have .items
            parnames,parvals = zip(*pars.items())
            parnames = [p.lower() for p in parnames]
            parvals = [p.value for p in parvals]
        elif parnames is None:
            parvals = pars
            parnames = self.parnames
        else:
            parvals = pars
        if len(pars) != len(parnames):
            # this should only be needed when other codes are changing the number of peaks
            # during a copy, as opposed to letting them be set by a __call__
            # (n_modelfuncs = n_ammonia can be called directly)
            # n_modelfuncs doesn't care how many peaks there are
            if len(pars) % len(parnames) == 0:
                parnames = [p for ii in range(len(pars)/len(parnames)) for p in parnames]
                npars = len(parvals) / self.npeaks
            else:
                raise ValueError("Wrong array lengths passed to n_ammonia!")
        else:
            npars = len(parvals) / self.npeaks

        self._components = []
        def L(x):
            v = np.zeros(len(x))
            for jj in xrange(self.npeaks):
                modelkwargs = kwargs.copy()
                for ii in xrange(npars):
                    name = parnames[ii+jj*npars].strip('0123456789').lower()
                    modelkwargs.update({name:parvals[ii+jj*npars]})
                v += ammonia(x,**modelkwargs)
            return v
        return L

    def components(self, xarr, pars, hyperfine=False):
        """
        Ammonia components don't follow the default, since in Galactic astronomy the hyperfine components should be well-separated.
        If you want to see the individual components overlaid, you'll need to pass hyperfine to the plot_fit call
        """

        comps=[]
        for ii in xrange(self.npeaks):
            if hyperfine:
                modelkwargs = dict(zip(self.parnames[ii*self.npars:(ii+1)*self.npars],pars[ii*self.npars:(ii+1)*self.npars]))
                comps.append( ammonia(xarr,return_components=True,**modelkwargs) )
            else:
                modelkwargs = dict(zip(self.parnames[ii*self.npars:(ii+1)*self.npars],pars[ii*self.npars:(ii+1)*self.npars]))
                comps.append( [ammonia(xarr,return_components=False,**modelkwargs)] )

        modelcomponents = np.concatenate(comps)

        return modelcomponents

    def multinh3fit(self, xax, data, npeaks=1, err=None,
            params=(20,20,14,1.0,0.0,0.5),
            parnames=None,
            fixed=(False,False,False,False,False,False),
            limitedmin=(True,True,True,True,False,True),
            limitedmax=(False,False,False,False,False,True), minpars=(2.73,2.73,0,0,0,0),
            parinfo=None,
            maxpars=(0,0,0,0,0,1), quiet=True, shh=True, veryverbose=False, **kwargs):
        """
        Fit multiple nh3 profiles (multiple can be 1)

        Inputs:
           xax - x axis
           data - y axis
           npeaks - How many nh3 profiles to fit?  Default 1 (this could supersede onedgaussfit)
           err - error corresponding to data

         These parameters need to have length = 6*npeaks.  If npeaks > 1 and length = 6, they will
         be replicated npeaks times, otherwise they will be reset to defaults:
           params - Fit parameters: [tkin, tex, ntot (or tau), width, offset, ortho fraction] * npeaks
                  If len(params) % 6 == 0, npeaks will be set to len(params) / 6
           fixed - Is parameter fixed?
           limitedmin/minpars - set lower limits on each parameter (default: width>0, Tex and Tkin > Tcmb)
           limitedmax/maxpars - set upper limits on each parameter
           parnames - default parameter names, important for setting kwargs in model ['tkin','tex','ntot','width','xoff_v','fortho']

           quiet - should MPFIT output each iteration?
           shh - output final parameters?

        Returns:
           Fit parameters
           Model
           Fit errors
           chi2
        """

        if parinfo is None:
            self.npars = len(params) / npeaks

            if len(params) != npeaks and (len(params) / self.npars) > npeaks:
                npeaks = len(params) / self.npars
            self.npeaks = npeaks

            if isinstance(params,np.ndarray): params=params.tolist()
            # this is actually a hack, even though it's decently elegant
            # somehow, parnames was being changed WITHOUT being passed as a variable
            # this doesn't make sense - at all - but it happened.
            # (it is possible for self.parnames to have npars*npeaks elements where
            # npeaks > 1 coming into this function even though only 6 pars are specified;
            # _default_parnames is the workaround)
            if parnames is None: parnames = copy.copy(self._default_parnames)

            partype_dict = dict(zip(['params','parnames','fixed','limitedmin','limitedmax','minpars','maxpars'],
                    [params,parnames,fixed,limitedmin,limitedmax,minpars,maxpars]))

            # make sure all various things are the right length; if they're not, fix them using the defaults
            for partype,parlist in partype_dict.iteritems():
                if len(parlist) != self.npars*self.npeaks:
                    # if you leave the defaults, or enter something that can be multiplied by npars to get to the
                    # right number of gaussians, it will just replicate
                    if len(parlist) == self.npars:
                        partype_dict[partype] *= npeaks
                    elif len(parlist) > self.npars:
                        # DANGER:  THIS SHOULD NOT HAPPEN!
                        print "WARNING!  Input parameters were longer than allowed for variable ",parlist
                        partype_dict[partype] = partype_dict[partype][:self.npars]
                    elif parlist==params: # this instance shouldn't really be possible
                        partype_dict[partype] = [20,20,1e10,1.0,0.0,0.5] * npeaks
                    elif parlist==fixed:
                        partype_dict[partype] = [False] * len(params)
                    elif parlist==limitedmax: # only fortho, fillingfraction have upper limits
                        partype_dict[partype] = (np.array(parnames) == 'fortho') + (np.array(parnames) == 'fillingfraction')
                    elif parlist==limitedmin: # no physical values can be negative except velocity
                        partype_dict[partype] = (np.array(parnames) != 'xoff_v')
                    elif parlist==minpars: # all have minima of zero except kinetic temperature, which can't be below CMB.  Excitation temperature technically can be, but not in this model
                        partype_dict[partype] = ((np.array(parnames) == 'tkin') + (np.array(parnames) == 'tex')) * 2.73
                    elif parlist==maxpars: # fractions have upper limits of 1.0
                        partype_dict[partype] = ((np.array(parnames) == 'fortho') + (np.array(parnames) == 'fillingfraction')).astype('float')
                    elif parlist==parnames: # assumes the right number of parnames (essential)
                        partype_dict[partype] = list(parnames) * self.npeaks

            if len(parnames) != len(partype_dict['params']):
                raise ValueError("Wrong array lengths AFTER fixing them")

            # used in components.  Is this just a hack?
            self.parnames = partype_dict['parnames']

            parinfo = [ {'n':ii, 'value':partype_dict['params'][ii],
                'limits':[partype_dict['minpars'][ii],partype_dict['maxpars'][ii]],
                'limited':[partype_dict['limitedmin'][ii],partype_dict['limitedmax'][ii]], 'fixed':partype_dict['fixed'][ii],
                'parname':partype_dict['parnames'][ii]+str(ii/self.npars),
                'mpmaxstep':float(partype_dict['parnames'][ii] in ('tex','tkin')), # must force small steps in temperature (True = 1.0)
                'error': 0}
                for ii in xrange(len(partype_dict['params'])) ]

            # hack: remove 'fixed' pars
            parinfo_with_fixed = parinfo
            parinfo = [p for p in parinfo_with_fixed if not p['fixed']]
            fixed_kwargs = dict((p['parname'].strip("0123456789").lower(),p['value']) for p in parinfo_with_fixed if p['fixed'])
            # don't do this - it breaks the NEXT call because npars != len(parnames) self.parnames = [p['parname'] for p in parinfo]
            # this is OK - not a permanent change
            parnames = [p['parname'] for p in parinfo]
            # not OK self.npars = len(parinfo)/self.npeaks
            parinfo = ParinfoList([Parinfo(p) for p in parinfo], preserve_order=True)
            #import pdb; pdb.set_trace()
        else:
            self.parinfo = ParinfoList([Parinfo(p) for p in parinfo], preserve_order=True)
            parinfo_with_fixed = None
            fixed_kwargs = {}

        fitfun_kwargs = dict(kwargs.items()+fixed_kwargs.items())

        npars = len(parinfo)/self.npeaks

        # (fortho0 is not fortho)
        # this doesn't work if parinfo_with_fixed is not None:
        # this doesn't work     for p in parinfo_with_fixed:
        # this doesn't work         # users can change the defaults while holding them fixed
        # this doesn't work         if p['fixed']:
        # this doesn't work             kwargs.update({p['parname']:p['value']})

        def mpfitfun(x,y,err):
            if err is None:
                def f(p,fjac=None): return [0,(y-self.n_ammonia(pars=p, parnames=parinfo.parnames, **fitfun_kwargs)(x))]
            else:
                def f(p,fjac=None): return [0,(y-self.n_ammonia(pars=p, parnames=parinfo.parnames, **fitfun_kwargs)(x))/err]
            return f

        if veryverbose:
            print "GUESSES: "
            print "\n".join(["%s: %s" % (p['parname'],p['value']) for p in parinfo])

        mp = mpfit(mpfitfun(xax,data,err),parinfo=parinfo,quiet=quiet)
        mpp = mp.params
        if mp.perror is not None: mpperr = mp.perror
        else: mpperr = mpp*0
        chi2 = mp.fnorm

        if mp.status == 0:
            raise Exception(mp.errmsg)

        for i,p in enumerate(mpp):
            parinfo[i]['value'] = p
            parinfo[i]['error'] = mpperr[i]

        if not shh:
            print "Fit status: ",mp.status
            print "Fit message: ",mp.errmsg
            print "Final fit values: "
            for i,p in enumerate(mpp):
                print parinfo[i]['parname'],p," +/- ",mpperr[i]
            print "Chi2: ",mp.fnorm," Reduced Chi2: ",mp.fnorm/len(data)," DOF:",len(data)-len(mpp)

        if any(['tex' in s for s in parnames]) and any(['tkin' in s for s in parnames]):
            texnum = (i for i,s in enumerate(parnames) if 'tex' in s)
            tkinnum = (i for i,s in enumerate(parnames) if 'tkin' in s)
            for txn,tkn in zip(texnum,tkinnum):
                if mpp[txn] > mpp[tkn]: mpp[txn] = mpp[tkn]  # force Tex>Tkin to Tex=Tkin (already done in n_ammonia)
        self.mp = mp

        if parinfo_with_fixed is not None:
            # self self.parinfo preserving the 'fixed' parameters
            # ORDER MATTERS!
            for p in parinfo:
                parinfo_with_fixed[p['n']] = p
            self.parinfo = ParinfoList([Parinfo(p) for p in parinfo_with_fixed], preserve_order=True)
        else:
            self.parinfo = parinfo
            self.parinfo = ParinfoList([Parinfo(p) for p in parinfo], preserve_order=True)

        # I don't THINK these are necessary?
        #self.parinfo = parinfo
        #self.parinfo = ParinfoList([Parinfo(p) for p in self.parinfo])

        # need to restore the fixed parameters....
        # though the above commented out section indicates that I've done and undone this dozens of times now
        # (a test has been added to test_nh3.py)
        # this was NEVER included or tested because it breaks the order
        #for par in parinfo_with_fixed:
        #    if par.parname not in self.parinfo.keys():
        #        self.parinfo.append(par)

        self.mpp = self.parinfo.values
        self.mpperr = self.parinfo.errors
        self.mppnames = self.parinfo.names
        self.model = self.n_ammonia(pars=self.mpp, parnames=self.mppnames, **kwargs)(xax)
        #if self.model.sum() == 0:
        #    print "DON'T FORGET TO REMOVE THIS ERROR!"
        #    raise ValueError("Model is zeros.")

        indiv_parinfo = [self.parinfo[jj*self.npars:(jj+1)*self.npars] for jj in xrange(len(self.parinfo)/self.npars)]
        modelkwargs = [
                dict([(p['parname'].strip("0123456789").lower(),p['value']) for p in pi])
                for pi in indiv_parinfo]
        self.tau_list = [ammonia(xax,return_tau=True,**mk) for mk in modelkwargs]

        return self.mpp,self.model,self.mpperr,chi2

    def moments(self, Xax, data, negamp=None, veryverbose=False,  **kwargs):
        """
        Returns a very simple and likely incorrect guess
        """

        # TKIN, TEX, ntot, width, center, ortho fraction
        return [20,10, 1e15, 1.0, 0.0, 1.0]

    def annotations(self):
        from decimal import Decimal # for formatting
        tex_key = {'tkin':'T_K','tex':'T_{ex}','ntot':'N','fortho':'F_o','width':'\\sigma','xoff_v':'v','fillingfraction':'FF','tau':'\\tau_{1-1}'}
        # small hack below: don't quantize if error > value.  We want to see the values.
        label_list = []
        for pinfo in self.parinfo:
            parname = tex_key[pinfo['parname'].strip("0123456789").lower()]
            parnum = int(pinfo['parname'][-1])
            if pinfo['fixed']:
                formatted_value = "%s" % pinfo['value']
                pm = ""
                formatted_error=""
            else:
                formatted_value = Decimal("%g" % pinfo['value']).quantize(Decimal("%0.2g" % (min(pinfo['error'],pinfo['value']))))
                pm = "$\\pm$"
                formatted_error = Decimal("%g" % pinfo['error']).quantize(Decimal("%0.2g" % pinfo['error']))
            label =  "$%s(%i)$=%8s %s %8s" % (parname, parnum, formatted_value, pm, formatted_error)
            label_list.append(label)
        labels = tuple(mpcb.flatten(label_list))
        return labels

class ammonia_model_vtau(ammonia_model):
    def __init__(self,**kwargs):
        super(ammonia_model_vtau,self).__init__()
        self.parnames = ['tkin','tex','tau','width','xoff_v','fortho']

    def moments(self, Xax, data, negamp=None, veryverbose=False,  **kwargs):
        """
        Returns a very simple and likely incorrect guess
        """

        # TKIN, TEX, ntot, width, center, ortho fraction
        return [20,10, 1, 1.0, 0.0, 1.0]

    def __call__(self,*args,**kwargs):
        if self.multisingle == 'single':
            return self.onepeakammoniafit(*args,**kwargs)
        elif self.multisingle == 'multi':
            return self.multinh3fit(*args,**kwargs)


# -*- coding: utf-8 -*-
"""
Sphinx directive to support embedded IPython code.

This directive allows pasting of entire interactive IPython sessions, prompts
and all, and their code will actually get re-executed at doc build time, with
all prompts renumbered sequentially. It also allows you to input code as a pure
python input by giving the argument python to the directive. The output looks
like an interactive ipython section.

To enable this directive, simply list it in your Sphinx ``conf.py`` file
(making sure the directory where you placed it is visible to sphinx, as is
needed for all Sphinx directives). For example, to enable syntax highlighting
and the IPython directive::

    extensions = ['IPython.sphinxext.ipython_console_highlighting',
                  'IPython.sphinxext.ipython_directive']

The IPython directive outputs code-blocks with the language 'ipython'. So
if you do not have the syntax highlighting extension enabled as well, then
all rendered code-blocks will be uncolored. By default this directive assumes
that your prompts are unchanged IPython ones, but this can be customized.
The configurable options that can be placed in conf.py are:

ipython_savefig_dir:
    The directory in which to save the figures. This is relative to the
    Sphinx source directory. The default is `html_static_path`.
ipython_rgxin:
    The compiled regular expression to denote the start of IPython input
    lines. The default is re.compile('In \[(\d+)\]:\s?(.*)\s*'). You
    shouldn't need to change this.
ipython_rgxout:
    The compiled regular expression to denote the start of IPython output
    lines. The default is re.compile('Out\[(\d+)\]:\s?(.*)\s*'). You
    shouldn't need to change this.
ipython_promptin:
    The string to represent the IPython input prompt in the generated ReST.
    The default is 'In [%d]:'. This expects that the line numbers are used
    in the prompt.
ipython_promptout:
    The string to represent the IPython prompt in the generated ReST. The
    default is 'Out [%d]:'. This expects that the line numbers are used
    in the prompt.
ipython_mplbackend:
    The string which specifies if the embedded Sphinx shell should import
    Matplotlib and set the backend. The value specifies a backend that is
    passed to `matplotlib.use()` before any lines in `ipython_execlines` are
    executed. If not specified in conf.py, then the default value of 'agg' is
    used. To use the IPython directive without matplotlib as a dependency, set
    the value to `None`. It may end up that matplotlib is still imported
    if the user specifies so in `ipython_execlines` or makes use of the
    @savefig pseudo decorator.
ipython_execlines:
    A list of strings to be exec'd in the embedded Sphinx shell. Typical
    usage is to make certain packages always available. Set this to an empty
    list if you wish to have no imports always available. If specified in
    conf.py as `None`, then it has the effect of making no imports available.
    If omitted from conf.py altogether, then the default value of
    ['import numpy as np', 'import matplotlib.pyplot as plt'] is used.
ipython_holdcount
    When the @suppress pseudo-decorator is used, the execution count can be
    incremented or not. The default behavior is to hold the execution count,
    corresponding to a value of `True`. Set this to `False` to increment
    the execution count after each suppressed command.

As an example, to use the IPython directive when `matplotlib` is not available,
one sets the backend to `None`::

    ipython_mplbackend = None

An example usage of the directive is:

.. code-block:: rst

    .. ipython::

        In [1]: x = 1

        In [2]: y = x**2

        In [3]: print(y)

See http://matplotlib.org/sampledoc/ipython_directive.html for additional
documentation.

ToDo
----

- Turn the ad-hoc test() function into a real test suite.
- Break up ipython-specific functionality from matplotlib stuff into better
  separated code.

Authors
-------

- John D Hunter: orignal author.
- Fernando Perez: refactoring, documentation, cleanups, port to 0.11.
- VáclavŠmilauer <eudoxos-AT-arcig.cz>: Prompt generalizations.
- Skipper Seabold, refactoring, cleanups, pure python addition
"""
from __future__ import print_function
from __future__ import unicode_literals

#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------

# Stdlib
import os
import re
import sys
import tempfile
import ast
from pandas.compat import zip, range, map, lmap, u, cStringIO as StringIO
import warnings

# To keep compatibility with various python versions
try:
    from hashlib import md5
except ImportError:
    from md5 import md5

# Third-party
import sphinx
from docutils.parsers.rst import directives
from docutils import nodes
from sphinx.util.compat import Directive

# Our own
from IPython import Config, InteractiveShell
from IPython.core.profiledir import ProfileDir
from IPython.utils import io
from IPython.utils.py3compat import PY3

if PY3:
    from io import StringIO
    text_type = str
else:
    from StringIO import StringIO
    text_type = unicode

#-----------------------------------------------------------------------------
# Globals
#-----------------------------------------------------------------------------
# for tokenizing blocks
COMMENT, INPUT, OUTPUT =  range(3)

#-----------------------------------------------------------------------------
# Functions and class declarations
#-----------------------------------------------------------------------------

def block_parser(part, rgxin, rgxout, fmtin, fmtout):
    """
    part is a string of ipython text, comprised of at most one
    input, one ouput, comments, and blank lines.  The block parser
    parses the text into a list of::

      blocks = [ (TOKEN0, data0), (TOKEN1, data1), ...]

    where TOKEN is one of [COMMENT | INPUT | OUTPUT ] and
    data is, depending on the type of token::

      COMMENT : the comment string

      INPUT: the (DECORATOR, INPUT_LINE, REST) where
         DECORATOR: the input decorator (or None)
         INPUT_LINE: the input as string (possibly multi-line)
         REST : any stdout generated by the input line (not OUTPUT)

      OUTPUT: the output string, possibly multi-line

    """
    block = []
    lines = part.split('\n')
    N = len(lines)
    i = 0
    decorator = None
    while 1:

        if i==N:
            # nothing left to parse -- the last line
            break

        line = lines[i]
        i += 1
        line_stripped = line.strip()
        if line_stripped.startswith('#'):
            block.append((COMMENT, line))
            continue

        if line_stripped.startswith('@'):
            # we're assuming at most one decorator -- may need to
            # rethink
            decorator = line_stripped
            continue

        # does this look like an input line?
        matchin = rgxin.match(line)
        if matchin:
            lineno, inputline = int(matchin.group(1)), matchin.group(2)

            # the ....: continuation string
            continuation = '   %s:'%''.join(['.']*(len(str(lineno))+2))
            Nc = len(continuation)
            # input lines can continue on for more than one line, if
            # we have a '\' line continuation char or a function call
            # echo line 'print'.  The input line can only be
            # terminated by the end of the block or an output line, so
            # we parse out the rest of the input line if it is
            # multiline as well as any echo text

            rest = []
            while i<N:

                # look ahead; if the next line is blank, or a comment, or
                # an output line, we're done

                nextline = lines[i]
                matchout = rgxout.match(nextline)
                #print "nextline=%s, continuation=%s, starts=%s"%(nextline, continuation, nextline.startswith(continuation))
                if matchout or nextline.startswith('#'):
                    break
                elif nextline.startswith(continuation):
                    nextline = nextline[Nc:]
                    if nextline and nextline[0] == ' ':
                        nextline = nextline[1:]

                    inputline += '\n' +  nextline

                else:
                    rest.append(nextline)
                i+= 1

            block.append((INPUT, (decorator, inputline, '\n'.join(rest))))
            continue

        # if it looks like an output line grab all the text to the end
        # of the block
        matchout = rgxout.match(line)
        if matchout:
            lineno, output = int(matchout.group(1)), matchout.group(2)
            if i<N-1:
                output = '\n'.join([output] + lines[i:])

            block.append((OUTPUT, output))
            break

    return block

class DecodingStringIO(StringIO, object):
    def __init__(self,buf='',encodings=('utf8',), *args, **kwds):
        super(DecodingStringIO, self).__init__(buf, *args, **kwds)
        self.set_encodings(encodings)

    def set_encodings(self, encodings):
        self.encodings = encodings

    def write(self,data):
        if isinstance(data, text_type):
            return super(DecodingStringIO, self).write(data)
        else:
            for enc in self.encodings:
                try:
                    data = data.decode(enc)
                    return super(DecodingStringIO, self).write(data)
                except :
                    pass
        # default to brute utf8 if no encoding succeded
            return super(DecodingStringIO, self).write(data.decode('utf8', 'replace'))

class EmbeddedSphinxShell(object):
    """An embedded IPython instance to run inside Sphinx"""

    def __init__(self, exec_lines=None,state=None):

        self.cout = DecodingStringIO(u'')

        if exec_lines is None:
            exec_lines = []

        self.state = state

        # Create config object for IPython
        config = Config()
        config.InteractiveShell.autocall = False
        config.InteractiveShell.autoindent = False
        config.InteractiveShell.colors = 'NoColor'

        # create a profile so instance history isn't saved
        tmp_profile_dir = tempfile.mkdtemp(prefix='profile_')
        profname = 'auto_profile_sphinx_build'
        pdir = os.path.join(tmp_profile_dir,profname)
        profile = ProfileDir.create_profile_dir(pdir)

        # Create and initialize global ipython, but don't start its mainloop.
        # This will persist across different EmbededSphinxShell instances.
        IP = InteractiveShell.instance(config=config, profile_dir=profile)

        # io.stdout redirect must be done after instantiating InteractiveShell
        io.stdout = self.cout
        io.stderr = self.cout

        # For debugging, so we can see normal output, use this:
        #from IPython.utils.io import Tee
        #io.stdout = Tee(self.cout, channel='stdout') # dbg
        #io.stderr = Tee(self.cout, channel='stderr') # dbg

        # Store a few parts of IPython we'll need.
        self.IP = IP
        self.user_ns = self.IP.user_ns
        self.user_global_ns = self.IP.user_global_ns

        self.input = ''
        self.output = ''

        self.is_verbatim = False
        self.is_doctest = False
        self.is_suppress = False

        # Optionally, provide more detailed information to shell.
        self.directive = None

        # on the first call to the savefig decorator, we'll import
        # pyplot as plt so we can make a call to the plt.gcf().savefig
        self._pyplot_imported = False

        # Prepopulate the namespace.
        for line in exec_lines:
            self.process_input_line(line, store_history=False)

    def clear_cout(self):
        self.cout.seek(0)
        self.cout.truncate(0)

    def process_input_line(self, line, store_history=True):
        """process the input, capturing stdout"""

        stdout = sys.stdout
        splitter = self.IP.input_splitter
        try:
            sys.stdout = self.cout
            splitter.push(line)
            more = splitter.push_accepts_more()
            if not more:
                try:
                    source_raw = splitter.source_raw_reset()[1]
                except:
                    # recent ipython #4504
                    source_raw = splitter.raw_reset()
                self.IP.run_cell(source_raw, store_history=store_history)
        finally:
            sys.stdout = stdout

    def process_image(self, decorator):
        """
        # build out an image directive like
        # .. image:: somefile.png
        #    :width 4in
        #
        # from an input like
        # savefig somefile.png width=4in
        """
        savefig_dir = self.savefig_dir
        source_dir = self.source_dir
        saveargs = decorator.split(' ')
        filename = saveargs[1]
        # insert relative path to image file in source
        outfile = os.path.relpath(os.path.join(savefig_dir,filename),
                    source_dir)

        imagerows = ['.. image:: %s'%outfile]

        for kwarg in saveargs[2:]:
            arg, val = kwarg.split('=')
            arg = arg.strip()
            val = val.strip()
            imagerows.append('   :%s: %s'%(arg, val))

        image_file = os.path.basename(outfile) # only return file name
        image_directive = '\n'.join(imagerows)
        return image_file, image_directive

    # Callbacks for each type of token
    def process_input(self, data, input_prompt, lineno):
        """
        Process data block for INPUT token.

        """
        decorator, input, rest = data
        image_file = None
        image_directive = None

        is_verbatim = decorator=='@verbatim' or self.is_verbatim
        is_doctest = (decorator is not None and \
                     decorator.startswith('@doctest')) or self.is_doctest
        is_suppress = decorator=='@suppress' or self.is_suppress
        is_okexcept = decorator=='@okexcept' or self.is_okexcept
        is_okwarning = decorator=='@okwarning' or self.is_okwarning
        is_savefig = decorator is not None and \
                     decorator.startswith('@savefig')

        # set the encodings to be used by DecodingStringIO
        # to convert the execution output into unicode if
        # needed. this attrib is set by IpythonDirective.run()
        # based on the specified block options, defaulting to ['ut
        self.cout.set_encodings(self.output_encoding)

        input_lines = input.split('\n')

        if len(input_lines) > 1:
           if input_lines[-1] != "":
               input_lines.append('') # make sure there's a blank line
                                       # so splitter buffer gets reset

        continuation = '   %s:'%''.join(['.']*(len(str(lineno))+2))

        if is_savefig:
            image_file, image_directive = self.process_image(decorator)

        ret = []
        is_semicolon = False

        # Hold the execution count, if requested to do so.
        if is_suppress and self.hold_count:
            store_history = False
        else:
            store_history = True

        # Note: catch_warnings is not thread safe
        with warnings.catch_warnings(record=True) as ws:
            for i, line in enumerate(input_lines):
                if line.endswith(';'):
                    is_semicolon = True

                if i == 0:
                    # process the first input line
                    if is_verbatim:
                        self.process_input_line('')
                        self.IP.execution_count += 1 # increment it anyway
                    else:
                        # only submit the line in non-verbatim mode
                        self.process_input_line(line, store_history=store_history)
                    formatted_line = '%s %s'%(input_prompt, line)
                else:
                    # process a continuation line
                    if not is_verbatim:
                        self.process_input_line(line, store_history=store_history)

                    formatted_line = '%s %s'%(continuation, line)

                if not is_suppress:
                    ret.append(formatted_line)

        if not is_suppress and len(rest.strip()) and is_verbatim:
            # the "rest" is the standard output of the
            # input, which needs to be added in
            # verbatim mode
            ret.append(rest)

        self.cout.seek(0)
        output = self.cout.read()
        if not is_suppress and not is_semicolon:
            ret.append(output)
        elif is_semicolon: # get spacing right
            ret.append('')

        # context information
        filename = self.state.document.current_source
        lineno = self.state.document.current_line

        # output any exceptions raised during execution to stdout
        # unless :okexcept: has been specified.
        if not is_okexcept and "Traceback" in output:
            s =  "\nException in %s at block ending on line %s\n" % (filename, lineno)
            s += "Specify :okexcept: as an option in the ipython:: block to suppress this message\n"
            sys.stdout.write('\n\n>>>' + ('-' * 73))
            sys.stdout.write(s)
            sys.stdout.write(output)
            sys.stdout.write('<<<' + ('-' * 73) + '\n\n')

        # output any warning raised during execution to stdout
        # unless :okwarning: has been specified.
        if not is_okwarning:
            for w in ws:
                s =  "\nWarning in %s at block ending on line %s\n" % (filename, lineno)
                s += "Specify :okwarning: as an option in the ipython:: block to suppress this message\n"
                sys.stdout.write('\n\n>>>' + ('-' * 73))
                sys.stdout.write(s)
                sys.stdout.write('-' * 76 + '\n')
                s=warnings.formatwarning(w.message, w.category,
                                         w.filename, w.lineno, w.line)
                sys.stdout.write(s)
                sys.stdout.write('<<<' + ('-' * 73) + '\n')

        self.cout.truncate(0)
        return (ret, input_lines, output, is_doctest, decorator, image_file,
                    image_directive)

    def process_output(self, data, output_prompt,
                       input_lines, output, is_doctest, decorator, image_file):
        """
        Process data block for OUTPUT token.

        """
        TAB = ' ' * 4

        if is_doctest and output is not None:

            found = output
            found = found.strip()
            submitted = data.strip()

            if self.directive is None:
                source = 'Unavailable'
                content = 'Unavailable'
            else:
                source = self.directive.state.document.current_source
                content = self.directive.content
                # Add tabs and join into a single string.
                content = '\n'.join([TAB + line for line in content])

            # Make sure the output contains the output prompt.
            ind = found.find(output_prompt)
            if ind < 0:
                e = ('output does not contain output prompt\n\n'
                     'Document source: {0}\n\n'
                     'Raw content: \n{1}\n\n'
                     'Input line(s):\n{TAB}{2}\n\n'
                     'Output line(s):\n{TAB}{3}\n\n')
                e = e.format(source, content, '\n'.join(input_lines),
                             repr(found), TAB=TAB)
                raise RuntimeError(e)
            found = found[len(output_prompt):].strip()

            # Handle the actual doctest comparison.
            if decorator.strip() == '@doctest':
                # Standard doctest
                if found != submitted:
                    e = ('doctest failure\n\n'
                         'Document source: {0}\n\n'
                         'Raw content: \n{1}\n\n'
                         'On input line(s):\n{TAB}{2}\n\n'
                         'we found output:\n{TAB}{3}\n\n'
                         'instead of the expected:\n{TAB}{4}\n\n')
                    e = e.format(source, content, '\n'.join(input_lines),
                                 repr(found), repr(submitted), TAB=TAB)
                    raise RuntimeError(e)
            else:
                self.custom_doctest(decorator, input_lines, found, submitted)

    def process_comment(self, data):
        """Process data fPblock for COMMENT token."""
        if not self.is_suppress:
            return [data]

    def save_image(self, image_file):
        """
        Saves the image file to disk.
        """
        self.ensure_pyplot()
        command = ('plt.gcf().savefig("%s", bbox_inches="tight", '
                   'dpi=100)' % image_file)

        #print 'SAVEFIG', command  # dbg
        self.process_input_line('bookmark ipy_thisdir', store_history=False)
        self.process_input_line('cd -b ipy_savedir', store_history=False)
        self.process_input_line(command, store_history=False)
        self.process_input_line('cd -b ipy_thisdir', store_history=False)
        self.process_input_line('bookmark -d ipy_thisdir', store_history=False)
        self.clear_cout()

    def process_block(self, block):
        """
        process block from the block_parser and return a list of processed lines
        """
        ret = []
        output = None
        input_lines = None
        lineno = self.IP.execution_count

        input_prompt = self.promptin % lineno
        output_prompt = self.promptout % lineno
        image_file = None
        image_directive = None

        for token, data in block:
            if token == COMMENT:
                out_data = self.process_comment(data)
            elif token == INPUT:
                (out_data, input_lines, output, is_doctest, decorator,
                    image_file, image_directive) = \
                          self.process_input(data, input_prompt, lineno)
            elif token == OUTPUT:
                out_data = \
                    self.process_output(data, output_prompt,
                                        input_lines, output, is_doctest,
                                        decorator, image_file)
            if out_data:
                ret.extend(out_data)

        # save the image files
        if image_file is not None:
            self.save_image(image_file)

        return ret, image_directive

    def ensure_pyplot(self):
        """
        Ensures that pyplot has been imported into the embedded IPython shell.

        Also, makes sure to set the backend appropriately if not set already.

        """
        # We are here if the @figure pseudo decorator was used. Thus, it's
        # possible that we could be here even if python_mplbackend were set to
        # `None`. That's also strange and perhaps worthy of raising an
        # exception, but for now, we just set the backend to 'agg'.

        if not self._pyplot_imported:
            if 'matplotlib.backends' not in sys.modules:
                # Then ipython_matplotlib was set to None but there was a
                # call to the @figure decorator (and ipython_execlines did
                # not set a backend).
                #raise Exception("No backend was set, but @figure was used!")
                import matplotlib
                matplotlib.use('agg')

            # Always import pyplot into embedded shell.
            self.process_input_line('import matplotlib.pyplot as plt',
                                    store_history=False)
            self._pyplot_imported = True

    def process_pure_python(self, content):
        """
        content is a list of strings. it is unedited directive content

        This runs it line by line in the InteractiveShell, prepends
        prompts as needed capturing stderr and stdout, then returns
        the content as a list as if it were ipython code
        """
        output = []
        savefig = False # keep up with this to clear figure
        multiline = False # to handle line continuation
        multiline_start = None
        fmtin = self.promptin

        ct = 0

        for lineno, line in enumerate(content):

            line_stripped = line.strip()
            if not len(line):
                output.append(line)
                continue

            # handle decorators
            if line_stripped.startswith('@'):
                output.extend([line])
                if 'savefig' in line:
                    savefig = True # and need to clear figure
                continue

            # handle comments
            if line_stripped.startswith('#'):
                output.extend([line])
                continue

            # deal with lines checking for multiline
            continuation  = u'   %s:'% ''.join(['.']*(len(str(ct))+2))
            if not multiline:
                modified = u"%s %s" % (fmtin % ct, line_stripped)
                output.append(modified)
                ct += 1
                try:
                    ast.parse(line_stripped)
                    output.append(u'')
                except Exception: # on a multiline
                    multiline = True
                    multiline_start = lineno
            else: # still on a multiline
                modified = u'%s %s' % (continuation, line)
                output.append(modified)

                # if the next line is indented, it should be part of multiline
                if len(content) > lineno + 1:
                    nextline = content[lineno + 1]
                    if len(nextline) - len(nextline.lstrip()) > 3:
                        continue
                try:
                    mod = ast.parse(
                            '\n'.join(content[multiline_start:lineno+1]))
                    if isinstance(mod.body[0], ast.FunctionDef):
                        # check to see if we have the whole function
                        for element in mod.body[0].body:
                            if isinstance(element, ast.Return):
                                multiline = False
                    else:
                        output.append(u'')
                        multiline = False
                except Exception:
                    pass

            if savefig: # clear figure if plotted
                self.ensure_pyplot()
                self.process_input_line('plt.clf()', store_history=False)
                self.clear_cout()
                savefig = False

        return output

    def custom_doctest(self, decorator, input_lines, found, submitted):
        """
        Perform a specialized doctest.

        """
        from .custom_doctests import doctests

        args = decorator.split()
        doctest_type = args[1]
        if doctest_type in doctests:
            doctests[doctest_type](self, args, input_lines, found, submitted)
        else:
            e = "Invalid option to @doctest: {0}".format(doctest_type)
            raise Exception(e)

class IPythonDirective(Directive):

    has_content = True
    required_arguments = 0
    optional_arguments = 4 # python, suppress, verbatim, doctest
    final_argumuent_whitespace = True
    option_spec = { 'python': directives.unchanged,
                    'suppress' : directives.flag,
                    'verbatim' : directives.flag,
                    'doctest' : directives.flag,
                    'okexcept': directives.flag,
                    'okwarning': directives.flag,
                    'output_encoding': directives.unchanged_required
                  }

    shell = None

    seen_docs = set()

    def get_config_options(self):
        # contains sphinx configuration variables
        config = self.state.document.settings.env.config

        # get config variables to set figure output directory
        confdir = self.state.document.settings.env.app.confdir
        savefig_dir = config.ipython_savefig_dir
        source_dir = os.path.dirname(self.state.document.current_source)
        if savefig_dir is None:
            savefig_dir = config.html_static_path
        if isinstance(savefig_dir, list):
            savefig_dir = savefig_dir[0] # safe to assume only one path?
        savefig_dir = os.path.join(confdir, savefig_dir)

        # get regex and prompt stuff
        rgxin      = config.ipython_rgxin
        rgxout     = config.ipython_rgxout
        promptin   = config.ipython_promptin
        promptout  = config.ipython_promptout
        mplbackend = config.ipython_mplbackend
        exec_lines = config.ipython_execlines
        hold_count = config.ipython_holdcount

        return (savefig_dir, source_dir, rgxin, rgxout,
                promptin, promptout, mplbackend, exec_lines, hold_count)

    def setup(self):
        # Get configuration values.
        (savefig_dir, source_dir, rgxin, rgxout, promptin, promptout,
         mplbackend, exec_lines, hold_count) = self.get_config_options()

        if self.shell is None:
            # We will be here many times.  However, when the
            # EmbeddedSphinxShell is created, its interactive shell member
            # is the same for each instance.

            if mplbackend:
                import matplotlib
                # Repeated calls to use() will not hurt us since `mplbackend`
                # is the same each time.
                matplotlib.use(mplbackend)

            # Must be called after (potentially) importing matplotlib and
            # setting its backend since exec_lines might import pylab.
            self.shell = EmbeddedSphinxShell(exec_lines, self.state)

            # Store IPython directive to enable better error messages
            self.shell.directive = self

        # reset the execution count if we haven't processed this doc
        #NOTE: this may be borked if there are multiple seen_doc tmp files
        #check time stamp?
        if not self.state.document.current_source in self.seen_docs:
            self.shell.IP.history_manager.reset()
            self.shell.IP.execution_count = 1
            self.shell.IP.prompt_manager.width = 0
            self.seen_docs.add(self.state.document.current_source)

        # and attach to shell so we don't have to pass them around
        self.shell.rgxin = rgxin
        self.shell.rgxout = rgxout
        self.shell.promptin = promptin
        self.shell.promptout = promptout
        self.shell.savefig_dir = savefig_dir
        self.shell.source_dir = source_dir
        self.shell.hold_count = hold_count

        # setup bookmark for saving figures directory
        self.shell.process_input_line('bookmark ipy_savedir %s'%savefig_dir,
                                      store_history=False)
        self.shell.clear_cout()

        return rgxin, rgxout, promptin, promptout

    def teardown(self):
        # delete last bookmark
        self.shell.process_input_line('bookmark -d ipy_savedir',
                                      store_history=False)
        self.shell.clear_cout()

    def run(self):
        debug = False

        #TODO, any reason block_parser can't be a method of embeddable shell
        # then we wouldn't have to carry these around
        rgxin, rgxout, promptin, promptout = self.setup()

        options = self.options
        self.shell.is_suppress = 'suppress' in options
        self.shell.is_doctest = 'doctest' in options
        self.shell.is_verbatim = 'verbatim' in options
        self.shell.is_okexcept = 'okexcept' in options
        self.shell.is_okwarning = 'okwarning' in options

        self.shell.output_encoding = [options.get('output_encoding', 'utf8')]

        # handle pure python code
        if 'python' in self.arguments:
            content = self.content
            self.content = self.shell.process_pure_python(content)

        parts = '\n'.join(self.content).split('\n\n')

        lines = ['.. code-block:: ipython', '']
        figures = []

        for part in parts:
            block = block_parser(part, rgxin, rgxout, promptin, promptout)
            if len(block):
                rows, figure = self.shell.process_block(block)
                for row in rows:
                    lines.extend(['   %s'%line for line in row.split('\n')])

                if figure is not None:
                    figures.append(figure)

        for figure in figures:
            lines.append('')
            lines.extend(figure.split('\n'))
            lines.append('')

        if len(lines)>2:
            if debug:
                print('\n'.join(lines))
            else:
                # This has to do with input, not output. But if we comment
                # these lines out, then no IPython code will appear in the
                # final output.
                self.state_machine.insert_input(
                    lines, self.state_machine.input_lines.source(0))

        # cleanup
        self.teardown()

        return []

# Enable as a proper Sphinx directive
def setup(app):
    setup.app = app

    app.add_directive('ipython', IPythonDirective)
    app.add_config_value('ipython_savefig_dir', None, 'env')
    app.add_config_value('ipython_rgxin',
                         re.compile('In \[(\d+)\]:\s?(.*)\s*'), 'env')
    app.add_config_value('ipython_rgxout',
                         re.compile('Out\[(\d+)\]:\s?(.*)\s*'), 'env')
    app.add_config_value('ipython_promptin', 'In [%d]:', 'env')
    app.add_config_value('ipython_promptout', 'Out[%d]:', 'env')

    # We could just let matplotlib pick whatever is specified as the default
    # backend in the matplotlibrc file, but this would cause issues if the
    # backend didn't work in headless environments. For this reason, 'agg'
    # is a good default backend choice.
    app.add_config_value('ipython_mplbackend', 'agg', 'env')

    # If the user sets this config value to `None`, then EmbeddedSphinxShell's
    # __init__ method will treat it as [].
    execlines = ['import numpy as np', 'import matplotlib.pyplot as plt']
    app.add_config_value('ipython_execlines', execlines, 'env')

    app.add_config_value('ipython_holdcount', True, 'env')

# Simple smoke test, needs to be converted to a proper automatic test.
def test():

    examples = [
        r"""
In [9]: pwd
Out[9]: '/home/jdhunter/py4science/book'

In [10]: cd bookdata/
/home/jdhunter/py4science/book/bookdata

In [2]: from pylab import *

In [2]: ion()

In [3]: im = imread('stinkbug.png')

@savefig mystinkbug.png width=4in
In [4]: imshow(im)
Out[4]: <matplotlib.image.AxesImage object at 0x39ea850>

""",
        r"""

In [1]: x = 'hello world'

# string methods can be
# used to alter the string
@doctest
In [2]: x.upper()
Out[2]: 'HELLO WORLD'

@verbatim
In [3]: x.st<TAB>
x.startswith  x.strip
""",
    r"""

In [130]: url = 'http://ichart.finance.yahoo.com/table.csv?s=CROX\
   .....: &d=9&e=22&f=2009&g=d&a=1&br=8&c=2006&ignore=.csv'

In [131]: print url.split('&')
['http://ichart.finance.yahoo.com/table.csv?s=CROX', 'd=9', 'e=22', 'f=2009', 'g=d', 'a=1', 'b=8', 'c=2006', 'ignore=.csv']

In [60]: import urllib

""",
    r"""\

In [133]: import numpy.random

@suppress
In [134]: numpy.random.seed(2358)

@doctest
In [135]: numpy.random.rand(10,2)
Out[135]:
array([[ 0.64524308,  0.59943846],
       [ 0.47102322,  0.8715456 ],
       [ 0.29370834,  0.74776844],
       [ 0.99539577,  0.1313423 ],
       [ 0.16250302,  0.21103583],
       [ 0.81626524,  0.1312433 ],
       [ 0.67338089,  0.72302393],
       [ 0.7566368 ,  0.07033696],
       [ 0.22591016,  0.77731835],
       [ 0.0072729 ,  0.34273127]])

""",

    r"""
In [106]: print x
jdh

In [109]: for i in range(10):
   .....:     print i
   .....:
   .....:
0
1
2
3
4
5
6
7
8
9
""",

        r"""

In [144]: from pylab import *

In [145]: ion()

# use a semicolon to suppress the output
@savefig test_hist.png width=4in
In [151]: hist(np.random.randn(10000), 100);

@savefig test_plot.png width=4in
In [151]: plot(np.random.randn(10000), 'o');
   """,

        r"""
# use a semicolon to suppress the output
In [151]: plt.clf()

@savefig plot_simple.png width=4in
In [151]: plot([1,2,3])

@savefig hist_simple.png width=4in
In [151]: hist(np.random.randn(10000), 100);

""",
     r"""
# update the current fig
In [151]: ylabel('number')

In [152]: title('normal distribution')

@savefig hist_with_text.png
In [153]: grid(True)

@doctest float
In [154]: 0.1 + 0.2
Out[154]: 0.3

@doctest float
In [155]: np.arange(16).reshape(4,4)
Out[155]:
array([[ 0,  1,  2,  3],
       [ 4,  5,  6,  7],
       [ 8,  9, 10, 11],
       [12, 13, 14, 15]])

In [1]: x = np.arange(16, dtype=float).reshape(4,4)

In [2]: x[0,0] = np.inf

In [3]: x[0,1] = np.nan

@doctest float
In [4]: x
Out[4]:
array([[ inf,  nan,   2.,   3.],
       [  4.,   5.,   6.,   7.],
       [  8.,   9.,  10.,  11.],
       [ 12.,  13.,  14.,  15.]])

        """,
        ]
    # skip local-file depending first example:
    examples = examples[1:]

    #ipython_directive.DEBUG = True  # dbg
    #options = dict(suppress=True)  # dbg
    options = dict()
    for example in examples:
        content = example.split('\n')
        IPythonDirective('debug', arguments=None, options=options,
                          content=content, lineno=0,
                          content_offset=None, block_text=None,
                          state=None, state_machine=None,
                          )

# Run test suite as a script
if __name__=='__main__':
    if not os.path.isdir('_static'):
        os.mkdir('_static')
    test()
    print('All OK? Check figures in _static/')


from setuptools import setup
import os

here = os.path.abspath(os.path.dirname(__file__))

setup(
    name="BIDS2ISATab",

    # Versions should comply with PEP440.  For a discussion on single-sourcing
    # the version across setup.py and the project code, see
    # http://packaging.python.org/en/latest/tutorial.html#version
    version='0.1.0',

    description="Command line tool generating ISA-Tab compatible description from a Brain Imaging Data Structure "
                "compatible dataset.",
    long_description="Command line tool generating ISA-Tab compatible description from a Brain Imaging Data Structure "
                     "compatible dataset.",

    # The project URL.
    url='https://github.com/INCF/BIDS2ISATab',

    # Choose your license
    license='BSD',

    classifiers=[
        # How mature is this project? Common values are
        #   3 - Alpha
        #   4 - Beta
        #   5 - Production/Stable
        'Development Status :: 4 - Beta',

        # Pick your license as you wish (should match "license" above)
        'License :: OSI Approved :: BSD License',

        # Specify the Python versions you support here. In particular, ensure
        # that you indicate whether you support Python 2, Python 3 or both.
        'Programming Language :: Python :: 2.7',
        'Programming Language :: Python :: 3.5',
    ],

    # What does your project relate to?
    keywords='bids isatab',

    # You can just specify the packages manually here if your project is
    # simple. Or you can use find_packages.
    packages=["bids2isatab"],

    # List run-time dependencies here.  These will be installed by pip when your
    # project is installed.
    install_requires = ["future",
                        "pandas",
                        'nibabel'],

    include_package_data=True,

    # To provide executable scripts, use entry points in preference to the
    # "scripts" keyword. Entry points provide cross-platform support and allow
    # pip to create the appropriate form of executable for the target platform.
    entry_points={
        'console_scripts': [
            'bids2isatab=bids2isatab.main:main',
        ],
    },
)


#!/usr/bin/env python
__author__ = 'greg'
from sklearn.cluster import DBSCAN
import numpy as np
import math

def dist(c1,c2):
    return math.sqrt((c1[0]-c2[0])**2 + (c1[1]-c2[1])**2)

class CannotSplit(Exception):
    def __init__(self,value):
        self.value = value
    def __str__(self):
        return ""
samples_needed = 3

def adaptiveDBSCAN(XYpts,user_ids):
    if XYpts == []:
        return []

    pts_in_each_cluster = []
    users_in_each_cluster = []
    cluster_centers = []

    #increase the epsilon until we don't have any nearby clusters corresponding to non-overlapping
    #sets of users
    X = np.array(XYpts)
    #for epsilon in [5,10,15,20,25,30]:
    for first_epsilon in [100,200,300,400]:
        db = DBSCAN(eps=first_epsilon, min_samples=samples_needed).fit(X)

        labels = db.labels_
        pts_in_each_cluster = []
        users_in_each_cluster = []
        cluster_centers = []

        for k in sorted(set(labels)):
            if k == -1:
                continue

            class_member_mask = (labels == k)
            pts_in_cluster = list(X[class_member_mask])
            xSet,ySet = zip(*pts_in_cluster)

            cluster_centers.append((np.mean(xSet),np.mean(ySet)))
            pts_in_each_cluster.append(pts_in_cluster[:])
            users_in_each_cluster.append([u for u,l in zip(user_ids,labels) if l == k])

        #do we have any adjacent clusters with non-overlapping sets of users
        #if so, we should merge them by increasing the epsilon value
        cluster_compare = []
        for cluster_index, (c1,users) in enumerate(zip(cluster_centers,users_in_each_cluster)):
            for cluster_index, (c2,users2) in enumerate(zip(cluster_centers[cluster_index+1:],users_in_each_cluster[cluster_index+1:])):
                overlappingUsers = [u for u in users if u in users2]
                cluster_compare.append((dist(c1,c2),overlappingUsers))

        cluster_compare.sort(key = lambda x:x[0])
        needToMerge = [] in [c[1] for c in cluster_compare[:10]]
        if not(needToMerge):
            break
    #print epsilon
    #print [c[1] for c in cluster_compare[:10]]
    centers_to_return = []
    assert not(needToMerge)

    #do we need to split any clusters?
    for cluster_index in range(len(cluster_centers)):
        #print "splitting"
        needToSplit = (sorted(users_in_each_cluster[cluster_index]) != sorted(list(set(users_in_each_cluster[cluster_index]))))
        if needToSplit:
            subcluster_centers = []
            stillToSplit = []
            X = np.array(pts_in_each_cluster[cluster_index])
            #for epsilon in [30,25,20,15,10,5,1,0.1,0.01]:
            for second_epsilon in range(200,1,-2):#[400,300,200,100,80,75,65,60,50,25,24,23,22,21,20,19,18,17,16,15,14,13,10,5,1]:
                db = DBSCAN(eps=second_epsilon, min_samples=samples_needed).fit(X)

                labels = db.labels_
                subcluster_centers = []

                needToSplit = False

                for k in sorted(set(labels)):
                    if k == -1:
                        continue

                    class_member_mask = (labels == k)
                    users_in_subcluster = [u for u,l in zip(users_in_each_cluster[cluster_index],labels) if l == k]
                    needToSplit =  (sorted(users_in_subcluster) != sorted(list(set(users_in_subcluster))))
                    if needToSplit:
                        stillToSplit = list(X[class_member_mask])
                        break

                    pts_in_cluster = list(X[class_member_mask])
                    xSet,ySet = zip(*pts_in_cluster)
                    subcluster_centers.append((np.mean(xSet),np.mean(ySet)))

                if not(needToSplit):
                    break

            if needToSplit:
                print "second is " + str(second_epsilon)
                print stillToSplit
                for i in range(len(stillToSplit)):
                    p1 = stillToSplit[i]
                    for j in range(len(stillToSplit[i+1:])):
                        p2 = stillToSplit[j+i+1]
                        print math.sqrt((p1[0]-p2[0])**2 + (p1[1]-p2[1])**2),
                        #print (i,j+i+1),
                    print
                print X
                print users_in_each_cluster[cluster_index]
                raise CannotSplit(pts_in_each_cluster[cluster_index])
            centers_to_return.extend(subcluster_centers)

            #if needToSplit:
            #    print pts_in_each_cluster[cluster_index]
            #    print users_in_each_cluster[cluster_index]
            #else:

        else:
            centers_to_return.append(cluster_centers[cluster_index])

    return centers_to_return