"""
===================================================
Faces recognition example using eigenfaces and SVMs
===================================================

The dataset used in this example is a preprocessed excerpt of the
"Labeled Faces in the Wild", aka LFW_:

  http://vis-www.cs.umass.edu/lfw/lfw-funneled.tgz (233MB)

.. _LFW: http://vis-www.cs.umass.edu/lfw/

Expected results for the top 5 most represented people in the dataset::

                   precision    recall  f1-score   support

     Ariel Sharon       0.67      0.92      0.77        13
     Colin Powell       0.75      0.78      0.76        60
  Donald Rumsfeld       0.78      0.67      0.72        27
    George W Bush       0.86      0.86      0.86       146
Gerhard Schroeder       0.76      0.76      0.76        25
      Hugo Chavez       0.67      0.67      0.67        15
       Tony Blair       0.81      0.69      0.75        36

      avg / total       0.80      0.80      0.80       322

"""
from __future__ import print_function

from time import time
import logging
import matplotlib.pyplot as plt

from sklearn.cross_validation import train_test_split
from sklearn.datasets import fetch_lfw_people
from sklearn.grid_search import GridSearchCV
from sklearn.metrics import classification_report
from sklearn.metrics import confusion_matrix
from sklearn.decomposition import RandomizedPCA
from sklearn.svm import SVC

print(__doc__)

# Display progress logs on stdout
logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s')

###############################################################################
# Download the data, if not already on disk and load it as numpy arrays

lfw_people = fetch_lfw_people(min_faces_per_person=70, resize=0.4)

# introspect the images arrays to find the shapes (for plotting)
n_samples, h, w = lfw_people.images.shape

# for machine learning we use the 2 data directly (as relative pixel
# positions info is ignored by this model)
X = lfw_people.data
n_features = X.shape[1]

# the label to predict is the id of the person
y = lfw_people.target
target_names = lfw_people.target_names
n_classes = target_names.shape[0]

print("Total dataset size:")
print("n_samples: %d" % n_samples)
print("n_features: %d" % n_features)
print("n_classes: %d" % n_classes)

###############################################################################
# Split into a training set and a test set using a stratified k fold

# split into a training and testing set
X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.25, random_state=42)

###############################################################################
# Compute a PCA (eigenfaces) on the face dataset (treated as unlabeled
# dataset): unsupervised feature extraction / dimensionality reduction
n_components = 150

print("Extracting the top %d eigenfaces from %d faces"
      % (n_components, X_train.shape[0]))
t0 = time()
pca = RandomizedPCA(n_components=n_components, whiten=True).fit(X_train)
print("done in %0.3fs" % (time() - t0))

eigenfaces = pca.components_.reshape((n_components, h, w))

print("Projecting the input data on the eigenfaces orthonormal basis")
t0 = time()
X_train_pca = pca.transform(X_train)
X_test_pca = pca.transform(X_test)
print("done in %0.3fs" % (time() - t0))

###############################################################################
# Train a SVM classification model

print("Fitting the classifier to the training set")
t0 = time()
param_grid = {'C': [1e3, 5e3, 1e4, 5e4, 1e5],
              'gamma': [0.0001, 0.0005, 0.001, 0.005, 0.01, 0.1], }
clf = GridSearchCV(SVC(kernel='rbf', class_weight='balanced'), param_grid)
clf = clf.fit(X_train_pca, y_train)
print("done in %0.3fs" % (time() - t0))
print("Best estimator found by grid search:")
print(clf.best_estimator_)

###############################################################################
# Quantitative evaluation of the model quality on the test set

print("Predicting people's names on the test set")
t0 = time()
y_pred = clf.predict(X_test_pca)
print("done in %0.3fs" % (time() - t0))

print(classification_report(y_test, y_pred, target_names=target_names))
print(confusion_matrix(y_test, y_pred, labels=range(n_classes)))

###############################################################################
# Qualitative evaluation of the predictions using matplotlib

def plot_gallery(images, titles, h, w, n_row=3, n_col=4):
    """Helper function to plot a gallery of portraits"""
    plt.figure(figsize=(1.8 * n_col, 2.4 * n_row))
    plt.subplots_adjust(bottom=0, left=.01, right=.99, top=.90, hspace=.35)
    for i in range(n_row * n_col):
        plt.subplot(n_row, n_col, i + 1)
        plt.imshow(images[i].reshape((h, w)), cmap=plt.cm.gray)
        plt.title(titles[i], size=12)
        plt.xticks(())
        plt.yticks(())

# plot the result of the prediction on a portion of the test set

def title(y_pred, y_test, target_names, i):
    pred_name = target_names[y_pred[i]].rsplit(' ', 1)[-1]
    true_name = target_names[y_test[i]].rsplit(' ', 1)[-1]
    return 'predicted: %s\ntrue:      %s' % (pred_name, true_name)

prediction_titles = [title(y_pred, y_test, target_names, i)
                     for i in range(y_pred.shape[0])]

plot_gallery(X_test, prediction_titles, h, w)

# plot the gallery of the most significative eigenfaces

eigenface_titles = ["eigenface %d" % i for i in range(eigenfaces.shape[0])]
plot_gallery(eigenfaces, eigenface_titles, h, w)

plt.show()


import numbers

import numpy as np

from scipy import sparse
from sklearn.cluster import KMeans

from . import GraphCRF
from ..inference import inference_dispatch

def kmeans_init(X, Y, all_edges, n_labels, n_states_per_label,
                symmetric=True):
    all_feats = []
    # iterate over samples
    for x, y, edges in zip(X, Y, all_edges):
        # first, get neighbor counts from nodes
        n_nodes = x.shape[0]
        labels_one_hot = np.zeros((n_nodes, n_labels), dtype=np.int)
        y = y.ravel()
        gx = np.ogrid[:n_nodes]
        labels_one_hot[gx, y] = 1

        size = np.prod(y.shape)
        graphs = [sparse.coo_matrix((np.ones(e.shape[0]), e.T), (size, size))
                  for e in edges]
        if symmetric:
            directions = [g + g.T for g in graphs]
        else:
            directions = [T for g in graphs for T in [g, g.T]]
        neighbors = [s * labels_one_hot.reshape(size, -1) for s in directions]
        neighbors = np.hstack(neighbors)
        # normalize (for borders)
        neighbors /= np.maximum(neighbors.sum(axis=1)[:, np.newaxis], 1)

        # add unaries
        features = np.hstack([x, neighbors])
        all_feats.append(features)
    all_feats_stacked = np.vstack(all_feats)
    Y_stacked = np.hstack(Y).ravel()
    # for each state, run k-means over whole dataset
    H = [np.zeros(y.shape, dtype=np.int) for y in Y]
    label_indices = np.hstack([0, np.cumsum(n_states_per_label)])
    for label in np.unique(Y_stacked):
        try:
            km = KMeans(n_clusters=n_states_per_label[label])
        except TypeError:
            # for old versions :-/
            km = KMeans(k=n_states_per_label[label])
        indicator = Y_stacked == label
        f = all_feats_stacked[indicator]
        km.fit(f)
        for feats_sample, y, h in zip(all_feats, Y, H):
            indicator_sample = y.ravel() == label
            if np.any(indicator_sample):
                pred = km.predict(feats_sample[indicator_sample]).astype(np.int)
                h.ravel()[indicator_sample] = pred + label_indices[label]
    return H

class LatentGraphCRF(GraphCRF):
    """CRF with latent states for variables.

    This is also called "hidden dynamics CRF".
    For each output variable there is an additional variable which
    can encode additional states and interactions.

    Parameters
    ----------
    n_labels : int
        Number of states of output variables.

    n_featues : int or None (default=None).
        Number of input features per input variable.
        ``None`` means it is equal to ``n_labels``.

    n_states_per_label : int or list (default=2)
        Number of latent states associated with each observable state.
        Can be either an integer, which means the same number
        of hidden states will be used for all observable states, or a list
        of integers of length ``n_labels``.

    inference_method : string, default="ad3"
        Function to call to do inference and loss-augmented inference.
        Possible values are:

            - 'max-product' for max-product belief propagation.
                Recommended for chains an trees. Loopy belief propagatin in case of a general graph.
            - 'lp' for Linear Programming relaxation using cvxopt.
            - 'ad3' for AD3 dual decomposition.
            - 'qpbo' for QPBO + alpha expansion.
            - 'ogm' for OpenGM inference algorithms.
    """
    def __init__(self, n_labels=None, n_features=None, n_states_per_label=2,
                 inference_method=None):
        self.n_labels = n_labels
        self.n_states_per_label = n_states_per_label
        GraphCRF.__init__(self, n_states=None, n_features=n_features,
                          inference_method=inference_method)

    def _set_size_joint_feature(self):
        if None in [self.n_features, self.n_labels]:
            return

        if isinstance(self.n_states_per_label, numbers.Integral):
            # same for all labels
            n_states_per_label = np.array([
                self.n_states_per_label for i in range(self.n_labels)])
        else:
            n_states_per_label = np.array(self.n_states_per_label)
            if len(n_states_per_label) != self.n_labels:
                raise ValueError("states_per_label must be integer"
                                 "or array-like of length n_labels. Got %s"
                                 % str(n_states_per_label))
        self.n_states_per_label = n_states_per_label
        self.n_states = np.sum(n_states_per_label)

        # compute mapping from latent states to labels
        ranges = np.cumsum(n_states_per_label)
        states_map = np.zeros(self.n_states, dtype=np.int)
        for l in range(1, self.n_labels):
            states_map[ranges[l - 1]: ranges[l]] = l
        self._states_map = states_map
        GraphCRF._set_size_joint_feature(self)

    def initialize(self, X, Y):
        n_features = X[0][0].shape[1]
        if self.n_features is None:
            self.n_features = n_features
        elif self.n_features != n_features:
            raise ValueError("Expected %d features, got %d"
                             % (self.n_features, n_features))

        n_labels = len(np.unique(np.hstack([y.ravel() for y in Y])))
        if self.n_labels is None:
            self.n_labels = n_labels
        elif self.n_labels != n_labels:
            raise ValueError("Expected %d states, got %d"
                             % (self.n_labels, n_labels))
        self._set_size_joint_feature()
        self._set_class_weight()

    def label_from_latent(self, h):
        return self._states_map[h]

    def init_latent(self, X, Y):
        # treat all edges the same
        edges = [[self._get_edges(x)] for x in X]
        features = np.array([self._get_features(x) for x in X])
        return kmeans_init(features, Y, edges, n_labels=self.n_labels,
                           n_states_per_label=self.n_states_per_label)

    def loss_augmented_inference(self, x, h, w, relaxed=False,
                                 return_energy=False):
        self.inference_calls += 1
        self._check_size_w(w)
        unary_potentials = self._get_unary_potentials(x, w)
        pairwise_potentials = self._get_pairwise_potentials(x, w)
        edges = self._get_edges(x)
        # do loss-augmentation
        for l in np.arange(self.n_states):
            # for each class, decrement features
            # for loss-agumention
            unary_potentials[self.label_from_latent(h)
                             != self.label_from_latent(l), l] += 1.

        return inference_dispatch(unary_potentials, pairwise_potentials, edges,
                                  self.inference_method, relaxed=relaxed,
                                  return_energy=return_energy)

    def latent(self, x, y, w):
        unary_potentials = self._get_unary_potentials(x, w)
        # forbid h that is incompoatible with y
        # by modifying unary params
        other_states = self._states_map != y[:, np.newaxis]
        max_entry = np.maximum(np.max(unary_potentials), 1)
        unary_potentials[other_states] = -1e2 * max_entry
        pairwise_potentials = self._get_pairwise_potentials(x, w)
        edges = self._get_edges(x)
        h = inference_dispatch(unary_potentials, pairwise_potentials, edges,
                               self.inference_method, relaxed=False)
        if (self.label_from_latent(h) != y).any():
            print("inconsistent h and y")
            h = np.hstack([0, np.cumsum(self.n_states_per_label)])[y]
        return h

    def loss(self, h, h_hat):
        if isinstance(h_hat, tuple):
            return self.continuous_loss(h, h_hat[0])
        return GraphCRF.loss(self, self.label_from_latent(h),
                             self.label_from_latent(h_hat))

    def continuous_loss(self, y, y_hat):
        # continuous version of the loss
        # y_hat is the result of linear programming
        y_hat_org = np.zeros((y_hat.shape[0], self.n_labels))
        for s in range(self.n_states):
            y_hat_org[:, self._states_map[s]] += y_hat[:, s]
        y_org = self.label_from_latent(y)
        return GraphCRF.continuous_loss(self, y_org, y_hat_org)

    def base_loss(self, y, y_hat):
        if isinstance(y_hat, tuple):
            return GraphCRF.continuous_loss(self, y, y_hat)
        return GraphCRF.loss(self, y, y_hat)


# -*- coding: utf-8 -*-
"""
Created on Fri Sep 07 14:28:16 2018

@author: Alexander
"""

# -*- coding: utf-8 -*-
"""
Created on Fri Jul 27 15:01:09 2018

@author: abooth
"""

from xmlstats import xmlstats
import numpy as np
import pandas as pd

access_token = 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'
user_agent = 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'

stats = xmlstats.Xmlstats(access_token, user_agent)

datetest = "20171017"
standingsOnDate = stats.standings(date=datetest, sport="nba")

sum([standing.won for standing in standingsOnDate.standing]) #3

teamIds = np.sort(np.array([standing.team_id for standing in standingsOnDate.standing]))

teamWins = []
teamLosses = []
for teamId in teamIds:
    teamWins.extend([standing.won for standing in standingsOnDate.standing if standing.team_id == teamId])
    teamLosses.extend([standing.lost for standing in standingsOnDate.standing if standing.team_id == teamId])

df = pd.DataFrame(np.column_stack([teamIds,teamWins, teamLosses]),
                  columns = ['xmlstatsTeamId','Wins'+datetest, 'Losses'+datetest])

from datetime import timedelta, date
import time

#https://stackoverflow.com/questions/1060279/iterating-through-a-range-of-dates-in-python
def daterange(start_date, end_date):
    for n in range(int ((end_date - start_date).days)):
        yield start_date + timedelta(n)

seasonResults = pd.DataFrame({'xmlstatsTeamId':teamIds})
start_date = date(2017, 10, 17)
end_date = date(2018, 4, 12)
for single_date in daterange(start_date, end_date):
    date_format = single_date.strftime("%Y%m%d")
    standingsOnDate = stats.standings(date=date_format, sport="nba")
    teamWins = []
    teamLosses = []
    for teamId in teamIds:
        teamWins.extend([standing.won for standing in standingsOnDate.standing if standing.team_id == teamId])
        teamLosses.extend([standing.lost for standing in standingsOnDate.standing if standing.team_id == teamId])
    seasonResults["Wins_"+date_format] = teamWins
    seasonResults["Losses_"+date_format] = teamLosses
    print(date_format)
    time.sleep(12) #6 requests a minute

seasonResults.to_csv("nba2017SeasonResults.csv", index=False)


from setuptools import setup, find_packages

__version__ = eval(open('mitty/version.py').read().split('=')[1])
setup(
    name='mitty',
    version=__version__,
    description='Simulator for genomic data',
    author='Seven Bridges Genomics',
    author_email='kaushik.ghose@sbgenomics.com',
    packages=find_packages(include=['mitty*']),
    include_package_data=True,
    entry_points={
      # Register the built in plugins
      'mitty.plugins.sfs': ['double_exp = mitty.plugins.site_frequency.double_exp'],
      'mitty.plugins.variants': ['snp = mitty.plugins.variants.snp_plugin',
                                 'delete = mitty.plugins.variants.delete_plugin',
                                 'uniformdel = mitty.plugins.variants.uniform_deletions',
                                 'uniformins = mitty.plugins.variants.uniform_insertions',
                                 'insert = mitty.plugins.variants.insert_plugin',
                                 #'inversion = mitty.plugins.variants.inversion_plugin',
                                 #'low_entropy_insert = mitty.plugins.variants.low_entropy_insert_plugin'
                                 ],
      'mitty.plugins.population': ['standard = mitty.plugins.population.standard',
                                   'vn = mitty.plugins.population.vn'],
      'mitty.plugins.reads': ['simple_sequential = mitty.plugins.reads.simple_sequential_plugin',
                              'simple_illumina = mitty.plugins.reads.simple_illumina_plugin'],
      # Command line scripts
      'console_scripts': ['genomes = mitty.genomes:cli',
                          'reads = mitty.reads:cli',
                          'perfectbam = mitty.benchmarking.perfectbam:cli',
                          'badbams = mitty.benchmarking.badbams:cli',
                          'alindel = mitty.benchmarking.indel_alignment_accuracy:cli',
                          'benchsummary = mitty.benchmarking.benchmark_summary:cli',
                          'vcf2pop = mitty.lib.vcf2pop:cli',
                          'bam2tfq = mitty.benchmarking.convert_bam_to_truth_fastq:cli',
                          'alindel_plot = mitty.benchmarking.indel_alignment_accuracy_plot:cli',
                          'misplot = mitty.benchmarking.misalignment_plot:cli',
                          'acubam = mitty.benchmarking.bam_accuracy:cli',
                          'migratedb = mitty.util.db_migrate:cli',
                          'plot_gc_bias = mitty.util.plot_gc_bias:cli',
                          'splitta = mitty.util.splitta:cli',
                          'kmers = mitty.util.kmers:cli',
                          'pybwa = mitty.util.pybwa:cli']
    },
    install_requires=[
      'cython',
      'setuptools>=11.0.0',
      'numpy>=1.9.0',
      'docopt>=0.6.2',
      'click>=3.3',
      'pysam>=0.8.1',
      'h5py>=2.5.0',
      'matplotlib>=1.3.0',
      'scipy'
    ],
)


import sys
import warnings

import numpy as np
import pandas as pd

def parse_title(title, split_str=" "):
    """
    Parse the title of GSE13100 into usable metadata. Should work with pandas apply()
    Args:
        title:
        split_str:

    Returns:

    """
    split = title.split(split_str)
    meta = []
    if len(split) == 2:
        meta = [split[0], "NA", "NA", split[1]]
    elif len(split) == 8:
        meta = ['MUT', split[4], int(split[5].replace('t', "")), split[-1]]
    elif len(split) == 6:
        meta = ['WT', split[2], int(split[3].replace('t', "")), split[-1]]
    return pd.Series(meta, index=['condition', 'rna_type', 'time', 'rep'])

def mi_to_array(mi):
    labels = np.array(mRNA.columns.labels)
    x = np.array([lev[labels[ii]].values.tolist() for ii, lev in enumerate(mi.levels)])
    return x.T

if __name__ == '__main__':
    # Change output for easier reading
    pd.set_option('display.width', 200)

    # Parse R GEOQuery data into a pandas multiindex
    data = pd.read_csv("../data/GSE13100/GSE13100_BgCorrected_data.csv", index_col=0)
    row_info = pd.read_csv("../data/GSE13100/GSE13100_BgCorrected_rowinfo.csv").fillna("NA")
    col_info = pd.read_csv("../data/GSE13100/GSE13100_BgCorrected_colinfo.csv")

    # Compile the experiment information
    exp_info = col_info.title.apply(parse_title)
    exp_info.insert(0, 'geo', col_info.geo_accession.values)

    # Make sure the order matches the data
    data.sort_index(axis=1, ascending=True, inplace=True)
    exp_info.sort_values('geo', ascending=True, inplace=True)
    if not np.array_equal(data.columns.values, exp_info.geo.values):
        warnings.warn('Data columns and experimental info are not equal. Values may not match labels')

    # Make the columns a multiindex
    data.columns = pd.MultiIndex.from_arrays(exp_info.values.T.tolist(), names=exp_info.columns.values)
    data.sort_index(axis=1, inplace=True)

    # Select only the RA data and quantile normalize it. Log2 tranform will happen in pydiffexp pipeline
    idx = pd.IndexSlice
    mRNA = data.loc[:, idx[:, :, 'TR', :, :]]
    mRNA.to_pickle("../data/GSE13100/bgcorrected_GSE13100_TR_data.pkl")
    sys.exit()

    # Plot the distributions of the RA abundance
    info_idx = mi_to_array(mRNA.columns)
    fig, ax = plt.subplots(6, 7)
    ax_list = ax.flatten()
    for ii in range(mRNA.shape[1]):
        color = 'red' if info_idx[ii, 1] == 'MUT' else 'blue'
        title = 'time: {}, rep: {}'.format(info_idx[ii, 3], info_idx[ii, 4])
        ax_list[ii].hist(np.log2(mRNA.values[:, ii]+1), color=color)
        ax_list[ii].set_title(title)
    plt.show()

    # Pickle Data
    mRNA.to_pickle('../data/GSE13100/log2_bgcorrected_GSE13100_TR_data.pkl')


from itertools import product
import numpy as np
from numpy.testing import assert_almost_equal, assert_array_almost_equal

from sklearn import datasets
from sklearn import manifold
from sklearn import neighbors
from sklearn import pipeline
from sklearn import preprocessing
from sklearn.utils.testing import assert_less

eigen_solvers = ['auto', 'dense', 'arpack']
path_methods = ['auto', 'FW', 'D']

def test_isomap_simple_grid():
    # Isomap should preserve distances when all neighbors are used
    N_per_side = 5
    Npts = N_per_side ** 2
    n_neighbors = Npts - 1

    # grid of equidistant points in 2D, n_components = n_dim
    X = np.array(list(product(range(N_per_side), repeat=2)))

    # distances from each point to all others
    G = neighbors.kneighbors_graph(X, n_neighbors,
                                   mode='distance').toarray()

    for eigen_solver in eigen_solvers:
        for path_method in path_methods:
            clf = manifold.Isomap(n_neighbors=n_neighbors, n_components=2,
                                  eigen_solver=eigen_solver,
                                  path_method=path_method)
            clf.fit(X)

            G_iso = neighbors.kneighbors_graph(clf.embedding_,
                                               n_neighbors,
                                               mode='distance').toarray()
            assert_array_almost_equal(G, G_iso)

def test_isomap_reconstruction_error():
    # Same setup as in test_isomap_simple_grid, with an added dimension
    N_per_side = 5
    Npts = N_per_side ** 2
    n_neighbors = Npts - 1

    # grid of equidistant points in 2D, n_components = n_dim
    X = np.array(list(product(range(N_per_side), repeat=2)))

    # add noise in a third dimension
    rng = np.random.RandomState(0)
    noise = 0.1 * rng.randn(Npts, 1)
    X = np.concatenate((X, noise), 1)

    # compute input kernel
    G = neighbors.kneighbors_graph(X, n_neighbors,
                                   mode='distance').toarray()

    centerer = preprocessing.KernelCenterer()
    K = centerer.fit_transform(-0.5 * G ** 2)

    for eigen_solver in eigen_solvers:
        for path_method in path_methods:
            clf = manifold.Isomap(n_neighbors=n_neighbors, n_components=2,
                                  eigen_solver=eigen_solver,
                                  path_method=path_method)
            clf.fit(X)

            # compute output kernel
            G_iso = neighbors.kneighbors_graph(clf.embedding_,
                                               n_neighbors,
                                               mode='distance').toarray()

            K_iso = centerer.fit_transform(-0.5 * G_iso ** 2)

            # make sure error agrees
            reconstruction_error = np.linalg.norm(K - K_iso) / Npts
            assert_almost_equal(reconstruction_error,
                                clf.reconstruction_error())

def test_transform():
    n_samples = 200
    n_components = 10
    noise_scale = 0.01

    # Create S-curve dataset
    X, y = datasets.samples_generator.make_s_curve(n_samples)

    # Compute isomap embedding
    iso = manifold.Isomap(n_components, 2)
    X_iso = iso.fit_transform(X)

    # Re-embed a noisy version of the points
    rng = np.random.RandomState(0)
    noise = noise_scale * rng.randn(*X.shape)
    X_iso2 = iso.transform(X + noise)

    # Make sure the rms error on re-embedding is comparable to noise_scale
    assert_less(np.sqrt(np.mean((X_iso - X_iso2) ** 2)), 2 * noise_scale)

def test_pipeline():
    # check that Isomap works fine as a transformer in a Pipeline
    # only checks that no error is raised.
    # TODO check that it actually does something useful
    X, y = datasets.make_blobs(random_state=0)
    clf = pipeline.Pipeline(
        [('isomap', manifold.Isomap()),
         ('clf', neighbors.KNeighborsClassifier())])
    clf.fit(X, y)
    assert_less(.9, clf.score(X, y))

if __name__ == '__main__':
    import nose
    nose.runmodule()


import os

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd

data = pd.read_csv(os.path.join(os.path.dirname(__file__), 'benchmarks.csv')).sort_values('mean')

def grouped_barplot(df, cat, subcat, val, err, subcats=None, **kwargs):
    # based on https://stackoverflow.com/a/42033734
    categories = df[cat].unique()
    x = np.arange(len(categories))
    subcats = subcats or df[subcat].unique()
    offsets = (np.arange(len(subcats)) - np.arange(len(subcats)).mean()) / (len(subcats) + 1.)
    width = np.diff(offsets).mean()
    for i, gr in enumerate(subcats):
        dfg = df[df[subcat] == gr]
        plt.bar(x + offsets[i], dfg[val].values, width=width,
                label="{}".format(gr), yerr=dfg[err].values, capsize=6, **kwargs)
    plt.xlabel(cat)
    plt.ylabel(val)
    plt.xticks(x, categories)
    plt.legend(title=subcat, loc='center left', bbox_to_anchor=(1, 0.5))

def plot_benchmarks(toolings=None):
    plt.figure(dpi=120, figsize=(10, 5))
    grouped_barplot(data, cat='formula', subcat='tooling', val='mean', err='stderr', subcats=toolings, log=True)
    plt.ylim(1e-2, None)
    plt.grid()
    plt.gca().set_axisbelow(True)
    plt.ylabel("Mean Time (s)")
    plt.xlabel("Formula")
    plt.tight_layout()

plot_benchmarks(toolings=['formulaic', 'R', 'patsy', 'formulaic_sparse', 'R_sparse'])
plt.savefig(os.path.join(os.path.dirname(__file__), 'benchmarks.png'))


"""
=======================================
Clustering text documents using k-means
=======================================

This is an example showing how the scikit-learn can be used to cluster
documents by topics using a bag-of-words approach. This example uses
a scipy.sparse matrix to store the features instead of standard numpy arrays.

Two feature extraction methods can be used in this example:

  - TfidfVectorizer uses a in-memory vocabulary (a python dict) to map the most
    frequent words to features indices and hence compute a word occurrence
    frequency (sparse) matrix. The word frequencies are then reweighted using
    the Inverse Document Frequency (IDF) vector collected feature-wise over
    the corpus.

  - HashingVectorizer hashes word occurrences to a fixed dimensional space,
    possibly with collisions. The word count vectors are then normalized to
    each have l2-norm equal to one (projected to the euclidean unit-ball) which
    seems to be important for k-means to work in high dimensional space.

    HashingVectorizer does not provide IDF weighting as this is a stateless
    model (the fit method does nothing). When IDF weighting is needed it can
    be added by pipelining its output to a TfidfTransformer instance.

Two algorithms are demoed: ordinary k-means and its more scalable cousin
minibatch k-means.

It can be noted that k-means (and minibatch k-means) are very sensitive to
feature scaling and that in this case the IDF weighting helps improve the
quality of the clustering by quite a lot as measured against the "ground truth"
provided by the class label assignments of the 20 newsgroups dataset.

This improvement is not visible in the Silhouette Coefficient which is small
for both as this measure seem to suffer from the phenomenon called
"Concentration of Measure" or "Curse of Dimensionality" for high dimensional
datasets such as text data. Other measures such as V-measure and Adjusted Rand
Index are information theoretic based evaluation scores: as they are only based
on cluster assignments rather than distances, hence not affected by the curse
of dimensionality.

Note: as k-means is optimizing a non-convex objective function, it will likely
end up in a local optimum. Several runs with independent random init might be
necessary to get a good convergence.

"""

# Author: Peter Prettenhofer <peter.prettenhofer@gmail.com>
#         Lars Buitinck <L.J.Buitinck@uva.nl>
# License: BSD 3 clause

from __future__ import print_function

from sklearn.datasets import fetch_20newsgroups
from sklearn.decomposition import TruncatedSVD
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.feature_extraction.text import HashingVectorizer
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import Normalizer
from sklearn import metrics

from sklearn.cluster import KMeans, MiniBatchKMeans

import logging
from optparse import OptionParser
import sys
from time import time

import numpy as np

# Display progress logs on stdout
logging.basicConfig(level=logging.INFO,
                    format='%(asctime)s %(levelname)s %(message)s')

# parse commandline arguments
op = OptionParser()
op.add_option("--lsa",
              dest="n_components", type="int",
              help="Preprocess documents with latent semantic analysis.")
op.add_option("--no-minibatch",
              action="store_false", dest="minibatch", default=True,
              help="Use ordinary k-means algorithm (in batch mode).")
op.add_option("--no-idf",
              action="store_false", dest="use_idf", default=True,
              help="Disable Inverse Document Frequency feature weighting.")
op.add_option("--use-hashing",
              action="store_true", default=False,
              help="Use a hashing feature vectorizer")
op.add_option("--n-features", type=int, default=10000,
              help="Maximum number of features (dimensions)"
                   " to extract from text.")
op.add_option("--verbose",
              action="store_true", dest="verbose", default=False,
              help="Print progress reports inside k-means algorithm.")

print(__doc__)
op.print_help()

(opts, args) = op.parse_args()
if len(args) > 0:
    op.error("this script takes no arguments.")
    sys.exit(1)

###############################################################################
# Load some categories from the training set
categories = [
    'alt.atheism',
    'talk.religion.misc',
    'comp.graphics',
    'sci.space',
]
# Uncomment the following to do the analysis on all the categories
#categories = None

print("Loading 20 newsgroups dataset for categories:")
print(categories)

dataset = fetch_20newsgroups(subset='all', categories=categories,
                             shuffle=True, random_state=42)

print("%d documents" % len(dataset.data))
print("%d categories" % len(dataset.target_names))
print()

labels = dataset.target
true_k = np.unique(labels).shape[0]

print("Extracting features from the training dataset using a sparse vectorizer")
t0 = time()
if opts.use_hashing:
    if opts.use_idf:
        # Perform an IDF normalization on the output of HashingVectorizer
        hasher = HashingVectorizer(n_features=opts.n_features,
                                   stop_words='english', non_negative=True,
                                   norm=None, binary=False)
        vectorizer = make_pipeline(hasher, TfidfTransformer())
    else:
        vectorizer = HashingVectorizer(n_features=opts.n_features,
                                       stop_words='english',
                                       non_negative=False, norm='l2',
                                       binary=False)
else:
    vectorizer = TfidfVectorizer(max_df=0.5, max_features=opts.n_features,
                                 min_df=2, stop_words='english',
                                 use_idf=opts.use_idf)
X = vectorizer.fit_transform(dataset.data)

print("done in %fs" % (time() - t0))
print("n_samples: %d, n_features: %d" % X.shape)
print()

if opts.n_components:
    print("Performing dimensionality reduction using LSA")
    t0 = time()
    # Vectorizer results are normalized, which makes KMeans behave as
    # spherical k-means for better results. Since LSA/SVD results are
    # not normalized, we have to redo the normalization.
    svd = TruncatedSVD(opts.n_components)
    lsa = make_pipeline(svd, Normalizer(copy=False))

    X = lsa.fit_transform(X)

    print("done in %fs" % (time() - t0))

    explained_variance = svd.explained_variance_ratio_.sum()
    print("Explained variance of the SVD step: {}%".format(
        int(explained_variance * 100)))

    print()

###############################################################################
# Do the actual clustering

if opts.minibatch:
    km = MiniBatchKMeans(n_clusters=true_k, init='k-means++', n_init=1,
                         init_size=1000, batch_size=1000, verbose=opts.verbose)
else:
    km = KMeans(n_clusters=true_k, init='k-means++', max_iter=100, n_init=1,
                verbose=opts.verbose)

print("Clustering sparse data with %s" % km)
t0 = time()
km.fit(X)
print("done in %0.3fs" % (time() - t0))
print()

print("Homogeneity: %0.3f" % metrics.homogeneity_score(labels, km.labels_))
print("Completeness: %0.3f" % metrics.completeness_score(labels, km.labels_))
print("V-measure: %0.3f" % metrics.v_measure_score(labels, km.labels_))
print("Adjusted Rand-Index: %.3f"
      % metrics.adjusted_rand_score(labels, km.labels_))
print("Silhouette Coefficient: %0.3f"
      % metrics.silhouette_score(X, km.labels_, sample_size=1000))

print()

if not (opts.n_components or opts.use_hashing):
    print("Top terms per cluster:")
    order_centroids = km.cluster_centers_.argsort()[:, ::-1]
    terms = vectorizer.get_feature_names()
    for i in range(true_k):
        print("Cluster %d:" % i, end='')
        for ind in order_centroids[i, :10]:
            print(' %s' % terms[ind], end='')
        print()


"""
To run this, you'll need to have installed.

  * scikit-learn

Does two benchmarks

First, we fix a training set, increase the number of
samples to classify and plot number of classified samples as a
function of time.

In the second benchmark, we increase the number of dimensions of the
training set, classify a sample and plot the time taken as a function
of the number of dimensions.
"""
import numpy as np
import pylab as pl
import gc
from datetime import datetime

# to store the results
scikit_classifier_results = []
scikit_regressor_results = []

mu_second = 0.0 + 10 ** 6  # number of microseconds in a second

def bench_scikit_tree_classifier(X, Y):
    """Benchmark with scikit-learn decision tree classifier"""

    from sklearn.tree import DecisionTreeClassifier

    gc.collect()

    # start time
    tstart = datetime.now()
    clf = DecisionTreeClassifier()
    clf.fit(X, Y).predict(X)
    delta = (datetime.now() - tstart)
    # stop time

    scikit_classifier_results.append(
        delta.seconds + delta.microseconds / mu_second)

def bench_scikit_tree_regressor(X, Y):
    """Benchmark with scikit-learn decision tree regressor"""

    from sklearn.tree import DecisionTreeRegressor

    gc.collect()

    # start time
    tstart = datetime.now()
    clf = DecisionTreeRegressor()
    clf.fit(X, Y).predict(X)
    delta = (datetime.now() - tstart)
    # stop time

    scikit_regressor_results.append(
        delta.seconds + delta.microseconds / mu_second)

if __name__ == '__main__':

    print('============================================')
    print('Warning: this is going to take a looong time')
    print('============================================')

    n = 10
    step = 10000
    n_samples = 10000
    dim = 10
    n_classes = 10
    for i in range(n):
        print('============================================')
        print('Entering iteration %s of %s' % (i, n))
        print('============================================')
        n_samples += step
        X = np.random.randn(n_samples, dim)
        Y = np.random.randint(0, n_classes, (n_samples,))
        bench_scikit_tree_classifier(X, Y)
        Y = np.random.randn(n_samples)
        bench_scikit_tree_regressor(X, Y)

    xx = range(0, n * step, step)
    pl.figure('scikit-learn tree benchmark results')
    pl.subplot(211)
    pl.title('Learning with varying number of samples')
    pl.plot(xx, scikit_classifier_results, 'g-', label='classification')
    pl.plot(xx, scikit_regressor_results, 'r-', label='regression')
    pl.legend(loc='upper left')
    pl.xlabel('number of samples')
    pl.ylabel('Time (s)')

    scikit_classifier_results = []
    scikit_regressor_results = []
    n = 10
    step = 500
    start_dim = 500
    n_classes = 10

    dim = start_dim
    for i in range(0, n):
        print('============================================')
        print('Entering iteration %s of %s' % (i, n))
        print('============================================')
        dim += step
        X = np.random.randn(100, dim)
        Y = np.random.randint(0, n_classes, (100,))
        bench_scikit_tree_classifier(X, Y)
        Y = np.random.randn(100)
        bench_scikit_tree_regressor(X, Y)

    xx = np.arange(start_dim, start_dim + n * step, step)
    pl.subplot(212)
    pl.title('Learning in high dimensional spaces')
    pl.plot(xx, scikit_classifier_results, 'g-', label='classification')
    pl.plot(xx, scikit_regressor_results, 'r-', label='regression')
    pl.legend(loc='upper left')
    pl.xlabel('number of dimensions')
    pl.ylabel('Time (s)')
    pl.axis('tight')
    pl.show()


# Created by Eugene M. Izhikevich, 2003 Modified by S. Fusi 2007
# Ported to Python by Eilif Muller, 2008.
#
# Notes:
#
# Requires matplotlib,ipython,numpy>=1.0.3
# On a debian/ubuntu based system:
# $ apt-get install python-matplotlib python-numpy ipython
#
# Start ipython with threaded plotting support:
# $ ipython -pylab
#
# At the resulting prompt, run the file by:
# In [1]: execfile('smallnet.py')

# Modules required
import numpy
import numpy.random as random
import acml_rng
# Bug fix for numpy version 1.0.4
numpy.lib.function_base.any = numpy.any

# For measuring performance
import time
t1 = time.time()

# Excitatory and inhibitory neuron counts
Ne = 1000
Ni = 4
N = Ne+Ni

# Synaptic couplings
Je = 250.0/Ne
Ji = 0.0

# reset depolarization (mV)
reset = 0.0

# refractory period (ms)
refr = 2.5

# Synaptic couplings (mV)
S = numpy.zeros((N,N))
S[:,:Ne] = Je*random.uniform(size=(N,Ne))
S[:,:Ni] = -Ji*random.uniform(size=(N,Ni))

# Connectivity
S[:,:Ne][random.uniform(size=(N,Ne))-0.9<=0.0]=0.0
S[:,Ne:][random.uniform(size=(N,Ni))-0.9<=0.0]=0.0

# (mV/ms) (lambda is a python keyword)
leak = 5.0
dt = 0.05
sdt = numpy.sqrt(dt)

# Statistics of the background external current
mb = 3.0; sb = 4.0
mue = mb; sigmae=sb

sigmai = 0.0

# State variable v, initial value of 0
v = numpy.zeros(N)

# Refractory period state variable
r = numpy.zeros(N)

# Spike timings in a list
firings = []
spikes = [[]]*N

print 'mu(nu=5Hz)=%f' % (mb+Ne*Je*.015-leak,)
print 'mu(nu=100Hz)=%f' % (mb+Ne*Je*.1-leak,)

# total duration of the simulation (ms)
duration = 400.0
t = numpy.arange(0.0,400.0,dt)
vt = numpy.zeros_like(t)

t2 = time.time()
print 'Elapsed time is ', str(t2-t1), ' seconds.'

t1 = time.time()

for i,ti in enumerate(t):
    # time for a strong external input
    if ti>150.0:
        mue = 6.5
        sigmae = 7.5

    # time to restore the initial statistics of the external current
    if ti>300.0:
        mue = mb
        sigmae = sb

    Iext = acml_rng.normal(1.0,N)
    Iext[:Ne]*=sigmae
    Iext[Ne:]*=sigmai

    # Which neurons fired?
    fired = numpy.nonzero(v>=20.0)[0]

    if len(fired)>0:

        # Save mean firing rate of the excitatory neurons

        v[fired] = reset
        r[fired] = refr

        # Append spikes to the spike list
        for n in fired:
            # Spikes are stored by a (neuron, time) pair
            # For easy plotting later
            firings.append((n,ti))
            # and as a list for each neuron
            spikes[n].append(ti)

        aux = v-dt*(leak-mue)+numpy.sum(S[:,fired],1)+sdt*Iext

    else:
        aux = v-dt*(leak-mue)+sdt*Iext;

    # Neurons not in the refractory period
    nr = numpy.nonzero(r<=0)[0]
    # Bound voltages above 0.0
    v[nr] = numpy.where(aux[nr]>=0.0,aux[nr],0.0)

    # Progress refractory variable
    nr = numpy.nonzero(r>0)[0]
    r[nr]-=dt

    # record the voltage trace of the zeroeth neuron
    vt[i] = v[0]

t2 = time.time()
print 'Elapsed time is ', str(t2-t1), ' seconds.'

# -------------------------------------------------------------------------
# Plot everything
# -------------------------------------------------------------------------

def myplot():

    global firings

    t1 = time.time()

    figure()

    # Membrane potential trace of the zeroeth neuron
    subplot(3,1,1)

    vt[vt>=20.0]=65.0
    plot(t,vt)
    ylabel(r'$V-V_{rest}\ \left[\rm{mV}\right]$')

    # Raster plot of the spikes of the network
    subplot(3,1,2)
    myfirings = array(firings)
    myfirings_100 = myfirings[myfirings[:,0]<min(100,Ne)]
    plot(myfirings_100[:,1],myfirings_100[:,0],'.')
    axis([0, duration, 0, min(100,Ne)])
    ylabel('Neuron index')

    # Mean firing rate of the excitatory population as a function of time
    subplot(3,1,3)
    # 1 ms resultion of rate histogram
    dx = 1.0
    x = arange(0,duration,dx)
    myfirings_Ne = myfirings[myfirings[:,0]<Ne]
    mean_fe,x = numpy.histogram(myfirings_Ne[:,1],x)
    plot(x,mean_fe/dx/Ne*1000.0,ls='steps')
    ylabel('Hz')
    xlabel('time [ms]')
    t2 = time.time()
    print 'Finished.  Elapsed', str(t2-t1), ' seconds.'

#myplot()


import numpy as np
import scipy
import re
import os

import hashlib
import csb

from csb.bio.io.wwpdb import StructureParser

def chunks(l, n):
    """ Yield successive n-sized chunks from l.
    """
    for i in xrange(0, len(l), n):
        yield l[i:i+n]

class ScatteringFactor(object):
    """
    Cacluates the density in reciprocal space as

    F(s) = sum_m f_m(s) exp(-B_m s**2 / 4) exp(i*2pi*s*r)

    where f_m(s) is approximated by four Gaussian distributions
    and exp(-B_m s**2 / 4) are the thermal fluctuations

    g_m(s) = f_m(s) * exp(-B_m s**2 / 4) are precomputed

    """

    def __init__(self, structure=None):
        if structure is None:
            self._atoms = list()
            self._bfactor = list()
            self._seq = list()
            self._elements = list()
        else:
            self._structure = structure
            # For now only non hydrogen atoms
            # TODO use hydrogens as well
            self._atoms = []
            for chain in structure:
                for residue in structure[chain]:
                    for atom in residue:
                        a = residue[atom]
                        if not a.name.startswith("H"):
                            self._atoms.append(residue[atom])
            self._seq = []
            self._bfactor = []
            self._elements = []

            for atom in self._atoms:
                self._seq.append(atom.element.name)
                self._elements.append(atom.element.name)

                if atom._bfactor is None:
                    self._bfactor.append(1.)
                else:
                    self._bfactor.append(atom._bfactor)

            self._seq = np.array(self._seq)
            self._elements = set(self._elements)
            self._bfactor = np.clip(self._bfactor, 1., 100.)

        self._atom_type_params  = {}
        self._read_sf(fn=os.path.expanduser("~/projects/xfel/py/xfel/core/atomsf.lib"))

    @classmethod
    def from_isd(cls, universe):
        obj = cls()
        atoms = universe.atoms

        for atom in atoms:
            element = str(atom.properties['element'].name)
            obj._elements.append(element)
            obj._atoms.append(atom)
            obj._seq.append(element)
            try:
                obj._bfactor.append(max(1.,atom.properties['bfactor']))
            except KeyError:
                obj._bfactor.append(1.)
        obj._seq = np.array(obj._seq)
        obj._bfactor = np.array(obj._bfactor)
        obj._elements = set(obj._elements)
        obj._bfactor = np.clip(obj._bfactor, 1., 100.)
        return obj

    def _read_sf(self, fn):
        """
        Reads the coefficients for the analystical approximation
        to scattering factors from ccp4 database
        """
        float_pattern = '[-+]?[0-9]*\.?[0-9]+(?:[eE][-+]?[0-9]+)?'
        atom_pattern = '[A-Za-z]'
        atom_pattern = '[A-Za-z0-9-+]+'
        line_pattern = ("({0})\s+({1})"
                        "\s+({1})\s+({1})"
                        "\s+({1})\s+({1})"
                        "\s+({1})\s+({1})"
                        "\s+({1})\s+({1})").format(atom_pattern,float_pattern)

        regex = re.compile(line_pattern)

        with open(fn) as file_handle:
            for line in file_handle:
                if line.startswith("#"):
                    continue

                m = regex.match(line)
                atom_name = m.groups()[0]
                a1, a2, a3, a4 = m.groups()[1], m.groups()[3], m.groups()[5], m.groups()[7]
                b1, b2, b3, b4 = m.groups()[2], m.groups()[4], m.groups()[6], m.groups()[8]
                c = m.groups()[9]

                a = np.array([a1,a2,a3,a4],np.double)
                b = np.array([b1,b2,b3,b4],np.double)

                self._atom_type_params[atom_name] = (a,b,float(c))

    def _calculate_gm(self, hkl):
        """
        calculates the the product of scattering factor and
        debye-waller factors
        """
        f = np.zeros((len(self._atoms), hkl.shape[0]))
        seq = self._seq
        bfactor = self._bfactor

        s_tols = 0.25 * (hkl**2).sum(-1)

        for atom_type in self._elements:
            a,b,c = self._atom_type_params[atom_type]
            indices = np.where(seq==atom_type)[0]
            fx = c + np.dot(np.exp(np.outer(-s_tols,b)),a)
            f[indices,:] = fx[:]

        f *= np.exp(np.outer(-bfactor,s_tols))

        return f

    def _calculate_gm_grad(self, hkl):
        """
        calculate the gradien of the scattering factor and
        debye-waller factor
        """
        seq = np.array([a.element.name for a in self._atoms])
        f = np.zeros((len(self._atoms), hkl.shape[0]))
        dfg = np.zeros((len(self._atoms), hkl.shape[0], 3))

        bfactors = np.array([a.bfactor for a in self._atoms])
        bfactors = np.clip(bfactors, 1., 100.)
        s_tols = 0.25 * (hkl**2).sum(-1)

        for atom_type in self._elements:
            a,b,c = self._atom_type_params[atom_type]
            indices = np.where(seq==atom_type)[0]
            bfactor = bfactors[indices]
            g = np.exp(np.outer(-s_tols,b))
            sf = np.dot(g, a) + c
            gsf = np.sum(g * a[np.newaxis,:] * b[np.newaxis,:] * -0.5, -1)

            dwf = np.exp(-np.outer(bfactor, s_tols))
            gdwf = dwf * (bfactor * - 0.5)[:,np.newaxis]

            grad = sf * gdwf + gsf * dwf

            f[indices,:] = dwf * sf
            dfg[indices,:,:] = grad[:,:,np.newaxis] * hkl

        return dfg, f

    def _calculate_scattering_factors(self, hkl):
        """
        creates an approximation of the density in reciprocal space by
        four gaussians

        returns the scattering vectors
        """
        seq = self._seq
        bfactor = self._bfactor
        f = np.zeros((len(self._atoms), hkl.shape[0]))

        s_tols = 0.25 * (hkl**2).sum(-1)

        for atom_type in self._elements:
            a,b,c = self._atom_type_params[atom_type]
            indices = np.where(seq==atom_type)[0]
            fx = c + np.dot(np.exp(np.outer(-s_tols,b)),a)
            f[indices,:] = fx[:]

        return f

    def _calculate_debyewaller_factors(self, hkl):
        """
        """
        b  = np.array(self._bfactor)
        s_tols = 0.25 * (hkl**2).sum(-1)
        t = np.exp(np.outer(-b,s_tols))

        return t

    def grad_s(self, X, hkl):
        """
        Gradient with respect to the reciprocal space coordinates

        @param X: atomic positions
        @param hkl: reciprocal space positions

        """
        seq = np.array([atom.element.name for atom in self._atoms])
        bfactor = np.array([atom.bfactor for atom in self._atoms])
        bfactor = np.clip(bfactor, 1., 100.)

        s_tols = 0.25 * (hkl**2).sum(-1)
        dw_factors = np.exp(np.outer(-bfactor, s_tols))

    def grad_hkl(self, X, hkl):
        seq = self._seq
        bfactor = self._bfactor
        bfactor = np.clip(bfactor, 1., 100.)

        dg = np.zeros((len(self._atoms), hkl.shape[0], hkl.shape[1]))
        g = np.zeros((len(self._atoms), hkl.shape[0]))
        s_tols = 0.25 * (hkl**2).sum(-1)

        dw_factors = np.exp(np.outer(-bfactor, s_tols))
        ddw_factors = bfactor[:,np.newaxis] * dw_factors

        for atom_type in self._elements:
            a,b,c = self._atom_type_params[atom_type]
            indices = np.where(seq==atom_type)[0]
            inner_exp = np.exp(np.outer(-s_tols,b))
            sf = np.dot(inner_exp, a) + c
            dsf = np.dot(inner_exp, a*b)
            gx = dsf * dw_factors[indices] + sf * ddw_factors[indices]
            g[indices,:] = sf[:] * dw_factors[indices]
            a = np.einsum('ab,bc->abc',gx, -0.5*hkl)
            dg[indices,:,:] = a

        phase = np.dot((2 * np.pi * X),hkl.T)
        fx=  np.sum(g * np.exp(1j * phase),0)

        g2 = np.einsum('ba,bc->bac',g , 2 * np.pi * 1j *X)
        dfx = np.einsum("abc,ab->bc",dg + g2,np.exp(1j * phase))

        return dfx, fx

    def calculate_structure_factors(self, X, hkl):
        """
        TODO do this calculation in chunks to save space
        """
        F = np.zeros(hkl.shape[0], dtype=np.complex128)

        lim = hkl.shape[0]
        step = 512
        for i in range(0,lim,step):
            _hkl = hkl[i:i+step]
            f = self._calculate_scattering_factors(_hkl)
            f *= self._calculate_debyewaller_factors(_hkl)
            phase = np.dot((2 * np.pi * X),_hkl.T)
            F[i:i+step] = np.sum(f * np.exp(1j * phase),0)
        return F

    def calculate_structure_factor_gradient(self, X, hkl):
        """
        calculates the gradient of the fourier density
        with respect to the atomic coordinates
        """
        G = np.zeros(hkl.shape, dtype=np.complex128)
        lim = hkl.shape[0]
        F = np.zeros(hkl.shape[0], dtype=np.complex128)

        step = 512

        for i in range(0, lim, step):
            _hkl = hkl[i:i+step]

            dfg, f = self._calculate_gm_grad(_hkl)

            phase = np.exp(1j * np.dot((2 * np.pi * X), _hkl.T))
            gphase = phase[:, :, np.newaxis] *\
                     1j * 2 * np.pi * X[:, np.newaxis, :]
            grad = dfg * phase[:, :, np.newaxis]
            grad += f[:, :, np.newaxis] * gphase
            F[i: i+step] = np.sum(f * phase, 0)
            G[i: i+step, :] = np.sum(grad, 0)

        return G, F

    def calculate_structure_factor_gradient2(self, X):
        """
        calculates the gradient of the fourier density
        with respect to the atomic coordinates
        """
        g_m = self._calculate_scattering_factors(hkl)
        g_m *= self._calculate_debyewaller_factors(hkl)

        phase = np.dot((2 * np.pi * X),self._hkl.T)
        fx = (g_m *1j * 2 * np.pi  * np.exp(1j * phase))
        dF_dx =  np.array([np.multiply.outer(s,fx_s) for s,fx_s in
                           zip(fx.T,self._hkl)])

        return dF_dx

    def calculate_intensity_gradient(self, X):
        """
        calculates the gradient of the intensity with respect to the atomic coordinates dI/dx
        """
        g_m = self._calculate_scattering_factors(self._hkl)
        g_m *= self._calculate_debyewaller_factors(self._hkl)
        phase = np.dot((2 * np.pi * X),self._hkl.T)
        F = np.sum(g_m * np.exp(1j * phase),0)
        fx = (g_m *1j * 2 * np.pi  * np.exp(1j * phase))
        dF_dx =  np.array([np.multiply.outer(s,fx_s) for s,fx_s in zip(fx.T,self._hkl)])
        dI_dx =  np.conj(F[:,np.newaxis,np.newaxis]) * dF_dx + F[:,np.newaxis,np.newaxis] * np.conj(dF_dx)

        return dI_dx

class Correlations(object):

    def __init__(self, angles, nbins):

        self._bin_angles(angles, nbins)

    def _bin_angles(self, angles, nbins):
        pass

    def calculate_from_density(self, rho):
        pass

class OnePhotonCorrelations(Correlations):

    def _bin_angles(self, angles, nbins):
        d = np.sqrt(np.sum(angles**2,-1))
        lower = d.min()
        upper = d.max()

        axes = np.linspace(lower, upper, nbins)
        indices = np.argsort(d)
        bins = [[] for x in xrange(nbins)]

        j = 0
        for i in range(0,axes.shape[0]):
            right_edge = axes[i]
            print right_edge, i
            while d[indices[j]] < right_edge:
                bins[i-1].append(indices[j])
                j += 1

        bins[-1] = indices[j:].tolist()

        self._axes  = axes
        self._bins = bins

    def calculate_from_density(self, rho):
        I = np.asarray([np.sum(rho.take(bin))
                        for bin in self._bins])

        return I

class CachedScatteringFactor(ScatteringFactor):

    def __init__(self, structure):
        super(CachedScatteringFactor,self).__init__(structure)

        self._f = None

    def calculate_structure_factors(self, X, hkl):
        if self._f is None:
            print "calc f"
            self._f = self._calculate_scattering_factors(hkl)
            self._f *= self._calculate_debyewaller_factors(hkl)
        else:
            print "using cached f"

        phase = np.dot((-2 * np.pi * X),hkl.T)
        F = np.sum(self._f * np.exp(1j * phase),0)

        return F

class SphericalSection(object):

    def get(self,
            n_points=20, radius=1.0,
            polar_min=0., polar_max=np.pi,
            azimut_min=0., azimut_max=2*np.pi):

        theta = np.linspace(polar_min,polar_max, n_points)
        phi = np.linspace(azimut_min, azimut_max, n_points)

        x = np.outer(radius*np.sin(theta), np.cos(phi))
        y = np.outer(radius*np.sin(theta), np.sin(phi))
        z = np.outer(radius*np.cos(theta), np.ones(n_points))

        return [x,y,z]

class EwaldSphereProjection(object):

    def get_indices(self, wavelength, x,y,z):
        """
        projects dectector points onto an Ewald Sphere
        x, y, z are the pixel coordinates

        x, y, z are all M x N matrices, where M x N is the detector size.

        It is assumed that the detector is perpendicular to the Z-axis

        """

        d = np.sqrt(x**2 + y**2 + z**2)

        h = 1/wavelength * (x/d)
        k = 1/wavelength * (y/d)
        l = 1/wavelength * (z/d)

        return h,k,l

    def project(self, structure_factor, angle):
        pass

if __name__ == "__main__":
    import matplotlib
    matplotlib.interactive(True)

    import time
    import os
    import seaborn as sns
    from mpl_toolkits.mplot3d import Axes3D
    import matplotlib.pyplot as plt
    import pylab
    from pylab import *

    from csb.bio.io.wwpdb import StructureParser
    from csb.bio.io.wwpdb import get
    from xfel.core.density import Density

    #structure = get("1L2Y")
    #structure = StructureParser(os.path.expanduser("~/data/pdb/caffeine2.pdb")).parse()

    #fn = os.path.expanduser("~/gsh.pdb")
    structure = StructureParser(os.path.expanduser("~/projects/xfel/data/GTT_short.pdb")).parse()

    x = np.linspace(-1.,1.,11)
    h, k, l = np.meshgrid(x,x,x)
    hkl = np.vstack([item.ravel() for item in [h,k,l]]).T
    hkl = np.ascontiguousarray(hkl)
    bf = np.random.random()

    def bfactors(hkl, bf):
        return np.exp(-0.25 * bf * (hkl**2).sum(-1))

    def bfactor_grad(hkl):
        return np.exp(-0.25 * bf * (hkl**2).sum(-1))[:,np.newaxis] * -0.5 * hkl * bf

    a = np.random.random(4,)
    b = np.random.random(4,)
    c = 0.3

    def sf(hkl,a,b,c):
        s_tols = -0.25 * (hkl**2).sum(-1)
        inner_exp = np.exp(np.outer(-s_tols,b))
        sf = np.dot(inner_exp, a) + c

        return sf

    def sf_grad(hkl, a, b, c):
        s_tols = -0.25 * (hkl**2).sum(-1)
        sf = np.exp(np.outer(-s_tols,b)) * a[np.newaxis,:] * b[np.newaxis,:] * 0.5

        return sf.sum(-1)[:,np.newaxis] * hkl

    def gm(hkl, a, b, c, bf):
        s_tols = -0.25 * (hkl**2).sum(-1)
        inner_exp = np.exp(np.outer(-s_tols,b))
        sf = np.dot(inner_exp, a) + c
        bf = np.exp(bf * s_tols)

        return sf * bf

    def gm_grad(hkl, a, b, c, bf):
        s_tols = -0.25 * (hkl**2).sum(-1)
        g = np.exp(np.outer(-s_tols,b))
        sf = np.dot(g, a) + c
        gsf = np.sum(g * a[np.newaxis,:] * b[np.newaxis,:] * 0.5, -1)

        bb = np.exp(bf * s_tols)
        gb = bb * bf * - 0.5
        grad = sf * gb  + gsf * bb
        return grad[:,np.newaxis] * hkl

    sf = ScatteringFactor(structure)
    X = np.array([a.vector for a in  sf._atoms])
    X -= X.mean(0)
    if False:

        n = 10
        X = X[:n]
        sf._seq = sf._seq[:n]
        sf._elements = ['N', 'C']
        sf._atoms = sf._atoms[:n]
        sf._bfactor = sf._bfactor[:n]

    dgm, f1 = sf._calculate_gm_grad(hkl)

    f = sf._calculate_scattering_factors(hkl)
    f *= sf._calculate_debyewaller_factors(hkl)

    scatter(f.real.ravel(), f1.real.ravel())

    dgm2 = dgm * 0.0
    eps = 1e-7
    for i in range(3):
        hkl[:, i] += eps
        fprime = sf._calculate_scattering_factors(hkl)
        fprime *= sf._calculate_debyewaller_factors(hkl)
        dgm2[:, :, i] = (fprime - f)/eps
        hkl[:, i] -= eps

    figure()
    scatter(dgm.real.ravel(), dgm2.real.ravel())

    G, FF = sf.calculate_structure_factor_gradient(X, hkl)
    G2 = G * 0.0
    F = sf.calculate_structure_factors(X, hkl)
    eps = 1e-7
    for i in range(3):
        hkl[:,i] += eps

        G2[:,i] = (sf.calculate_structure_factors(X, hkl) - F)/eps
        hkl[:,i] -= eps

    figure()
    scatter(G.real.ravel(), G2.real.ravel())
    scatter(G.imag.ravel(), G2.imag.ravel())

    figure()
    scatter(F.real.ravel(), FF.real.ravel())
    show()

    t0 = time.time()
    G, FF = sf.calculate_structure_factor_gradient(X, hkl)
    print "hkl gradient: {} \n".format(time.time() - t0)
    t0 = time.time()
    g = sf.grad_hkl(X, hkl)
    print "X gradient: {} \n".format(time.time() - t0)

    raise
    sf = ScatteringFactor(structure)
    sf._hkl = hkl

    X = np.array([a.vector for a in  sf._atoms])
    X -= X.mean(0)

    g,g2 = sf.grad_hkl(X, hkl)
    F = sf.calculate_structure_factors(X,hkl)
    gi= sf.calculate_intensity_gradient(X)

    raise

    F = F.reshape(h.shape)
    rho = np.fft.fftshift(np.abs(np.fft.ifftn(F,[250,250,250])))

    grid = Density.from_voxels(np.abs(F)**2,1.)
    grid.write_gaussian(os.path.expanduser("~/mr.cube"))
    raise
    grid = Density.from_voxels(rho,1.)
    grid.write_gaussian(os.path.expanduser("~/mr2.cube"))

    raise

    if True:
        fig = pylab.figure()
        ax = fig.add_subplot(131)
        xi, yi= np.mgrid[0:500:1,0:500:1]
        ax.contour(xi,yi, rho.sum(0), 30)
        pylab.show()
        ax = fig.add_subplot(132)
        xi, yi= np.mgrid[0:500:1,0:500:1]
        ax.contour(xi,yi, rho.sum(1), 30)
        pylab.show()
        ax = fig.add_subplot(133)
        xi, yi= np.mgrid[0:500:1,0:500:1]
        ax.contour(xi,yi, rho.sum(2), 30)
        pylab.show()

    raise

    from mayavi import mlab
    xi, yi, zi = np.mgrid[0:500:1,0:500:1,0:500:1]
    obj = mlab.contour3d(rho, contours=10, transparent=True)
    mlab.show()

    from mayavi import mlab
    obj = mlab.contour3d(np.abs(F), contours=10, transparent=True)
    mlab.show()

    raise
    for ii in range(0,F.shape[0],25):
        fig = pylab.figure()
        ax = fig.add_subplot(111)
        xi, yi= np.mgrid[0:500:1,0:500:1]
        ax.contour(xi,yi,rho[ii,:,:], 30)
        pylab.show()

    I = np.abs(F)**2

    fig = pylab.figure()
    ax = fig.add_subplot(111)
    nx, ny, nz = I.shape
    xi, yi= np.mgrid[0:nx:1,0:ny:1]
    ax.contour(xi,yi, I.sum(2), 15)


"""
========================================================
Statistic the SV Stat after AGE Process
========================================================
Author: Shujia Huang & Siyang Liu
Date  : 2014-03-07 0idx:54:15
"""
import sys
import re
import os
import string
import numpy as np
import matplotlib.pyplot as plt

def DrawFig(figureFile, distance, properDepth, imProperDepth, nr, aa, bb, mscore, misprob, aveIden, inbCoe):

    fig = plt.figure(num=None, figsize=(16, 30), facecolor='w', edgecolor='k')
    title  = ['Distance distribution', 'NRatio', 'Perfect Depth', 'Imperfect depth', '', '', '']
    ylabel = ['The position of breakpoint', 'N Ratio of varints', \
              'Perfect Depth', 'Both ImPerfect Depth', 'InbreedCoefficient', \
              'Map score', 'Mismapping Probability'  , 'Average Identity', \
              'ProperReadDepth', 'ImProperReadDepth']
    al = 0.5
    for i, data in enumerate ([distance, nr, aa, bb, inbCoe, mscore, misprob, aveIden, properDepth, imProperDepth ]):
        plt.subplot(10,2,2 * i + 1)
        #plt.title(title[i], fontsize=16)
        P = data[:,0] == 1; N = data[:,0] == 2; X = data[:,0] == 3
        plt.scatter(data[:,1][N], data[:,2][N], marker='o', c = 'r', alpha=al, linewidths = 0.1, label = 'Negative(%d)'%len(data[:,1][N])) # Negative
        plt.scatter(data[:,1][P], data[:,2][P], marker='o', c = 'g', alpha=al, linewidths = 0.1, label = 'Positive(%d)'%len(data[:,1][P])) # Positive
        plt.scatter(data[:,1][X], data[:,2][X], marker='*', c = 'Y', alpha=al, linewidths = 0.1, label = 'Positive->Negative(%d)' % len(data[:,1][X])) # Positive->Negative
        plt.legend(loc='upper right')
        plt.xlim(-10, 50)
        if i == 9: plt.xlabel('Score', fontsize=16)
        plt.ylabel(ylabel[i], fontsize=16)

        plt.subplot(10, 2, 2*i + 2)
        NEW  = data[:,0] == 0
        good = data[:,1][NEW] >= VQ_CUTOFF
        bad  = data[:,1][NEW] <  VQ_CUTOFF

        plt.scatter(data[:,1][NEW][bad], data[:,2][NEW][bad], marker='o', c = 'm', alpha=al, linewidths = 0.1, label = 'bad(%d)' % len(data[:,1][NEW][bad])) # bad
        plt.scatter(data[:,1][NEW][good], data[:,2][NEW][good], marker='o', c = 'b', alpha=al, linewidths = 0.1, label = 'good(%d)' % len(data[:,1][NEW][good])) # good
        plt.xlim(-3, 30)
        plt.legend(loc='upper right')
        if i == 9: plt.xlabel('Score', fontsize=16)

    fig.savefig(figureFile + '.png')
    #fig.savefig(figureFile + '.pdf')

def DrawPhredScale (figureFile, phredScal):

    fig = plt.figure()

    ylabel = ['Phred Scale']
    for i, data in enumerate ([phredScal ]):
        plt.subplot(2, 1, 2 * i + 1)
        P = data[:,0] == 1; N = data[:,0] == 2; X = data[:,0] == 3
        plt.scatter(data[:,1][N], data[:,2][N], marker='o', c = 'r', alpha=0.5, linewidths = 0, label = 'Negative(%d)'%len(data[:,1][N])) # Negative
        plt.scatter(data[:,1][P], data[:,2][P], marker='o', c = 'g', alpha=0.5, linewidths = 0, label = 'Positive(%d)'%len(data[:,1][P])) # Positive
        plt.scatter(data[:,1][X], data[:,2][X], marker='o', c = 'Y', alpha=0.5, linewidths = 0, label = 'Positive->Negative(%d)' % len(data[:,1][X])) # Positive->Negative
        plt.legend(loc='upper left')
        plt.ylabel(ylabel[i], fontsize=16)

        plt.subplot(2, 1, 2*i + 2)
        NEW  = data[:,0] == 0
        good = data[:,1][NEW] >= VQ_CUTOFF
        bad  = data[:,1][NEW] <  VQ_CUTOFF

        plt.scatter(data[:,1][NEW][bad] , data[:,2][NEW][bad] , marker='o', c = 'm', alpha=0.5, linewidths = 0, label = 'bad(%d)' % len(data[:,1][NEW][bad])) # bad
        plt.scatter(data[:,1][NEW][good], data[:,2][NEW][good], marker='o', c = 'b', alpha=0.5, linewidths = 0, label = 'good(%d)' % len(data[:,1][NEW][good])) # good
        plt.legend(loc='upper left')
        plt.xlabel('Score'  , fontsize=16)
        plt.ylabel(ylabel[i], fontsize=16)

    fig.savefig(figureFile + '.png')
    #fig.savefig(figureFile + '.pdf')

def Accum (data, isBig = False):

    tmpD= data
    k   = sorted(tmpD.keys(), key = lambda d: float(d))
    dat = []
    for i in range(len(k)):
        if isBig:
            for j in range(i,len(k)): tmpD[k[i]][1] += tmpD[k[j]][0]
        else:
            for j in range(i+1): tmpD[k[i]][1] += tmpD[k[j]][0]
        dat.append([float(k[i]), float(tmpD[k[i]][0]), float(tmpD[k[i]][1]) ])

    return dat

def SampleFaLen (faLenFile):

    if faLenFile[-3:] == '.gz': I = os.popen('gzip -dc %s' % faLenFile)
    else                      : I = open(faLenFile)
    data = {}
    while 1:
        lines = I.readlines (100000)
        if not lines: break
        for line in lines:
            col = line.strip('\n').split()
            data[col[0]] = string.atoi(col[1])
    I.close()
    return data

def LoadFaLen (faLenLstFile):

    data = {}
    I    = open (faLenLstFile)
    for line in I.readlines():

        if len(line.strip('\n').split()) != 2: raise ValueError('[ERROR] The format of Fa length list maybe not right. It could just be: "sample FalenghtFile", but found',line)
        sampleId, fileName = line.strip('\n').split()
        if sampleId not in data: data[sampleId] = {}
        data[sampleId] = SampleFaLen(fileName)
    I.close()
    return data

def main (argv):

    qFaLen    = LoadFaLen(argv[1])
    figPrefix = 'test'
    if len(argv) > 2: figPrefix = argv[2]
    if argv[0][-3:] == '.gz':
        I = os.popen('gzip -dc %s' % argv[0])
    else:
        I = open (argv[0])

    s, annotations, mark = set(), [], []
    print '#Chr\tPosition\tDistance\tLeftIden\tRightIden\tAveIden\tN-Ratio\tAA'
    while 1: # VCF format

        lines = I.readlines(100000)
        if not lines: break

        for line in lines:

            col = line.strip('\n').split()
            if re.search(r'^#CHROM', line): col2sam = { i+9:sam for i,sam in enumerate(col[9:]) }
            if re.search(r'^#', line): continue
            key = col[0] + ':' + col[1]
            if key in s: continue
            s.add(key)

            #if re.search(r'^PASS', col[6]): continue
            #if not re.search(r'_TRAIN_SITE', col[7]): continue
            #if not re.search(r'^PASS', col[6]): continue
            isbad = False
            for i, sample in enumerate (col[9:]):
                if re.search(r'NULL', sample): isbad = True
            if isbad: continue

            fmat = { k:i for i,k in enumerate(col[8].split(':')) }
            if 'VS' not in fmat or 'QR' not in fmat: continue
            if 'AGE' not in fmat: continue
            if len(annotations) == 0: annotations = [[] for _ in col[9:] ]

            vcfinfo = { d.split('=')[0]: d.split('=')[1] for d in col[7].split(';') if len(d.split('=')) == 2 }
            vq      = string.atof(vcfinfo['VQ'])
            inb     = string.atof(vcfinfo['InbCoeff'])
            if ('POSITIVE_TRAIN_SITE' in col[7]) and ('NEGATIVE_TRAIN_SITE' in col[7]):
                mark.append([3, vq, inb])
            elif 'POSITIVE_TRAIN_SITE' in col[7]:
                mark.append([1, vq, inb])
            elif 'NEGATIVE_TRAIN_SITE' in col[7]:
                mark.append([2, vq, inb])
            else:
                mark.append([0, vq, inb])

            # GT:AA:AE:FN:MIP:MS:QR:RR:VS:VT
            for i, sample in enumerate (col[9:]):

                sampleId = col2sam[9+i]

                field = sample.split(':')
                if sample == './.' or len(field) < fmat['QR'] + 1 or field[fmat['QR']].split(',')[-1] == '.' or field[fmat['AS']] == '.':
                    annotations[i].append([0, 0, 0, 0, 0, 0, 0, 0, 0])
                    continue
                qr      = field[fmat['QR']].split(',')[-1]
                qregion = np.array(qr.split('-'))
                if len(qregion) > 3: qId = qregion[0] + '-' + qregion[1]
                else               : qId = qregion[0]
                qSta = string.atoi(qregion[-2])
                qEnd = string.atoi(qregion[-1])

                if sampleId not in qFaLen:
                    raise ValueError ('[ERROR] The sample name $s(in vcf) is not in the name of Fa list.' % sampleId)
                if qId not in qFaLen[sampleId]:
                    raise ValueError ('[ERROR]', qId, 'is not been found in file', opt.qFalen, '\n')
                qSta= int(qSta * 100 / qFaLen[sampleId][qId] + 0.5)
                qEnd= int(qEnd * 100 / qFaLen[sampleId][qId] + 0.5)
                if qSta > 100 or qEnd > 100:
                    raise ValueError ('[ERROR] Query size Overflow! sample: %s; scaffold: %s' % (sampleId, qId))

                leg = qSta
                if 100 - qEnd < qSta: leg = qEnd
                nn  = string.atof(sample.split(':')[fmat['NR']])
                n   = round(1000 * nn) / 10.0                                  # N ratio
                alt = string.atoi(sample.split(':')[fmat['AA']].split(',')[1]) # Alternate perfect
                bot = string.atoi(sample.split(':')[fmat['AA']].split(',')[3]) # Both imperfect
                pro, ipr = [0,0]
                ms  = string.atoi(sample.split(':')[fmat['AS']])               # Mapping score
                mip = string.atof(sample.split(':')[fmat['MS']])               # Mismapping probability
                if sample.split(':')[fmat['AGE']] != '.':
                    aveI = string.atoi(sample.split(':')[fmat['AGE']].split(',')[3]) # ave_iden in AGE
                else:
                    aveI = 0

                annotations[i].append([leg, n, alt, bot, pro, ipr, ms, mip, aveI])
    I.close()
    print >> sys.stderr, '# Number of Positions: %d' % len(mark)
    if len(mark) != len(annotations[0]):
        raise ValueError ('[ERROR] The size is not match mark=%d, annotations=%d!' % (len(mark), len(annotations)))

    annotations = np.array(annotations);
    sampleNum   = len(annotations)

    data, distance, properDepth, imProperDepth, nr, aa, bb, mscore, misprob, aveIden = [],[],[],[],[],[],[],[],[],[]
    inbreedCoe, phredScal = [], []
    for i in range(len(annotations[0])):

        anno    = np.array([annotations[s][i] for s in range(sampleNum) if len(annotations[s][i][annotations[s][i]!=0]) > 0 ]) # each person in the same position

        score  = np.array([annotations[s][i][-3] for s in range(sampleNum) if annotations[s][i][-3] > 0 ])
        msprob = np.array([annotations[s][i][-2] for s in range(sampleNum) if annotations[s][i][-3] > 0 ])
        phred  = -10 * np.log10(1.0 - score.sum() / np.sum(score/(1.0 - msprob))) # Phred scale

        if len(anno) == 0: continue
        leg, n, alt, bot, pro,ipr, ms, mip, aveI = np.median(anno, axis=0)

        distance.append      ([mark[i][0], mark[i][1], leg ])
        properDepth.append   ([mark[i][0], mark[i][1], pro ])
        imProperDepth.append ([mark[i][0], mark[i][1], ipr ])
        nr.append            ([mark[i][0], mark[i][1], n   ])
        aa.append            ([mark[i][0], mark[i][1], alt ])
        bb.append            ([mark[i][0], mark[i][1], bot ])
        mscore.append        ([mark[i][0], mark[i][1], ms  ])
        misprob.append       ([mark[i][0], mark[i][1], mip ])
        aveIden.append       ([mark[i][0], mark[i][1], aveI])
        phredScal.append     ([mark[i][0], mark[i][1], phred])
        inbreedCoe.append    ([mark[i][0], mark[i][1], mark[i][2]])

        data.append([leg, alt, pro, ipr, n, bot])
        print mark[i][0], mark[i][1], mark[i][2], '\t', leg, '\t', pro, '\t', ipr,'\t', n, '\t', alt, '\t', bot

    data = np.array(data)
    print >> sys.stderr, '\nPosition\tALTernatePerfect\tLeftIdentity\tRightIdentity\tAveIden\tNRatio\tBothImperfect'
    print >> sys.stderr, 'Means: ', data.mean(axis=0), '\nstd : ', data.std(axis=0), '\nMedian: ', np.median(data, axis=0)
    print >> sys.stderr, '25 Percentile:', np.percentile(data, 25,axis=0), '\n50 Percentile:', np.percentile(data, 50,axis=0), '\n75 Percentile:', np.percentile(data, 75,axis=0)

    DrawFig(figPrefix, \
             np.array (distance     ), \
             np.array (properDepth  ), \
             np.array (imProperDepth), \
             np.array (nr           ), \
             np.array (aa           ), \
             np.array (bb           ), \
             np.array (mscore       ), \
             np.array (misprob      ), \
             np.array (aveIden      ), \
             np.array (inbreedCoe   ) )
    DrawPhredScale (figPrefix + '.phred', np.array(phredScal))

if __name__ == '__main__':

    VQ_CUTOFF = 3.0
    main(sys.argv[1:])


#!/usr/bin/env python
__author__ = 'greghines'
import numpy as np
import os
import pymongo
import sys
import cPickle as pickle
import bisect
import csv
import matplotlib.pyplot as plt
import random
import math
import urllib
import matplotlib.cbook as cbook

def index(a, x):
    'Locate the leftmost value exactly equal to x'
    i = bisect.bisect_left(a, x)
    if i != len(a) and a[i] == x:
        return i
    raise ValueError

if os.path.exists("/home/ggdhines"):
    sys.path.append("/home/ggdhines/PycharmProjects/reduction/experimental/clusteringAlg")
    sys.path.append("/home/ggdhines/PycharmProjects/reduction/experimental/classifier")
else:
    sys.path.append("/home/greg/github/reduction/experimental/clusteringAlg")
    sys.path.append("/home/greg/github/reduction/experimental/classifier")
#from divisiveDBSCAN import DivisiveDBSCAN
from divisiveDBSCAN_multi import DivisiveDBSCAN
from divisiveKmeans import DivisiveKmeans
from iterativeEM import IterativeEM

if os.path.exists("/home/ggdhines"):
    base_directory = "/home/ggdhines"
else:
    base_directory = "/home/greg"

client = pymongo.MongoClient()
db = client['condor_2014-11-23']
classification_collection = db["condor_classifications"]
subject_collection = db["condor_subjects"]

big_userList = []
big_subjectList = []
animal_count = 0

f = open(base_directory+"/Databases/condor_ibcc.csv","wb")
f.write("a,b,c\n")
alreadyDone = []
animals_in_image = {}
animal_index = -1
global_user_list = []
animal_to_image = []
zooniverse_list = []
condor_votes = {}
animal_votes = {}
#subject_vote = {}
results = []

to_sample_from = list(subject_collection.find({"state":"complete"}))
to_sample_from2 = list(subject_collection.find({"classification_count":1,"state":"active"}))

votes = []

sample = random.sample(to_sample_from,100)
#sample.extend(random.sample(to_sample_from2,1000))
# for subject_index,subject in enumerate(sample):
#     print "== " + str(subject_index)
#     zooniverse_id = subject["zooniverse_id"]
#     for user_index,classification in enumerate(classification_collection.find({"subjects.zooniverse_id":zooniverse_id})):
#         if "user_name" in classification:
#             user = classification["user_name"]
#         else:
#             user = classification["user_ip"]
#
#         try:
#             tt = index(big_userList,user)
#         except ValueError:
#             bisect.insort(big_userList,user)

for subject_index,subject in enumerate(sample):
    print subject_index
    zooniverse_id = subject["zooniverse_id"]

    annotation_list = []
    user_list = []
    animal_list = []
    #local_users = []

    for user_index,classification in enumerate(classification_collection.find({"subjects.zooniverse_id":zooniverse_id})):
        try:
            mark_index = [ann.keys() for ann in classification["annotations"]].index(["marks",])
            markings = classification["annotations"][mark_index].values()[0]

            if "user_name" in classification:
                user = classification["user_name"]
            else:
                user = classification["user_ip"]
            found_condor = False

            for animal in markings.values():
                scale = 1.875
                x = scale*float(animal["x"])
                y = scale*float(animal["y"])

                animal_type = animal["animal"]
                if not(animal_type in ["carcassOrScale","carcass"]):
                    annotation_list.append((x,y))
                    #print annotation_list
                    user_list.append(user)
                    animal_list.append(animal_type)
                    if not(user in global_user_list):
                        global_user_list.append(user)
                    #local_users.append(user)

                if animal_type == "condor":
                    found_condor = True

        except (ValueError,KeyError):
            pass

    #if there were any markings on the image, use divisive kmeans to cluster the points so that each
    #cluster represents an image
    if annotation_list != []:
        user_identified,clusters = DivisiveKmeans(3).fit2(annotation_list,user_list,debug=True)

        #fix split clusters if necessary
        if user_identified != []:
            user_identified,clusters = DivisiveKmeans(3).__fix__(user_identified,clusters,annotation_list,user_list,200)

            for center,c in zip(user_identified,clusters):
                animal_index += 1
                #animal_votes.append([])
                animal_to_image.append(zooniverse_id)

                if not(zooniverse_id in animals_in_image):
                    animals_in_image[zooniverse_id] = [animal_index]
                else:
                    animals_in_image[zooniverse_id].append(animal_index)

                results.append((zooniverse_id,center))
                for pt in c:
                    pt_index = annotation_list.index(pt)
                    user_index = global_user_list.index(user_list[pt_index])
                    animal_type = animal_list[annotation_list.index(pt)]

                    if animal_type == "condor":
                        votes.append((user_index,animal_index,1))
                        if not(animal_index in animal_votes):
                            animal_votes[animal_index] = [1]
                        else:
                            animal_votes[animal_index].append(1)
                    else:
                        votes.append((user_index,animal_index,0))
                        if not(animal_index in animal_votes):
                            animal_votes[animal_index] = [0]
                        else:
                            animal_votes[animal_index].append(0)

print "=====---"
#print votes
classify = IterativeEM()
classify.__classify__(votes)

most_likely = classify.__getMostLikely__()
estimates = classify.__getEstimates__()

X = []
Y = []
X2 = []
Y2 = []

#for subject_index,zooniverse_id in enumerate(big_subjectList):
for ii in range(animal_index):
    x = np.mean(animal_votes[ii])

    y = estimates[ii][1]
    X.append(x)
    Y.append(y)

    if math.fabs(x-y) > 0.3:
        zooniverse_id,(centerX,centerY) = results[ii]
        print x,y
        subject = subject_collection.find_one({"zooniverse_id":zooniverse_id})

        url = subject["location"]["standard"]
        slash_index = url.rfind("/")
        object_id = url[slash_index+1:]

        if not(os.path.isfile(base_directory+"/Databases/condors/images/"+object_id)):
            urllib.urlretrieve (url, base_directory+"/Databases/condors/images/"+object_id)

        image_file = cbook.get_sample_data(base_directory+"/Databases/condors/images/"+object_id)
        image = plt.imread(image_file)

        fig, ax = plt.subplots()
        im = ax.imshow(image)
        plt.plot([centerX,],[centerY,],'o')
        plt.show()

    # #if ((x < 0.5) and (y > 0.5)) or ((x > 0.5) and (y < 0.5)):
    #     subject = subject_collection.find_one({"zooniverse_id":zooniverse_id})
    #     print x,y
    #     print subject["location"]["standard"]

    #    #print most_likely[subject_index],estimates[subject_index],np.mean(subject_vote[zooniverse_id])
    #else:
    #    print estimates[subject_index],0

plt.plot(X,Y,'.',color="blue")
plt.plot(X2,Y2,'.',color="red")
plt.xlim((-0.05,1.05))
plt.ylim((-0.05,1.05))
plt.show()


#!/usr/bin/python
# -*- coding: utf-8 -*-

"""
=========================================================
Linear Regression Example
=========================================================
This example uses the only the first feature of the `diabetes` dataset, in
order to illustrate a two-dimensional plot of this regression technique. The
straight line can be seen in the plot, showing how linear regression attempts
to draw a straight line that will best minimize the residual sum of squares
between the observed responses in the dataset, and the responses predicted by
the linear approximation.

The coefficients, the residual sum of squares and the variance score are also
calculated.

"""
print(__doc__)

# Code source: Jaques Grobler
# License: BSD 3 clause

import matplotlib.pyplot as plt
import numpy as np
from sklearn import datasets, linear_model
from sklearn.metrics import mean_squared_error, r2_score

# Load the diabetes dataset
diabetes = datasets.load_diabetes()

# Use only one feature
diabetes_X = diabetes.data[:, np.newaxis, 2]

# Split the data into training/testing sets
diabetes_X_train = diabetes_X[:-20]
diabetes_X_test = diabetes_X[-20:]

# Split the targets into training/testing sets
diabetes_y_train = diabetes.target[:-20]
diabetes_y_test = diabetes.target[-20:]

# Create linear regression object
regr = linear_model.LinearRegression()

# Train the model using the training sets
regr.fit(diabetes_X_train, diabetes_y_train)

# Make predictions using the testing set
diabetes_y_pred = regr.predict(diabetes_X_test)

# The coefficients
print('Coefficients: \n', regr.coef_)
# The mean squared error
print("Mean squared error: %.2f"
      % mean_squared_error(diabetes_y_test, diabetes_y_pred))
# Explained variance score: 1 is perfect prediction
print('Variance score: %.2f' % r2_score(diabetes_y_test, diabetes_y_pred))

# Plot outputs
plt.scatter(diabetes_X_test, diabetes_y_test,  color='black')
plt.plot(diabetes_X_test, diabetes_y_pred, color='blue', linewidth=3)

plt.xticks(())
plt.yticks(())

plt.show()


# -*- coding: utf-8 -*-
#
# HyperSpy User Guide documentation build configuration file, created by
# sphinx-quickstart on Wed Feb 29 15:14:48 2012.
#
# This file is execfile()d with the current directory set to its containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import sys
import os
sys.path.append('../../')
sys.path.append(os.path.abspath('../sphinxext'))

from hyperspy import Release
# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#sys.path.insert(0, os.path.abspath('.'))

# -- General configuration -----------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
#needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = [
    'sphinx.ext.autodoc',
    'gen_rst',
    'numpydoc',
    'matplotlib.sphinxext.only_directives',
    'sphinx.ext.intersphinx',
    'sphinx.ext.pngmath',
    'sphinx.ext.autosummary',
    'ipython_console_highlighting']  # , 'rst2pdf.pdfbuilder']

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix of source filenames.
source_suffix = '.rst'

# The encoding of source files.
#source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'HyperSpy User Guide [Draft]'
copyright = u'2011-2013, The HyperSpy Developers'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = Release.version
# The full version, including alpha/beta/rc tags.
release = Release.version

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['_build']

# The reST default role (used for this markup: `text`) to use for all documents.
#default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []

# -- Options for HTML output ---------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
html_theme = 'default'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}

# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
html_title = "HyperSpy User Guide v%s" % Release.version

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
#html_domain_indices = True

# If false, no index is generated.
#html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None

# Output file base name for HTML help builder.
htmlhelp_basename = 'HyperSpyUserGuidedoc'

# -- Options for LaTeX output --------------------------------------------

latex_elements = {
    # The paper size ('letterpaper' or 'a4paper').
    #'papersize': 'letterpaper',

    # The font size ('10pt', '11pt' or '12pt').
    #'pointsize': '10pt',

    # Additional stuff for the LaTeX preamble.
    #'preamble': '',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, documentclass [howto/manual]).
latex_documents = [
    ('index', 'HyperSpyUserGuide.tex', u'HyperSpy User Guide',
     u'The HyperSpy Developers', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
latex_logo = '_static/hyperspy_logo.png'

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False

# If true, show page references after internal links.
#latex_show_pagerefs = False

# If true, show URL addresses after external links.
#latex_show_urls = False

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_domain_indices = True

# -- Options for manual page output --------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    ('index', 'hyperspyuserguide', u'HyperSpy User Guide Documentation',
     [u'The HyperSpy Developers'], 1)
]

# If true, show URL addresses after external links.
#man_show_urls = False

# -- Options for Texinfo output ------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
    ('index', 'HyperSpyUserGuide', u'HyperSpy User Guide Documentation',
     u'The HyperSpy Developers', 'HyperSpyUserGuide', 'One line description of project.',
     'Miscellaneous'),
]

# Documents to append as an appendix to all manuals.
#texinfo_appendices = []

# If false, no module index is generated.
#texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.
#texinfo_show_urls = 'footnote'

# -- Options for Epub output ---------------------------------------------

# Bibliographic Dublin Core info.
epub_title = u'HyperSpy User Guide'
epub_author = u'The HyperSpy Developers'
epub_publisher = u'he HyperSpy Developers'
epub_copyright = u'2011-2013, he HyperSpy Developers'

# The language of the text. It defaults to the language option
# or en if the language is not set.
#epub_language = ''

# The scheme of the identifier. Typical schemes are ISBN or URL.
#epub_scheme = ''

# The unique identifier of the text. This can be a ISBN number
# or the project homepage.
#epub_identifier = ''

# A unique identification for the text.
#epub_uid = ''

# A tuple containing the cover image and cover page html template filenames.
#epub_cover = ()

# HTML files that should be inserted before the pages created by sphinx.
# The format is a list of tuples containing the path and title.
#epub_pre_files = []

# HTML files shat should be inserted after the pages created by sphinx.
# The format is a list of tuples containing the path and title.
#epub_post_files = []

# A list of files that should not be packed into the epub file.
#epub_exclude_files = []

# The depth of the table of contents in toc.ncx.
#epub_tocdepth = 3

# Allow duplicate toc entries.
#epub_tocdup = True

# Example configuration for intersphinx: refer to the Python standard library.
intersphinx_mapping = {'hyperspyweb': ('http://hyperspy.org/', None)}


import os
import subprocess
import sklearn.cluster
import numpy as np
import autogp
from autogp import likelihoods
from autogp import kernels
import tensorflow as tf
from autogp import datasets
from autogp import losses
from autogp  import util
import pandas
import scipy.io as sio

DATA_DIR = "experiments/data/"
TRAIN_PATH = DATA_DIR + "sarcos_inv.mat"
TEST_PATH = DATA_DIR + "sarcos_inv_test"

def init_z(train_inputs, num_inducing):
    # Initialize inducing points using clustering.
    mini_batch = sklearn.cluster.MiniBatchKMeans(num_inducing)
    cluster_indices = mini_batch.fit_predict(train_inputs)
    inducing_locations = mini_batch.cluster_centers_
    return inducing_locations

def get_sarcos_data():
    print "Getting sarcos data ..."
    os.chdir('experiments/data')
    subprocess.call(["./get_sarcos_data.sh"])
    os.chdir("../../")
    print "done"

def sarcos_all_joints_data():
    """
    Loads and returns data of SARCOS dataset for all joints.

    Returns
    -------
    data : list
        A list of length = 1, where each element is a dictionary which contains ``train_outputs``,
        ``train_inputs``, ``test_outputs``, ``test_inputs``, and ``id``
    """

    train = sio.loadmat(TRAIN_PATH)['sarcos_inv']
    test = sio.loadmat(TEST_PATH)['sarcos_inv_test']
    return{
        'train_inputs': train[:, :21],
        'train_outputs': train[:, 21:],
        'test_inputs': test[:, :21],
        'test_outputs': test[:, 21:],
        'id': 0
    }

if __name__ == '__main__':
    FLAGS = util.util.get_flags()
    BATCH_SIZE = FLAGS.batch_size
    LEARNING_RATE = FLAGS.learning_rate
    DISPLAY_STEP = FLAGS.display_step
    EPOCHS = FLAGS.n_epochs
    NUM_SAMPLES =  FLAGS.mc_train
    NUM_INDUCING = FLAGS.n_inducing
    IS_ARD = FLAGS.is_ard

    if os.path.exists(TRAIN_PATH) is False:  # directory does not exist, download the data
        get_sarcos_data()

    d = sarcos_all_joints_data()
    data = datasets.DataSet(d['train_inputs'].astype(np.float32), d['train_outputs'].astype(np.float32))
    test = datasets.DataSet(d['test_inputs'].astype(np.float32), d['test_outputs'].astype(np.float32))

    # Setup initial values for the model.
    likelihood = likelihoods.RegressionNetwork(7, 0.1)
    kern = [kernels.RadialBasis(data.X.shape[1], lengthscale=8.0, input_scaling = IS_ARD) for i in range(8)]
    # kern = [kernels.ArcCosine(data.X.shape[1], 1, 3, 5.0, 1.0, input_scaling=True) for i in range(10)]

    Z = init_z(data.X, NUM_INDUCING)
    m = autogp.GaussianProcess(likelihood, kern, Z, num_samples=NUM_SAMPLES)

    # setting up loss to be reported during training
    error_rate = None #losses.StandardizedMeanSqError(d['train_outputs'].astype(np.float32), data.Dout)

    import time
    o = tf.train.RMSPropOptimizer(LEARNING_RATE)
    start = time.time()
    m.fit(data, o, loo_steps=0, var_steps=50, epochs = EPOCHS, batch_size = BATCH_SIZE, display_step=DISPLAY_STEP, test = test,
            loss = error_rate )
    print time.time() - start

    ypred = m.predict(test.X)[0]
    print("Final " + error_rate.get_name() + "=" + "%.4f" % error_rate.eval(test.Y, ypred))