# coding: utf-8

# ### Sentiment Analysis on "Jallikattu" with Twitter Data Feed
#DataScienceForSocialCause

#
# Twitter is flooded with Jallikattu issue, let us find peoples sentiment with Data Science tools. Following is the approach
# * Register a Twitter API handle for data feed
# * Pull out tweets on search query 'jallikattu'
# * Using NLP packages find the sentiment of the tweet (Positive, Neutral or Negative)
# * Plot pie chart of the sentiment
# * Plot a masked word cloud of tags used
#
# Finall output we expect is a masked word cloud of popular tags used in twitter with font size propotional to the frequency of use. Let's dive in ...

# ### Loading necessary packages
#
# In particular we will be using tweepy to register an api handle with twitter and get the data feed. [Tweepy Document](http://docs.tweepy.org/en/v3.5.0/)
# TextBlob package to determine the sentiment of the tweets. [TextBlob Document](https://textblob.readthedocs.io/en/dev/)
#
#

# In[1]:

# import tweepy for twitter datastream and textblob for processing tweets
import tweepy
import textblob

# wordcloud package is used to produce the cool masked tag cloud above
from wordcloud import WordCloud

# pickle to serialize/deserialize python objects
import pickle

# regex package to extract hasttags from tweets
import re

# os for loading files from local system, matplotlib, np and PIL for ploting
from os import path
import matplotlib.pyplot as plt
import numpy as np
from PIL import Image

# ### We will create a Twitter API handle for fetching data
#
# * Inorder to qualify for a Twitter API handle you need to be a **`Phone Verified`** Twitter user.
#     1. Goto Twitter settings page [twitter.com/settings/account](https://twitter.com/settings/account)
#     2. Choose Mobile tab on left pane, then enter your phone number and verify by OTP
#     3. Now you should be able to register new API handle for your account for `programmatic tweeting`
#
#
# * Now goto Twitter [Application Management](https://apps.twitter.com/) page
# * Click *Create New App*button
#     1. Enter a Unique App name(global namespace), you might have to try few time to get it correct
#     2. Description can be anything you wish
#     3. website can be some <yourname>.com, you dont really have to own the domain
#     4. Leave the callback URL empty, agree to the terms and condition unconditionally
#     5. Click create
#
#
# * You can find the api credentials in [Application Management](https://apps.twitter.com/) consol
# * Choose the App and goto *keys and access tokens* tab to get API_KEY, API_SECRET, ACCESS_TOKEN and ACCESS_TOKEN_SECRET
#
# #### RUN THE CODE BLOCK BELOW ONLY ON FIRST TIME YOU CONFIGURE THE TWITTER API

# In[ ]:

# make sure to exclued this folder in git ignore
path_to_cred_file = path.abspath('../restricted/api_credentials.p')

# we will store twitter handle credentials in a pickle file (object de-serialization)
# code for pickling credentials need to be run only once during initial configuration
# fill the following dictionary with your twitter credentials
twitter_credentials = {'api_key':'API_KEY',                        'api_secret':'API_SECRET',                        'access_token':'ACCESS_TOKEN',                        'access_token_secret':'ACCESS_TOKEN_SECRET'}
pickle.dump(twitter_credentials,open(path_to_cred_file, "wb"))
print("Pickled credentials saved to :\n"+path_to_cred_file+"\n")
print("\n".join(["{:20} : {}".format(key,value) for key,value in twitter_credentials.items()]))

# #### From second run you can load the credentials securely form stored file
# If you want to check the credentials uncomment the last line in below code block

# In[2]:

# make sure to exclued this folder in git ignore
path_to_cred_file = path.abspath('../restricted/api_credentials.p')

# load saved twitter credentials
twitter_credentials = pickle.load(open(path_to_cred_file,'rb'))
#print("\n".join(["{:20} : {}".format(key,value) for key,value in twitter_credentials.items()]))

# ### Creating an Open Auth Instance
# With the created api and token we will open an open auth instance to authenticate our twitter account.
#
# If you feel that your twitter api credentials have been compromised you can just generate a new set of access token-secret pair, access token is like RSA to authenticate your api key.

# In[3]:

# lets create an open authentication handler and initialize it with our twitter handlers api key
auth = tweepy.OAuthHandler(twitter_credentials['api_key'],twitter_credentials['api_secret'])

# access token is like password for the api key,
auth.set_access_token(twitter_credentials['access_token'],twitter_credentials['access_token_secret'])

# ### Twitter API Handle
#
# Tweepy comes with a Twitter API wrapper class called 'API', passing the open auth instance to this API creates a live Twitter handle to our account.
#
#
# **ATTENTION: Please beware that this is a handle you your own account not any pseudo account, if you tweet something with this it will be your tweet** This is the reason I took care not to expose my api credentials, if you expose anyone can mess up your Twitter account.
#
#
# Let's open the twitter handle and print the Name and Location of the twitter account owner, you should be seeing your name.

# In[4]:

# lets create an instance of twitter api wrapper
api = tweepy.API(auth)

# lets do some self check
user = api.me()
print("{}\n{}".format(user.name,user.location))

# ### Inspiration for this Project
# I drew inspiration for this project from the ongoing issue on traditional bull fighting AKA *Jallikattu*. Here I'm trying read  pulse of the people based on tweets.
#
# We are searching for key word *Jallikattu* in Twitters public tweets, in the retured search result we are taking 150 tweets to do our **Sentiment Analysis**. Please dont go for large number of tweets there is an upper limit of 450 tweets, for more on api rate limits checkout [Twitter Developer Doc](https://dev.twitter.com/rest/public/rate-limits).

# In[5]:

# now lets get some data to check the sentiment on it
# lets search for key word jallikattu and check the sentiment on it
query = 'jallikattu'
tweet_cnt = 150
peta_tweets = api.search(q=query,count=tweet_cnt)

# ### Processing Tweets
#
# Once we get the tweets, we will iterate through the tweets and do following oprations
# 1. Pass the tweet text to TextBlob to process the tweet
# 2. Processed tweets will have two attributes
#   * Polarity which is a numerical value between -1 to 1, the sentiment of the text can be infered from this.
#   * Subjectivity this shows wheather the text is stated as a fact or an opinion, value ranges from 0 to 1
# 3. For each tweet we will find sentiment of the text (positive, neutral or negative) and update a counter variable accordingly, this counter is later ploted as a **pie chart**.
# 4. Then we pass the tweet text to a regular expression to extract hash tags, which we later use to create an awesome **word cloud visualization**.

# In[6]:

# lets go over the tweets
sentiment_polarity = [0,0,0]
tags = []

for tweet in peta_tweets:
    processed_tweet = textblob.TextBlob(tweet.text)
    polarity = processed_tweet.sentiment.polarity
    upd_index = 0 if polarity > 0 else (1 if polarity == 0 else 2)
    sentiment_polarity[upd_index] = sentiment_polarity[upd_index]+1
    tags.extend(re.findall(r"#(\w+)", tweet.text))
    #print(tweet.text)
    #print(processed_tweet.sentiment,'\n')

sentiment_label = ['Positive','Neutral','Negative']
#print("\n".join(["{:8} tweets count {}".format(s,val) for s,val in zip(sentiment_label,sentiment_polarity)]))

# plotting sentiment pie chart
colors = ['yellowgreen', 'gold', 'coral']

# lets explode the positive sentiment for visual appeal
explode = (0.1, 0, 0)
plt.pie(sentiment_polarity,labels=sentiment_label,colors=colors,explode=explode,shadow=True,autopct='%1.1f%%')
plt.axis('equal')
plt.legend(bbox_to_anchor=(1.3,1))
plt.title('Twitter Sentiment on \"'+query+'\"')
plt.show()

# ### Sentiment Analysis
#
# We can see that majority is neutral which is contributed by
# 1. Tweets with media only(photo, video)
# 2. Tweets in regional language. Textblob do not work on our indian languages.
# 3. Some tweets contains only stop words or the words that do not give any positive or negative perspective.
# 4. Polarity is calculated by the number of positive words like "great, awesome, etc." or negative words like "hate, bad, etc"
#
# One more point to note is that TextBlob is not a complete NLP package it does not do context aware search, such sophisticated deep learing abilities are available only with likes of Google.

# In[7]:

# lets process the hash tags in the tweets and make a word cloud visualization
# normalizing tags by converting all tags to lowercase
tags = [t.lower() for t in tags]

# get unique count of tags to take count for each
uniq_tags = list(set(tags))
tag_count = []

# for each unique hash tag take frequency of occurance
for tag in uniq_tags:
    tag_count.append((tag,tags.count(tag)))

# lets print the top five tags
tag_count =sorted(tag_count,key=lambda x:-x[1])[:5]
print("\n".join(["{:8}  {}".format(tag,val) for tag,val in tag_count]))

# ### Simple Word Cloud with Twitter #tags
#
# Let us viualize the tags used in for Jallikattu by creating a tag cloud. The wordcloud package takes a single string of tags separated by whitespace. We will concatinate the tags and pass it to generate method to create a tag cloud image.

# In[8]:

# we will create a vivid tag cloud visualization
# creating a single string of texts from tags, the tag's font size is proportional to its frequency
text = " ".join(tags)

# this generates an image from the long string, if you wish you may save it to local
wc = WordCloud().generate(text)

# we will display the image with matplotlibs image show, removed x and y axis ticks
plt.imshow(wc)
plt.axis("off")
plt.show()

# ### Masked Word Cloud
#
# The tag cloud can be masked using a grascale stencil image the wordcloud package neatly arranges the word in side the mask image. I have supreimposed generated word cloud image on to the mask image to provide a detailing otherwise the background of the word cloud will be white and it will appeare like words are hanging in space instead.
#
#
# Inorder to make the image superimposing work well, we need to manipulate image transparency using image alpha channel. If you look at the visual only fine detail of mask image is seen in the tag cloud this is bacause word cloud is layed on mask image and the transparency of word cloud image is 90% so only 10% of mask image is seen.

# In[11]:

# we can also create a masked word cloud from the tags by using grayscale image as stencil
# lets load the mask image from local
bull_mask = np.array(Image.open(path.abspath('../asset/bull_mask_1.jpg')))

wc_mask = WordCloud(background_color="white", mask=bull_mask).generate(text)
mask_image = plt.imshow(bull_mask, cmap=plt.cm.gray)
word_cloud = plt.imshow(wc_mask,alpha=0.9)
plt.axis("off")
plt.title("Twitter Hash Tag Word Cloud for "+query)
plt.show()

# The tag cloud marks the key moments like the call for protest in Chennai Marina, Alanganallur. Also shows one of a leading actors support for the cause and calls for ban on peta.
#
# This code will give different output over time as new tweet are added in timeline and old ones are pushed down,

# #### Thank you for showing intrest in my work
# If you liked it and want to be notified of my future work follow me on
#
#
# [Knowme](https://knome.ultimatix.net/users/286632-prabakaran-k)
#
#
# [@iPrabakaran](https://twitter.com/iPrabakaran) Twitter
#
#
# [GitHub](https://github.com/nixphix)


# tests.checks
# Performs checking that visualizers adhere to Yellowbrick conventions.
#
# Author:   Benjamin Bengfort <bbengfort@districtdatalabs.com>
# Created:  Mon May 22 11:18:06 2017 -0700
#
# Copyright (C) 2017 District Data Labs
# For license information, see LICENSE.txt
#
# ID: checks.py [4131cb1] benjamin@bengfort.com $

"""
Performs checking that visualizers adhere to Yellowbrick conventions.
"""

##########################################################################
## Imports
##########################################################################

import sys
sys.path.append("..")

import numpy as np
import matplotlib.pyplot as plt

from yellowbrick.base import ModelVisualizer, ScoreVisualizer
from yellowbrick.classifier.base import ClassificationScoreVisualizer
from yellowbrick.cluster.base import ClusteringScoreVisualizer
from yellowbrick.features.base import FeatureVisualizer, DataVisualizer
from yellowbrick.regressor.base import RegressionScoreVisualizer
from yellowbrick.text.base import TextVisualizer

##########################################################################
## Checking runable
##########################################################################

def check_visualizer(Visualizer):
    """
    Check if visualizer adheres to Yellowbrick conventions.

    This function runs an extensive test-suite for input validation, return
    values, exception handling, and more. Additional tests for scoring or
    tuning visualizers will be run if the Visualizer clss inherits from the
    corresponding object.
    """
    name = Visualizer.__name__
    for check in _yield_all_checks(name, Visualizer):
        check(name, Visualizer)

##########################################################################
## Generate the specific per-visualizer checking
##########################################################################

def _yield_all_checks(name, Visualizer):
    """
    Composes the checks required for the specific visualizer.
    """

    # Global Checks
    yield check_instantiation
    yield check_estimator_api

    # Visualizer Type Checks
    if issubclass(Visualizer, RegressionScoreVisualizer):
        for check in _yield_regressor_checks(name, Visualizer):
            yield check

    if issubclass(Visualizer, ClassificationScoreVisualizer):
        for check in _yield_classifier_checks(name, Visualizer):
            yield check

    if issubclass(Visualizer, ClusteringScoreVisualizer):
        for check in _yield_clustering_checks(name, Visualizer):
            yield check

    if issubclass(Visualizer, FeatureVisualizer):
        for check in _yield_feature_checks(name, Visualizer):
            yield check

    if issubclass(Visualizer, TextVisualizer):
        for check in _yield_text_checks(name, Visualizer):
            yield check

    # Other checks

def _yield_regressor_checks(name, Visualizer):
    """
    Checks for regressor visualizers
    """
    pass

def _yield_classifier_checks(name, Visualizer):
    """
    Checks for classifier visualizers
    """
    pass

def _yield_clustering_checks(name, Visualizer):
    """
    Checks for clustering visualizers
    """
    pass

def _yield_feature_checks(name, Visualizer):
    """
    Checks for feature visualizers
    """
    pass

def _yield_text_checks(name, Visualizer):
    """
    Checks for text visualizers
    """
    pass

##########################################################################
## Checking Functions
##########################################################################

def check_instantiation(name, Visualizer, args, kwargs):
    # assert that visualizers can be passed an axes object.
    ax = plt.gca()

    viz = Visualizer(*args, **kwargs)
    assert viz.ax == ax

def check_estimator_api(name, Visualizer):
    X = np.random.rand((5, 10))
    y = np.random.randint(0,2, 10)

    # Ensure fit returns self.
    viz = Visualizer()
    self = viz.fit(X, y)
    assert viz == self

if __name__ == '__main__':
    import sys
    sys.path.append("..")

    from yellowbrick.classifier import *
    from yellowbrick.cluster import *
    from yellowbrick.features import *
    from yellowbrick.regressor import *
    from yellowbrick.text import *

    visualizers = [
        ClassBalance, ClassificationReport, ConfusionMatrix, ROCAUC,
        KElbowVisualizer, SilhouetteVisualizer,
        ScatterVisualizer, JointPlotVisualizer, Rank2D, RadViz, ParallelCoordinates,
        AlphaSelection, ManualAlphaSelection,
        PredictionError, ResidualsPlot,
        TSNEVisualizer, FreqDistVisualizer, PosTagVisualizer
    ]

    for visualizer in visualizers:
        check_visualizer(visualizer)


DNS Server/Source - Copy/Lib/site-packages/dask/dataframe/tests/test_merge_column_and_index.py

import dask.dataframe as dd
import numpy as np
import pandas as pd
import pytest

from dask.dataframe.utils import assert_eq, PANDAS_VERSION

# Fixtures
# ========
@pytest.fixture
def df_left():
    # Create frame with 10 partitions
    # Frame has 11 distinct idx values
    partition_sizes = np.array([3, 4, 2, 5, 3, 2, 5, 9, 4, 7, 4])
    idx = [i for i, s in enumerate(partition_sizes) for _ in range(s)]
    k = [i for s in partition_sizes for i in range(s)]
    vi = range(len(k))

    return pd.DataFrame(dict(
        idx=idx,
        k=k,
        v1=vi
    )).set_index(['idx'])

@pytest.fixture
def df_right():
    # Create frame with 10 partitions
    # Frame has 11 distinct idx values
    partition_sizes = np.array([4, 2, 5, 3, 2, 5, 9, 4, 7, 4, 8])
    idx = [i for i, s in enumerate(partition_sizes) for _ in range(s)]
    k = [i for s in partition_sizes for i in range(s)]
    vi = range(len(k))

    return pd.DataFrame(dict(
        idx=idx,
        k=k,
        v1=vi
    )).set_index(['idx'])

@pytest.fixture
def ddf_left(df_left):
    # Create frame with 10 partitions
    # Skip division on 2 so there is one mismatch with ddf_right
    return dd.repartition(df_left, [0, 1, 3, 4, 5, 6, 7, 8, 9, 10, 11])

@pytest.fixture
def ddf_left_unknown(ddf_left):
    return ddf_left.clear_divisions()

@pytest.fixture
def ddf_left_single(df_left):
    return dd.from_pandas(df_left, npartitions=1, sort=False)

@pytest.fixture
def ddf_right(df_right):
    # Create frame with 10 partitions
    # Skip division on 3 so there is one mismatch with ddf_left
    return dd.repartition(df_right, [0, 1, 2, 4, 5, 6, 7, 8, 9, 10, 11])

@pytest.fixture
def ddf_right_unknown(ddf_right):
    return ddf_right.clear_divisions()

@pytest.fixture
def ddf_right_single(df_right):
    return dd.from_pandas(df_right, npartitions=1, sort=False)

@pytest.fixture(params=['inner', 'left', 'right', 'outer'])
def how(request):
    return request.param

@pytest.fixture(params=[
    'idx',
    ['idx'],
    ['idx', 'k'],
    ['k', 'idx']])
def on(request):
    return request.param

# Tests
# =====
@pytest.mark.skipif(PANDAS_VERSION < '0.23.0',
                    reason="Need pandas col+index merge support (pandas-dev/pandas#14355)")
def test_merge_known_to_known(df_left, df_right, ddf_left, ddf_right, on, how):
    # Compute expected
    expected = df_left.merge(df_right, on=on, how=how)

    # Perform merge
    result = ddf_left.merge(ddf_right, on=on, how=how, shuffle='tasks')

    # Assertions
    assert_eq(result, expected)
    assert_eq(result.divisions, tuple(range(12)))
    assert len(result.__dask_graph__()) < 80

@pytest.mark.skipif(PANDAS_VERSION < '0.23.0',
                    reason="Need pandas col+index merge support (pandas-dev/pandas#14355)")
@pytest.mark.parametrize('how', ['inner', 'left'])
def test_merge_known_to_single(df_left, df_right, ddf_left, ddf_right_single, on, how):
    # Compute expected
    expected = df_left.merge(df_right, on=on, how=how)

    # Perform merge
    result = ddf_left.merge(ddf_right_single, on=on, how=how, shuffle='tasks')

    # Assertions
    assert_eq(result, expected)
    assert_eq(result.divisions, ddf_left.divisions)
    assert len(result.__dask_graph__()) < 30

@pytest.mark.skipif(PANDAS_VERSION < '0.23.0',
                    reason="Need pandas col+index merge support (pandas-dev/pandas#14355)")
@pytest.mark.parametrize('how', ['inner', 'right'])
def test_merge_single_to_known(df_left, df_right, ddf_left_single, ddf_right, on, how):
    # Compute expected
    expected = df_left.merge(df_right, on=on, how=how)

    # Perform merge
    result = ddf_left_single.merge(ddf_right, on=on, how=how, shuffle='tasks')

    # Assertions
    assert_eq(result, expected)
    assert_eq(result.divisions, ddf_right.divisions)
    assert len(result.__dask_graph__()) < 30

@pytest.mark.skipif(PANDAS_VERSION < '0.23.0',
                    reason="Need pandas col+index merge support (pandas-dev/pandas#14355)")
def test_merge_known_to_unknown(df_left, df_right, ddf_left, ddf_right_unknown, on, how):
    # Compute expected
    expected = df_left.merge(df_right, on=on, how=how)

    # Perform merge
    result = ddf_left.merge(ddf_right_unknown, on=on, how=how, shuffle='tasks')

    # Assertions
    assert_eq(result, expected)
    assert_eq(result.divisions, tuple(None for _ in range(11)))
    assert len(result.__dask_graph__()) >= 400

@pytest.mark.skipif(PANDAS_VERSION < '0.23.0',
                    reason="Need pandas col+index merge support (pandas-dev/pandas#14355)")
def test_merge_unknown_to_known(df_left, df_right, ddf_left_unknown, ddf_right, on, how):
    # Compute expected
    expected = df_left.merge(df_right, on=on, how=how)

    # Perform merge
    result = ddf_left_unknown.merge(ddf_right, on=on, how=how, shuffle='tasks')

    # Assertions
    assert_eq(result, expected)
    assert_eq(result.divisions, tuple(None for _ in range(11)))
    assert len(result.__dask_graph__()) > 400

@pytest.mark.skipif(PANDAS_VERSION < '0.23.0',
                    reason="Need pandas col+index merge support (pandas-dev/pandas#14355)")
def test_merge_unknown_to_unknown(df_left, df_right, ddf_left_unknown, ddf_right_unknown, on, how):
    # Compute expected
    expected = df_left.merge(df_right, on=on, how=how)

    # Merge unknown to unknown
    result = ddf_left_unknown.merge(ddf_right_unknown, on=on, how=how, shuffle='tasks')

    # Assertions
    assert_eq(result, expected)
    assert_eq(result.divisions, tuple(None for _ in range(11)))
    assert len(result.__dask_graph__()) > 400


# -*- coding: utf-8 -*-
#
# connplotter_tutorial.py
#
# This file is part of NEST.
#
# Copyright (C) 2004 The NEST Initiative
#
# NEST is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 2 of the License, or
# (at your option) any later version.
#
# NEST is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with NEST.  If not, see <http://www.gnu.org/licenses/>.

# !========================
# ! ConnPlotter: A Tutorial
# !========================
# !
# ! :Author: Hans Ekkehard Plesser
# ! :Institution: Norwegian University of Life Sciences, Simula
# !               Research Laboratory, RIKEN Brain Sciences Institute
# ! :Version: 0.7
# ! :Date: 1 December 2009
# ! :Copyright: Hans Ekkehard Plesser
# ! :License: Creative Commons Attribution-Noncommercial-Share Alike License
# !           v 3.0
# !
# ! :Note: For best results, you should run this script with PyReport by
# !        Gael Varoquaux, available from
# !        http://gael-varoquaux.info/computers/pyreport/
# !
# !        Please set using_pyreport to True if you want to run the
# !        script through pyreport. Otherwise, figures will not be captured
# !        correctly.

using_pyreport = False

# ! Introduction
# !=============
# ! This tutorial gives a brief introduction to the ConnPlotter
# ! toolbox.  It is by no means complete.

# ! Avoid interactive backend when using pyreport
if using_pyreport:
    import matplotlib

    matplotlib.use("Agg")

# ! Import pylab to call pylab.show() so that pyreport
# ! can capture figures created. Must come before import
# ! ConnPlotter so we get the correct show().
import pylab

# ! If not using pyreport, disable pylab.show() until we reach end of script
if not using_pyreport:
    pylab_show = pylab.show

    def nop(s=None):
        pass

    pylab.show = nop

# ! Import ConnPlotter and its examples
import ConnPlotter as cpl
import ConnPlotter.examples as ex

# ! Turn of warnings about resized figure windows
import warnings

warnings.simplefilter("ignore")

# ! Define a helper function to show LaTeX tables on the fly
def showTextTable(connPattern, fileTrunk):
    """
    Shows a Table of Connectivity as textual table.

    Arguments:
    connPattern  ConnectionPattern instance
    fileTrunk    Eventual PNG image will be fileTrunk.png
    """
    import subprocess as subp  # to call LaTeX etc
    import os  # to remove files
    # Write to LaTeX file so we get a nice textual representation
    # We want a complete LaTeX document, so we set ``standalone``
    # to ``True``.
    connPattern.toLaTeX(file=fileTrunk + '.tex', standalone=True,
                        enumerate=True)
    # Create PDF, crop, and convert to PNG
    try:
        devnull = open('/dev/null', 'w')
        subp.call(['pdflatex', fileTrunk], stdout=devnull, stderr=subp.STDOUT)
        # need wrapper, since pdfcrop does not begin with #!
        subp.call(['pdfcrop ' + fileTrunk + '.pdf ' + fileTrunk + '-crop.pdf'],
                  shell=True,
                  stdout=devnull, stderr=subp.STDOUT)
        devnull.close()
        os.rename(fileTrunk + '-crop.pdf', fileTrunk + '.pdf')
        for suffix in ('.tex', '-crop.pdf', '.png', '.aux', '.log'):
            if os.path.exists(fileTrunk + suffix):
                os.remove(fileTrunk + suffix)
    except:
        raise Exception('Could not create PDF Table.')

# ! Simple network
# ! ==============

# ! This is a simple network with two layers A and B; layer B has two
# ! populations, E and I. On the NEST side, we use only synapse type
# ! ``static_synapse``. ConnPlotter then infers that synapses with positive
# ! weights should have type ``exc``, those with negative weight type ``inh``.
# !  Those two types are know to ConnPlotter.

# ! Obtain layer, connection and model list from the example set
s_layer, s_conn, s_model = ex.simple()

# ! Create Connection Pattern representation
s_cp = cpl.ConnectionPattern(s_layer, s_conn)

# ! Show pattern as textual table (we cheat a little and include PDF directly)
showTextTable(s_cp, 'simple_tt')
# $ \centerline{\includegraphics{simple_tt.pdf}}

# ! Show pattern in full detail
# ! ---------------------------

# ! A separate patch is shown for each pair of populations.
# !
# !  - Rows represent senders, columns targets.
# !  - Layer names are given to the left/above, population names to the right
# !    and below.
# !  - Excitatory synapses shown in blue, inhibitory in red.
# !  - Each patch has its own color scale.
s_cp.plot()
pylab.show()

# ! Let us take a look at what this connection pattern table shows:
# !
# ! - The left column, with header "A", is empty: The "A" layer receives
# !   no input.
# ! - The right column shows input to layer "B"
# !
# !   * The top row, labeled "A", has two patches in the "B" column:
# !
# !     + The left patch shows relatively focused input to the "E" population
# !       in layer "B" (first row of "Connectivity" table).
# !     + The right patch shows wider input to the "I" population in layer
# !       "B" (second row of "Connectivity" table).
# !     + Patches are red, indicating excitatory connections.
# !     + In both cases, mask are circular, and the product of connection
# !       weight and probability is independent of the distance between sender
# !       and target neuron.
# !
# !   * The grey rectangle to the bottom right shows all connections from
# !     layer "B" populations to layer "B" populations. It is subdivided into
# !     two rows and two columns:
# !
# !     + Left column: inputs to the "E" population.
# !     + Right column: inputs to the "I" population.
# !     + Top row: projections from the "E" population.
# !     + Bottom row: projections from the "I" population.
# !     + There is only one type of synapse for each sender-target pair,
# !       so there is only a single patch per pair.
# !     + Patches in the top row, from population "E" show excitatory
# !        connections, thus they are red.
# !     + Patches in the bottom row, from population "I" show inhibitory
# !       connections, thus they are blue.
# !     + The patches in detail are:
# !
# !       - **E to E** (top-left, row 3+4 in table): two rectangular
# !          projections at 90 degrees.
# !       - **E to I** (top-right, row 5 in table): narrow gaussian projection.
# !       - **I to E** (bottom-left, row 6 in table): wider gaussian projection
# !       - **I to I** (bottom-right, row 7 in table): circular projection
# !         covering entire layer.
# !
# ! - **NB:** Color scales are different, so one **cannot** compare connection
# !   strengths between patches.

# ! Full detail, common color scale
# ! -------------------------------
s_cp.plot(globalColors=True)
pylab.show()

# ! This figure shows the same data as the one above, but now all patches use
# ! a common color scale, so full intensity color (either red or blue)
# ! indicates the strongest connectivity. From this we see that
# !
# ! - A to B/E is stronger than A to B/I
# ! - B/E to B/I is the strongest of all connections at the center
# ! - B/I to B/E is stronger than B/I to B/I

# ! Aggregate by groups
# ! -------------------
# ! For each pair of population groups, sum connections of the same type
# ! across populations.
s_cp.plot(aggrGroups=True)
pylab.show()

# ! In the figure above, all excitatory connections from B to B layer have been
# ! combined into one patch, as have all inhibitory connections from B to B.
# ! In the upper-right corner, all connections from layer A to layer B have
# ! been combined; the patch for inhibitory connections is missing, as there
# ! are none.

# ! Aggregate by groups and synapse models
# ! --------------------------------------
s_cp.plot(aggrGroups=True, aggrSyns=True)
pylab.show()
# ! When aggregating across synapse models, excitatory and inhibitory
# ! connections are combined. By default, excitatory connections are weights
# ! with +1, inhibitory connections with -1 in the sum. This may yield kernels
# ! with positive and negative values. They are shown on a red-white-blue scale
# ! as follows:
# !
# ! - White always represents 0.
# ! - Positive values are represented by increasingly saturated red.
# ! - Negative values are represented by increasingly saturated blue.
# ! - Colorscales are separate for red and blue:
# !
# !   * largest positive value: fully saturated red
# !   * largest negative value: fully saturated blue
# !
# ! - Each patch has its own colorscales.
# ! - When ``aggrSyns=True`` is combined with ``globalColors=True``,
# !   all patches use the same minimum and maximum in their red and blue
# !   color scales. The the minimum is the negative of the maximum, so that
# !   blue and red intesities can be compared.
s_cp.plot(aggrGroups=True, aggrSyns=True, globalColors=True)
pylab.show()

# ! - We can explicitly set the limits of the color scale; if values exceeding
# !   the limits are present, this is indicated by an arrowhead at the end of
# !   the colorbar. User-defined color limits need not be symmetric about 0.
s_cp.plot(aggrGroups=True, aggrSyns=True, globalColors=True,
          colorLimits=[-2, 3])
pylab.show()

# ! Save pattern to file
# ! --------------------
# s_cp.plot(file='simple_example.png')

# ! This saves the detailed diagram to the given file. If you want to save
# ! the pattern in several file formats, you can pass a tuple of file names,
# ! e.g., ``s_cp.plot(file=('a.eps', 'a.png'))``.
# !
# ! **NB:** Saving directly to PDF may lead to files with artifacts. We
# ! recommend to save to EPS and the convert to PDF.

# ! Build network in NEST
# ! ---------------------
import nest
import nest.topology as topo

# ! Create models
for model in s_model:
    nest.CopyModel(model[0], model[1], model[2])

# ! Create layers, store layer info in Python variable
for layer in s_layer:
    exec ('%s = topo.CreateLayer(layer[1])' % layer[0])

# ! Create connections, need to insert variable names
for conn in s_conn:
    eval('topo.ConnectLayers(%s,%s,conn[2])' % (conn[0], conn[1]))

nest.Simulate(10)
# ! **Ooops:*** Nothing happened? Well, it did, but pyreport cannot capture the
# ! output directly generated by NEST. The absence of an error message in this
# ! place shows that network construction and simulation went through.

# ! Inspecting the connections actually created
# ! :::::::::::::::::::::::::::::::::::::::::::
# ! The following block of messy and makeshift code plots the targets of the
# ! center neuron of the B/E population in the B/E and the B/I populations.
B_top = nest.GetStatus(RG, 'topology')[0]
ctr_id = topo.GetElement(RG,
                         [int(B_top['rows'] / 2), int(B_top['columns'] / 2)])

# find excitatory element in B
E_id = [gid for gid in ctr_id
        if nest.GetStatus([gid], 'model')[0] == 'E']

# get all targets, split into excitatory and inhibitory
alltgts = nest.GetStatus(
    nest.GetConnections(E_id, synapse_model='static_synapse'), 'target')
Etgts = [t for t in alltgts if nest.GetStatus([t], 'model')[0] == 'E']
Itgts = [t for t in alltgts if nest.GetStatus([t], 'model')[0] == 'I']

# obtain positions of targets
Etpos = tuple(zip(*topo.GetPosition(Etgts)))
Itpos = tuple(zip(*topo.GetPosition(Itgts)))

# plot excitatory
pylab.clf()
pylab.subplot(121)
pylab.scatter(Etpos[0], Etpos[1])
ctrpos = pylab.array(topo.GetPosition(E_id)[0])
ax = pylab.gca()
ax.add_patch(pylab.Circle(ctrpos, radius=0.02, zorder=99,
                          fc='r', alpha=0.4, ec='none'))
ax.add_patch(
    pylab.Rectangle(ctrpos + pylab.array((-0.4, -0.2)), 0.8, 0.4, zorder=1,
                    fc='none', ec='r', lw=3))
ax.add_patch(
    pylab.Rectangle(ctrpos + pylab.array((-0.2, -0.4)), 0.4, 0.8, zorder=1,
                    fc='none', ec='r', lw=3))
ax.add_patch(
    pylab.Rectangle(ctrpos + pylab.array((-0.5, -0.5)), 1.0, 1.0, zorder=1,
                    fc='none', ec='k', lw=3))
ax.set(aspect='equal', xlim=[-0.5, 0.5], ylim=[-0.5, 0.5],
       xticks=[], yticks=[])

# plot inhibitory
pylab.subplot(122)
pylab.scatter(Itpos[0], Itpos[1])
ctrpos = topo.GetPosition(E_id)[0]
ax = pylab.gca()
ax.add_patch(pylab.Circle(ctrpos, radius=0.02, zorder=99,
                          fc='r', alpha=0.4, ec='none'))
ax.add_patch(pylab.Circle(ctrpos, radius=0.1, zorder=2,
                          fc='none', ec='r', lw=2, ls='dashed'))
ax.add_patch(pylab.Circle(ctrpos, radius=0.2, zorder=2,
                          fc='none', ec='r', lw=2, ls='dashed'))
ax.add_patch(pylab.Circle(ctrpos, radius=0.3, zorder=2,
                          fc='none', ec='r', lw=2, ls='dashed'))
ax.add_patch(pylab.Circle(ctrpos, radius=0.5, zorder=2,
                          fc='none', ec='r', lw=3))
ax.add_patch(pylab.Rectangle((-0.5, -0.5), 1.0, 1.0, zorder=1,
                             fc='none', ec='k', lw=3))
ax.set(aspect='equal', xlim=[-0.5, 0.5], ylim=[-0.5, 0.5],
       xticks=[], yticks=[])
pylab.show()

# ! Thick red lines mark the mask, dashed red lines to the right one, two and
# ! three standard deviations. The sender location is marked by the red spot
# ! in the center. Layers are 40x40 in size.

# ! A more complex network
# ! ======================
# !
# ! This network has layers A and B, with E and I populations in B. The added
# ! complexity comes from the fact that we now have four synapse types: AMPA,
# ! NMDA, GABA_A and GABA_B. These synapse types are known to ConnPlotter.

# ! Setup and tabular display
c_layer, c_conn, c_model = ex.complex()
c_cp = cpl.ConnectionPattern(c_layer, c_conn)
showTextTable(c_cp, 'complex_tt')
# $ \centerline{\includegraphics{complex_tt.pdf}}

# ! Pattern in full detail
# ! ----------------------
c_cp.plot()
pylab.show()

# ! Note the following differences to the simple pattern case:
# !
# ! - For each pair of populations, e.g., B/E as sender and B/E as target,
# !   we now have two patches representing AMPA and NMDA synapse for the E
# !   population, GABA_A and _B for the I population.
# ! - Colors are as follows:
# !
# !   :AMPA: red
# !   :NMDA: orange
# !   :GABA_A: blue
# !   :GABA_B: purple
# ! - Note that the horizontal rectangular pattern (table line 3) describes
# !   AMPA synapses, while the vertical rectangular pattern (table line 4)
# !   describes NMDA synapses.

# ! Full detail, common color scale
# ! -------------------------------
c_cp.plot(globalColors=True)
pylab.show()

# ! As above, but now with a common color scale.
# ! **NB:** The patch for the B/I to B/I connection may look empty, but it
# ! actually shows a very light shade of red. Rules are as follows:
# !
# ! - If there is no connection between two populations, show the grey layer
# !   background.
# ! - All parts of the target layer that are outside the mask or strictly zero
# !   are off-white.
# ! - If it looks bright white, it is a very diluted shade of the color for the
# !   pertaining synpase type.

# ! Full detail, explicit color limits
# ! ----------------------------------
c_cp.plot(colorLimits=[0, 1])
pylab.show()

# ! As above, but the common color scale is now given explicitly.
# ! The arrow at the right end of the color scale indicates that the values
# ! in the kernels extend beyond +1.

# ! Aggregate by synapse models
# ! -----------------------------

# ! For each population pair, connections are summed across
# ! synapse models.
# !
# !  - Excitatory kernels are weighted with +1, inhibitory kernels with -1.
# !  - The resulting kernels are shown on a color scale ranging from red
# !    (inhibitory) via white (zero) to blue (excitatory).
# !  - Each patch has its own color scale
c_cp.plot(aggrSyns=True)
pylab.show()
# !
# ! - AMPA and NMDA connections from B/E to B/E are now combined to form a
# !   cross.
# ! - GABA_A and GABA_B connections from B/I to B/E are two concentric spots.

# ! Aggregate by population group
# ! ------------------------------
c_cp.plot(aggrGroups=True)
pylab.show()
# ! This is in many ways orthogonal to aggregation by synapse model:
# ! We keep synapse types separat, while we combine across populations. Thus,
# ! we have added the horizonal bar (B/E to B/E, row 3) with the spot
# ! (B/E to B/I, row 5).

# ! Aggregate by population group and synapse model
# ! -----------------------------------------------------------------
c_cp.plot(aggrGroups=True, aggrSyns=True)
pylab.show()
# ! All connection are combined for each pair of sender/target layer.

# ! CPTs using the total charge deposited (TCD) as intensity
# ! -----------------------------------------------------------
# ! TCD-based CPTs are currently only available for the ht_neuron, since
# ! ConnPlotter does not know how to obtain \int g(t) dt from NEST for other
# ! conductance-based model neurons.
# ! We need to create a separate ConnectionPattern instance for each membrane
# ! potential we want to use in the TCD computation
c_cp_75 = cpl.ConnectionPattern(c_layer, c_conn, intensity='tcd',
                                mList=c_model, Vmem=-75.0)
c_cp_45 = cpl.ConnectionPattern(c_layer, c_conn, intensity='tcd',
                                mList=c_model, Vmem=-45.0)

# ! In order to obtain a meaningful comparison between both membrane
# ! potentials, we use the same global color scale.

# ! V_m = -75 mV
# ! ::::::::::::::
c_cp_75.plot(colorLimits=[0, 150])
pylab.show()

# ! V_m = -45 mV
# ! ::::::::::::::
c_cp_45.plot(colorLimits=[0, 150])
pylab.show()
# ! Note that the NMDA projection virtually vanishes for V_m=-75mV, but is very
# ! strong for V_m=-45mV. GABA_A and GABA_B projections are also stronger,
# ! while AMPA is weaker for V_m=-45mV.

# ! Non-Dale network model
# ! ======================

# ! By default, ConnPlotter assumes that networks follow Dale's law, i.e.,
# ! either make excitatory or inhibitory connections. If this assumption
# ! is violated, we need to inform ConnPlotter how synapse types are grouped.
# ! We look at a simple example here.

# ! Load model
nd_layer, nd_conn, nd_model = ex.non_dale()

# ! We specify the synapse configuration using the synTypes argument:
# !
# ! - synTypes is a tuple.
# ! - Each element in the tuple represents a group of synapse models
# ! - Any sender can make connections with synapses from **one group only**.
# ! - Each synapse model is specified by a ``SynType``.
# ! - The SynType constructor takes three arguments:
# !
# !   * The synapse model name
# !   * The weight to apply then aggregating across synapse models
# !   * The color to use for the synapse type
# !
# ! - Synapse names must be unique, and must form a superset of all synapse
# !   models in the network.
nd_cp = cpl.ConnectionPattern(nd_layer, nd_conn, synTypes=(
    (cpl.SynType('exc', 1.0, 'b'), cpl.SynType('inh', -1.0, 'r')),))
showTextTable(nd_cp, 'non_dale_tt')
# $ \centerline{\includegraphics{non_dale_tt.pdf}}

nd_cp.plot()
pylab.show()

# ! Note that we now have red and blue patches side by side, as the same
# ! population can make excitatory and inhibitory connections.

# ! Configuring the ConnectionPattern display
# ! =========================================

# ! I will now show you a few ways in which you can configure how ConnPlotter
# ! shows connection patterns.

# ! User defined synapse types
# ! --------------------------
# !
# ! By default, ConnPlotter knows two following sets of synapse types.
# !
# ! exc/inh
# !   - Used automatically when all connections have the same synapse_model.
# !   - Connections with positive weight are assigned model exc, those with
# !     negative weight model inh.
# !   - When computing totals, exc has weight +1, inh weight -1
# !   - Exc is colored blue, inh red.
# !
# ! AMPA/NMDA/GABA_A/GABA_B
# !   - Used if the set of ``synapse_model`` s in the network is a subset of
# !     those four types.
# !   - AMPA/NMDA carry weight +1, GABA_A/GABA_B weight -1.
# !   - Colors are as follows:
# !
# !      :AMPA: blue
# !      :NMDA: green
# !      :GABA_A: red
# !      :GABA_B: magenta
# !
# !
# ! We saw a first example of user-defined synapse types in the non-Dale
# ! example above. In that case, we only changed the grouping. Here, I will
# ! demonstrate the effect of different ordering, weighting, and color
# ! specifications. We use the complex model from above as example.
# !
# ! *NOTE*: It is most likey a *bad idea* to change the colors or placement of
# ! synapse types. If everyone uses the same design rules, we will all be able
# ! to read each others figures much more easily.

# ! Placement of synapse types
# ! ::::::::::::::::::::::::::
# !
# ! The ``synTypes`` nested tuple defines the placement of patches for
# ! different synapse models. Default layout is
# !
# ! ====== ======
# ! AMPA   NMDA
# ! GABA_A GABA_B
# ! ====== ======
# !
# ! All four matrix elements are shown in this layout only when using
# ! ``mode='layer'`` display. Otherwise, one or the other row is shown.
# ! Note that synapses that can arise from a layer simultaneously, must
# ! always be placed on one matrix row, i.e., in one group. As an example,
# ! we now invert placement, without any other changes:

cinv_syns = ((cpl.SynType('GABA_B', -1, 'm'), cpl.SynType('GABA_A', -1, 'r')),
             (cpl.SynType('NMDA', 1, 'g'), cpl.SynType('AMPA', 1, 'b')))
cinv_cp = cpl.ConnectionPattern(c_layer, c_conn, synTypes=cinv_syns)
cinv_cp.plot()
pylab.show()

# ! Notice that on each row the synapses are exchanged compared to the original
# ! figure above. When displaying by layer, also the rows have traded place:
cinv_cp.plot(aggrGroups=True)
pylab.show()

# ! Totals are not affected:
cinv_cp.plot(aggrGroups=True, aggrSyns=True)
pylab.show()

# ! Weighting of synapse types in ``totals`` mode
# ! :::::::::::::::::::::::::::::::::::::::::::::
# !
# ! Different synapses may have quite different efficacies, so weighting them
# ! all with +-1 when computing totals may give a wrong impression. Different
# ! weights can be supplied as second argument to SynTypes(). We return to the
# ! normal placement of synapses and
# ! create two examples with very different weights:
cw1_syns = ((cpl.SynType('AMPA', 10, 'b'), cpl.SynType('NMDA', 1, 'g')),
            (cpl.SynType('GABA_A', -2, 'g'), cpl.SynType('GABA_B', -10, 'b')))
cw1_cp = cpl.ConnectionPattern(c_layer, c_conn, synTypes=cw1_syns)
cw2_syns = ((cpl.SynType('AMPA', 1, 'b'), cpl.SynType('NMDA', 10, 'g')),
            (cpl.SynType('GABA_A', -20, 'g'), cpl.SynType('GABA_B', -1, 'b')))
cw2_cp = cpl.ConnectionPattern(c_layer, c_conn, synTypes=cw2_syns)

# ! We first plot them both in population mode
cw1_cp.plot(aggrSyns=True)
pylab.show()

cw2_cp.plot(aggrSyns=True)
pylab.show()

# ! Finally, we plot them aggregating across groups and synapse models
cw1_cp.plot(aggrGroups=True, aggrSyns=True)
pylab.show()

cw2_cp.plot(aggrGroups=True, aggrSyns=True)
pylab.show()

# ! Alternative colors for synapse patches
# ! ::::::::::::::::::::::::::::::::::::::
# ! Different colors can be specified using any legal color specification.
# ! Colors should be saturated, as they will be mixed with white. You may
# ! also provide a colormap explicitly. For this example, we use once more
# ! normal placement and weights. As all synapse types are shown in layer
# ! mode, we use that mode for display here.
cc_syns = (
    (cpl.SynType('AMPA', 1, 'maroon'), cpl.SynType('NMDA', 1, (0.9, 0.5, 0))),
    (cpl.SynType('GABA_A', -1, '0.7'), cpl.SynType('GABA_B', 1, pylab.cm.hsv)))
cc_cp = cpl.ConnectionPattern(c_layer, c_conn, synTypes=cc_syns)
cc_cp.plot(aggrGroups=True)
pylab.show()
# ! We get the following colors:
# !
# ! AMPA     brownish
# ! NMDA     golden orange
# ! GABA_A   jet colormap from red (max) to blue (0)
# ! GABA_B   grey
# !
# ! **NB:** When passing an explicit colormap, parts outside the mask will be
# ! shown to the "bad" color of the colormap, usually the "bottom" color in the
# ! map. To let points outside the mask appear in white, set the bad color of
# ! the colormap; unfortunately, this modifies the colormap.
pylab.cm.hsv.set_bad(cpl.colormaps.bad_color)
ccb_syns = (
    (cpl.SynType('AMPA', 1, 'maroon'),
     cpl.SynType('NMDA', 1, (0.9, 0.5, 0.1))),
    (cpl.SynType('GABA_A', -1, '0.7'),
     cpl.SynType('GABA_B', 1, pylab.cm.hsv)))
ccb_cp = cpl.ConnectionPattern(c_layer, c_conn, synTypes=ccb_syns)
ccb_cp.plot(aggrGroups=True)
pylab.show()

# ! Other configuration options
# ! ---------------------------
# !
# ! Some more adjustments are possible by setting certain module properties.
# ! Some of these need to be set before ConnectionPattern() is constructed.
# !

# ! Background color for masked parts of each patch
cpl.colormaps.bad_color = 'cyan'

# ! Background for layers
cpl.plotParams.layer_bg = (0.8, 0.8, 0.0)

# ! Resolution for patch computation
cpl.plotParams.n_kern = 5

# ! Physical size of patches: longest egde of largest patch, in mm
cpl.plotParams.patch_size = 40

# ! Margins around the figure (excluding labels)
cpl.plotParams.margins.left = 40
cpl.plotParams.margins.top = 30
cpl.plotParams.margins.bottom = 15
cpl.plotParams.margins.right = 30

# ! Fonts for layer and population labels
import matplotlib.font_manager as fmgr

cpl.plotParams.layer_font = fmgr.FontProperties(family='serif', weight='bold',
                                                size='xx-large')
cpl.plotParams.pop_font = fmgr.FontProperties('small')

# ! Orientation for layer and population label
cpl.plotParams.layer_orientation = {'sender': 'vertical', 'target': 60}
cpl.plotParams.pop_orientation = {'sender': 'horizontal', 'target': -45}

# ! Font for legend titles and ticks, tick placement, and tick format
cpl.plotParams.legend_title_font = fmgr.FontProperties(family='serif',
                                                       weight='bold',
                                                       size='large')
cpl.plotParams.legend_tick_font = fmgr.FontProperties(family='sans-serif',
                                                      weight='light',
                                                      size='xx-small')
cpl.plotParams.legend_ticks = [0, 1, 2]
cpl.plotParams.legend_tick_format = '%.1f pA'

cx_cp = cpl.ConnectionPattern(c_layer, c_conn)
cx_cp.plot(colorLimits=[0, 2])
pylab.show()

# ! Several more options are available to control the format of the color bars
# ! (they all are members of plotParams):
# !  * legend_location : if 'top', place synapse name atop color bar
# !  * cbwidth : width of single color bar relative to figure
# !  * margins.colbar : height of lower margin set aside for color bar, in mm
# !  * cbheight : height of single color bar relative to margins.colbar
# !  * cbwidth : width of single color bar relative to figure width
# !  * cbspace : spacing between color bars, relative to figure width
# !  * cboffset : offset of first color bar from left margin, relative to
# !    figure width

# ! You can also specify the width of the final figure, but this may not work
# ! well with on-screen display or here in pyreport. Width is in mm.
# ! Note that left and right margin combined are 70mm wide, so only 50mm are
# ! left for the actual CPT.
cx_cp.plot(fixedWidth=120)
pylab.show()

# ! If not using pyreport, we finally show and block
if not using_pyreport:
    print("")
    print("The connplotter_tutorial script is done. " +
          "Call pylab.show() and enjoy the figures!")
    print(
        "You may need to close all figures manually " +
        "to get the Python prompt back.")
    print("")
    pylab.show = pylab_show


# The next script will format a phenotype table (junctions, events, trasncripts...)
# for runnning FastQTL analysis

#This version is for formatting the SCLC phenotype

"""
@authors: Juan L. Trincado
@email: juanluis.trincado@upf.edu

generate_boxplot_event.py: Generates a boxplot with the PSI values, given which samples are in which conditions
"""

import sys
import logging
import matplotlib.pyplot as plt
import numpy as np
import re

from argparse import ArgumentParser, RawTextHelpFormatter

description = \
    "Description:\n\n" + \
    "This script accept a phenotype table (junctions, events, transcripts...)\n" + \
    "and a genotype table (mutations associated to K-mers or SMRs) and returns a formatted table\n" + \
    "for using with FastQTL"

parser = ArgumentParser(description=description, formatter_class=RawTextHelpFormatter,
                        add_help=True)
parser.add_argument("-i", "--input", required=True,
                    help="Input file")
parser.add_argument("-e", "--event", required=True, type=str,
                    help="Event to plot")
parser.add_argument('-g', '--groups',
                    action="store",
                    required=True,
                    type=str,
                    nargs="*",
                    help="Ranges of column numbers specifying the replicates per condition. "
                         "Column numbers have to be continuous, with no overlapping or missing columns between them. "
                         "Ex: 1-3,4-6")
parser.add_argument('-c', '--conds',
                    action="store",
                    required=False,
                    default="0",
                    type=str,
                    nargs="*",
                    help="Name of each one of the conditions. Ex: Mutated,Non_mutated")
parser.add_argument("-o", "--output", required=True,
                    help="Output path")

# create logger
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)

# create console handler and set level to info
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)

# create formatter
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')

# add formatter to ch
ch.setFormatter(formatter)

# add ch to logger
logger.addHandler(ch)

def main():

    args = parser.parse_args()

    input_file = args.input
    event = args.event
    groups = re.findall(r"[\w]+", args.groups[0])
    output_path = args.output

    # input_file = "/home/juanluis/Desktop/Work/Master_class/events.psi"
    # event = "ENSG00000149554;SE:chr11:125496728-125497502:125497725-125499127:+"
    # groups = ['1','3','4','6']
    # output_path = "/home/juanluis/Desktop/Work/Master_class/"

    try:

        logger.info("Reading input file...")

        dict_PSI = {}
        cond = 1
        success = False
        file = open(input_file)
        for line in file:
            tokens = line.rstrip().split("\t")
            if (tokens[0]==event):
                success = True
                for i,x in enumerate(groups):
                    if(i%2==1):
                        continue
                    PSI = []
                    samples = range(int(groups[i]),int(groups[i+1])+1)
                    #Get the PSI of this group of samples
                    for j in samples:
                        PSI.append(tokens[j])
                    dict_PSI[cond] = PSI
                    cond = cond + 1
                break

        if(success):
            #Create the boxplot
            data_to_plot = []
            for key in dict_PSI.keys():
                data_to_plot.append(list(map(float,dict_PSI[key])))
            # Create a figure instance
            fig = plt.figure(figsize=(9, 6))
            # Create an axes instance
            ax = fig.add_subplot(111)
            # Create the boxplot
            bp = ax.boxplot(data_to_plot, patch_artist=True, sym='')
            # change the style of fliers and their fill
            for flier in bp['fliers']:
                flier.set(marker='.', color='#000000', alpha=0.7)
            # Assign different colors
            colors = ['lightblue', 'pink']
            for patch, color in zip(bp['boxes'], colors):
                patch.set_facecolor(color)
            for j in range(len(data_to_plot)):
                y = data_to_plot[j]
                x = np.random.normal(1 + j, 0.02, size=len(y))
                plt.plot(x, y, 'ko', alpha=0.5)
            # Custom x-axis labels if the user has input conditions
            if (args.conds != "0"):
                conditions = re.findall(r"[\w]+", args.conds[0])
                ax.set_xticklabels(conditions)
            # Leave just ticks in the bottom
            ax.get_xaxis().tick_bottom()
            ax.set_ylabel('PSI')
            # Set the title
            title = "Event: " + event
            ax.set_title(title, fontsize=10)
            # Add a horizontal grid to the plot,
            ax.yaxis.grid(True, linestyle='-', which='major', color='lightgrey', alpha=0.5)
            # Set the limits for the y axes
            ax.set_ylim([-0.05, 1.05])
            # Save the figure
            output_path = output_path + "/" + event + ".png"
            logger.info("Created " + output_path)
            fig.savefig(output_path, bbox_inches='tight')
        else:
            logger.info("Event not found.")

        logger.info("Done.")

        exit(0)

    except Exception as error:
        logger.error(repr(error))
        logger.error("Aborting execution")
        sys.exit(1)

if __name__ == '__main__':
    main()


# Authors: Gael Varoquaux <gael.varoquaux@normalesup.org>
#          Justin Vincent
#          Lars Buitinck
# License: BSD 3 clause

import numpy as np

from nose.tools import assert_equal
from nose.tools import assert_false
from nose.tools import assert_true
from numpy.testing import (assert_almost_equal,
                           assert_array_almost_equal)

from sklearn.utils.fixes import divide, expit
from sklearn.utils.fixes import astype

def test_expit():
    # Check numerical stability of expit (logistic function).

    # Simulate our previous Cython implementation, based on
    #http://fa.bianp.net/blog/2013/numerical-optimizers-for-logistic-regression
    assert_almost_equal(expit(1000.), 1. / (1. + np.exp(-1000.)), decimal=16)
    assert_almost_equal(expit(-1000.), np.exp(-1000.) / (1. + np.exp(-1000.)),
                        decimal=16)

    x = np.arange(10)
    out = np.zeros_like(x, dtype=np.float32)
    assert_array_almost_equal(expit(x), expit(x, out=out))

def test_divide():
    assert_equal(divide(.6, 1), .600000000000)

def test_astype_copy_memory():
    a_int32 = np.ones(3, np.int32)

    # Check that dtype conversion works
    b_float32 = astype(a_int32, dtype=np.float32, copy=False)
    assert_equal(b_float32.dtype, np.float32)

    # Changing dtype forces a copy even if copy=False
    assert_false(np.may_share_memory(b_float32, a_int32))

    # Check that copy can be skipped if requested dtype match
    c_int32 = astype(a_int32, dtype=np.int32, copy=False)
    assert_true(c_int32 is a_int32)

    # Check that copy can be forced, and is the case by default:
    d_int32 = astype(a_int32, dtype=np.int32, copy=True)
    assert_false(np.may_share_memory(d_int32, a_int32))

    e_int32 = astype(a_int32, dtype=np.int32)
    assert_false(np.may_share_memory(e_int32, a_int32))


# Author: Lars Buitinck <L.J.Buitinck@uva.nl>
# License: BSD 3 clause

import numbers

import numpy as np
import scipy.sparse as sp

from . import _hashing
from ..base import BaseEstimator, TransformerMixin

def _iteritems(d):
    """Like d.iteritems, but accepts any collections.Mapping."""
    return d.iteritems() if hasattr(d, "iteritems") else d.items()

class FeatureHasher(BaseEstimator, TransformerMixin):
    """Implements feature hashing, aka the hashing trick.

    This class turns sequences of symbolic feature names (strings) into
    scipy.sparse matrices, using a hash function to compute the matrix column
    corresponding to a name. The hash function employed is the signed 32-bit
    version of Murmurhash3.

    Feature names of type byte string are used as-is. Unicode strings are
    converted to UTF-8 first, but no Unicode normalization is done.
    Feature values must be (finite) numbers.

    This class is a low-memory alternative to DictVectorizer and
    CountVectorizer, intended for large-scale (online) learning and situations
    where memory is tight, e.g. when running prediction code on embedded
    devices.

    Read more in the :ref:`User Guide <feature_hashing>`.

    Parameters
    ----------
    n_features : integer, optional
        The number of features (columns) in the output matrices. Small numbers
        of features are likely to cause hash collisions, but large numbers
        will cause larger coefficient dimensions in linear learners.
    dtype : numpy type, optional
        The type of feature values. Passed to scipy.sparse matrix constructors
        as the dtype argument. Do not set this to bool, np.boolean or any
        unsigned integer type.
    input_type : string, optional
        Either "dict" (the default) to accept dictionaries over
        (feature_name, value); "pair" to accept pairs of (feature_name, value);
        or "string" to accept single strings.
        feature_name should be a string, while value should be a number.
        In the case of "string", a value of 1 is implied.
        The feature_name is hashed to find the appropriate column for the
        feature. The value's sign might be flipped in the output (but see
        non_negative, below).
    non_negative : boolean, optional, default np.float64
        Whether output matrices should contain non-negative values only;
        effectively calls abs on the matrix prior to returning it.
        When True, output values can be interpreted as frequencies.
        When False, output values will have expected value zero.

    Examples
    --------
    >>> from sklearn.feature_extraction import FeatureHasher
    >>> h = FeatureHasher(n_features=10)
    >>> D = [{'dog': 1, 'cat':2, 'elephant':4},{'dog': 2, 'run': 5}]
    >>> f = h.transform(D)
    >>> f.toarray()
    array([[ 0.,  0., -4., -1.,  0.,  0.,  0.,  0.,  0.,  2.],
           [ 0.,  0.,  0., -2., -5.,  0.,  0.,  0.,  0.,  0.]])

    See also
    --------
    DictVectorizer : vectorizes string-valued features using a hash table.
    sklearn.preprocessing.OneHotEncoder : handles nominal/categorical features
      encoded as columns of integers.
    """

    def __init__(self, n_features=(2 ** 20), input_type="dict",
                 dtype=np.float64, non_negative=False):
        self._validate_params(n_features, input_type)

        self.dtype = dtype
        self.input_type = input_type
        self.n_features = n_features
        self.non_negative = non_negative

    @staticmethod
    def _validate_params(n_features, input_type):
        # strangely, np.int16 instances are not instances of Integral,
        # while np.int64 instances are...
        if not isinstance(n_features, (numbers.Integral, np.integer)):
            raise TypeError("n_features must be integral, got %r (%s)."
                            % (n_features, type(n_features)))
        elif n_features < 1 or n_features >= 2 ** 31:
            raise ValueError("Invalid number of features (%d)." % n_features)

        if input_type not in ("dict", "pair", "string"):
            raise ValueError("input_type must be 'dict', 'pair' or 'string',"
                             " got %r." % input_type)

    def fit(self, X=None, y=None):
        """No-op.

        This method doesn't do anything. It exists purely for compatibility
        with the scikit-learn transformer API.

        Returns
        -------
        self : FeatureHasher

        """
        # repeat input validation for grid search (which calls set_params)
        self._validate_params(self.n_features, self.input_type)
        return self

    def transform(self, raw_X, y=None):
        """Transform a sequence of instances to a scipy.sparse matrix.

        Parameters
        ----------
        raw_X : iterable over iterable over raw features, length = n_samples
            Samples. Each sample must be iterable an (e.g., a list or tuple)
            containing/generating feature names (and optionally values, see
            the input_type constructor argument) which will be hashed.
            raw_X need not support the len function, so it can be the result
            of a generator; n_samples is determined on the fly.
        y : (ignored)

        Returns
        -------
        X : scipy.sparse matrix, shape = (n_samples, self.n_features)
            Feature matrix, for use with estimators or further transformers.

        """
        raw_X = iter(raw_X)
        if self.input_type == "dict":
            raw_X = (_iteritems(d) for d in raw_X)
        elif self.input_type == "string":
            raw_X = (((f, 1) for f in x) for x in raw_X)
        indices, indptr, values = \
            _hashing.transform(raw_X, self.n_features, self.dtype)
        n_samples = indptr.shape[0] - 1

        if n_samples == 0:
            raise ValueError("Cannot vectorize empty sequence.")

        X = sp.csr_matrix((values, indices, indptr), dtype=self.dtype,
                          shape=(n_samples, self.n_features))
        X.sum_duplicates()  # also sorts the indices
        if self.non_negative:
            np.abs(X.data, X.data)
        return X


"""
=========================
Bayesian Ridge Regression
=========================

Computes a Bayesian Ridge Regression on a synthetic dataset.

See :ref:`bayesian_ridge_regression` for more information on the regressor.

Compared to the OLS (ordinary least squares) estimator, the coefficient
weights are slightly shifted toward zeros, which stabilises them.

As the prior on the weights is a Gaussian prior, the histogram of the
estimated weights is Gaussian.

The estimation of the model is done by iteratively maximizing the
marginal log-likelihood of the observations.

We also plot predictions and uncertainties for Bayesian Ridge Regression
for one dimensional regression using polynomial feature expansion.
Note the uncertainty starts going up on the right side of the plot.
This is because these test samples are outside of the range of the training
samples.
"""
print(__doc__)

import numpy as np
import matplotlib.pyplot as plt
from scipy import stats

from sklearn.linear_model import BayesianRidge, LinearRegression

# #############################################################################
# Generating simulated data with Gaussian weights
np.random.seed(0)
n_samples, n_features = 100, 100
X = np.random.randn(n_samples, n_features)  # Create Gaussian data
# Create weights with a precision lambda_ of 4.
lambda_ = 4.
w = np.zeros(n_features)
# Only keep 10 weights of interest
relevant_features = np.random.randint(0, n_features, 10)
for i in relevant_features:
    w[i] = stats.norm.rvs(loc=0, scale=1. / np.sqrt(lambda_))
# Create noise with a precision alpha of 50.
alpha_ = 50.
noise = stats.norm.rvs(loc=0, scale=1. / np.sqrt(alpha_), size=n_samples)
# Create the target
y = np.dot(X, w) + noise

# #############################################################################
# Fit the Bayesian Ridge Regression and an OLS for comparison
clf = BayesianRidge(compute_score=True)
clf.fit(X, y)

ols = LinearRegression()
ols.fit(X, y)

# #############################################################################
# Plot true weights, estimated weights, histogram of the weights, and
# predictions with standard deviations
lw = 2
plt.figure(figsize=(6, 5))
plt.title("Weights of the model")
plt.plot(clf.coef_, color='lightgreen', linewidth=lw,
         label="Bayesian Ridge estimate")
plt.plot(w, color='gold', linewidth=lw, label="Ground truth")
plt.plot(ols.coef_, color='navy', linestyle='--', label="OLS estimate")
plt.xlabel("Features")
plt.ylabel("Values of the weights")
plt.legend(loc="best", prop=dict(size=12))

plt.figure(figsize=(6, 5))
plt.title("Histogram of the weights")
plt.hist(clf.coef_, bins=n_features, color='gold', log=True,
         edgecolor='black')
plt.scatter(clf.coef_[relevant_features], 5 * np.ones(len(relevant_features)),
            color='navy', label="Relevant features")
plt.ylabel("Features")
plt.xlabel("Values of the weights")
plt.legend(loc="upper left")

plt.figure(figsize=(6, 5))
plt.title("Marginal log-likelihood")
plt.plot(clf.scores_, color='navy', linewidth=lw)
plt.ylabel("Score")
plt.xlabel("Iterations")

# Plotting some predictions for polynomial regression
def f(x, noise_amount):
    y = np.sqrt(x) * np.sin(x)
    noise = np.random.normal(0, 1, len(x))
    return y + noise_amount * noise

degree = 10
X = np.linspace(0, 10, 100)
y = f(X, noise_amount=0.1)
clf_poly = BayesianRidge()
clf_poly.fit(np.vander(X, degree), y)

X_plot = np.linspace(0, 11, 25)
y_plot = f(X_plot, noise_amount=0)
y_mean, y_std = clf_poly.predict(np.vander(X_plot, degree), return_std=True)
plt.figure(figsize=(6, 5))
plt.errorbar(X_plot, y_mean, y_std, color='navy',
             label="Polynomial Bayesian Ridge Regression", linewidth=lw)
plt.plot(X_plot, y_plot, color='gold', linewidth=lw,
         label="Ground Truth")
plt.ylabel("Output y")
plt.xlabel("Feature X")
plt.legend(loc="lower left")
plt.show()


#!/usr/bin/env python3

#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import logging
from argparse import ArgumentParser
import os
import re
import shutil
import subprocess
import sys
import tempfile
from threading import Thread, Lock
import time
import uuid
import queue as Queue
from multiprocessing import Manager

# Append `SPARK_HOME/dev` to the Python path so that we can import the sparktestsupport module
sys.path.append(os.path.join(os.path.dirname(os.path.realpath(__file__)), "../dev/"))

from sparktestsupport import SPARK_HOME  # noqa (suppress pep8 warnings)
from sparktestsupport.shellutils import which, subprocess_check_output  # noqa
from sparktestsupport.modules import all_modules, pyspark_sql  # noqa

python_modules = dict((m.name, m) for m in all_modules if m.python_test_goals if m.name != 'root')

def print_red(text):
    print('\033[31m' + text + '\033[0m')

SKIPPED_TESTS = None
LOG_FILE = os.path.join(SPARK_HOME, "python/unit-tests.log")
FAILURE_REPORTING_LOCK = Lock()
LOGGER = logging.getLogger()

# Find out where the assembly jars are located.
# TODO: revisit for Scala 2.13
for scala in ["2.12"]:
    build_dir = os.path.join(SPARK_HOME, "assembly", "target", "scala-" + scala)
    if os.path.isdir(build_dir):
        SPARK_DIST_CLASSPATH = os.path.join(build_dir, "jars", "*")
        break
else:
    raise RuntimeError("Cannot find assembly build directory, please build Spark first.")

def run_individual_python_test(target_dir, test_name, pyspark_python):
    env = dict(os.environ)
    env.update({
        'SPARK_DIST_CLASSPATH': SPARK_DIST_CLASSPATH,
        'SPARK_TESTING': '1',
        'SPARK_PREPEND_CLASSES': '1',
        'PYSPARK_PYTHON': which(pyspark_python),
        'PYSPARK_DRIVER_PYTHON': which(pyspark_python),
        # Preserve legacy nested timezone behavior for pyarrow>=2, remove after SPARK-32285
        'PYARROW_IGNORE_TIMEZONE': '1',
    })

    # Create a unique temp directory under 'target/' for each run. The TMPDIR variable is
    # recognized by the tempfile module to override the default system temp directory.
    tmp_dir = os.path.join(target_dir, str(uuid.uuid4()))
    while os.path.isdir(tmp_dir):
        tmp_dir = os.path.join(target_dir, str(uuid.uuid4()))
    os.mkdir(tmp_dir)
    env["TMPDIR"] = tmp_dir
    metastore_dir = os.path.join(tmp_dir, str(uuid.uuid4()))
    while os.path.isdir(metastore_dir):
        metastore_dir = os.path.join(metastore_dir, str(uuid.uuid4()))
    os.mkdir(metastore_dir)

    # Also override the JVM's temp directory by setting driver and executor options.
    java_options = "-Djava.io.tmpdir={0} -Dio.netty.tryReflectionSetAccessible=true".format(tmp_dir)
    spark_args = [
        "--conf", "spark.driver.extraJavaOptions='{0}'".format(java_options),
        "--conf", "spark.executor.extraJavaOptions='{0}'".format(java_options),
        "--conf", "spark.sql.warehouse.dir='{0}'".format(metastore_dir),
        "pyspark-shell"
    ]
    env["PYSPARK_SUBMIT_ARGS"] = " ".join(spark_args)

    LOGGER.info("Starting test(%s): %s", pyspark_python, test_name)
    start_time = time.time()
    try:
        per_test_output = tempfile.TemporaryFile()
        retcode = subprocess.Popen(
            [os.path.join(SPARK_HOME, "bin/pyspark")] + test_name.split(),
            stderr=per_test_output, stdout=per_test_output, env=env).wait()
        shutil.rmtree(tmp_dir, ignore_errors=True)
    except:
        LOGGER.exception("Got exception while running %s with %s", test_name, pyspark_python)
        # Here, we use os._exit() instead of sys.exit() in order to force Python to exit even if
        # this code is invoked from a thread other than the main thread.
        os._exit(1)
    duration = time.time() - start_time
    # Exit on the first failure.
    if retcode != 0:
        try:
            with FAILURE_REPORTING_LOCK:
                with open(LOG_FILE, 'ab') as log_file:
                    per_test_output.seek(0)
                    log_file.writelines(per_test_output)
                per_test_output.seek(0)
                for line in per_test_output:
                    decoded_line = line.decode("utf-8", "replace")
                    if not re.match('[0-9]+', decoded_line):
                        print(decoded_line, end='')
                per_test_output.close()
        except:
            LOGGER.exception("Got an exception while trying to print failed test output")
        finally:
            print_red("\nHad test failures in %s with %s; see logs." % (test_name, pyspark_python))
            # Here, we use os._exit() instead of sys.exit() in order to force Python to exit even if
            # this code is invoked from a thread other than the main thread.
            os._exit(-1)
    else:
        skipped_counts = 0
        try:
            per_test_output.seek(0)
            # Here expects skipped test output from unittest when verbosity level is
            # 2 (or --verbose option is enabled).
            decoded_lines = map(lambda line: line.decode("utf-8", "replace"), iter(per_test_output))
            skipped_tests = list(filter(
                lambda line: re.search(r'test_.* \(pyspark\..*\) ... (skip|SKIP)', line),
                decoded_lines))
            skipped_counts = len(skipped_tests)
            if skipped_counts > 0:
                key = (pyspark_python, test_name)
                assert SKIPPED_TESTS is not None
                SKIPPED_TESTS[key] = skipped_tests
            per_test_output.close()
        except:
            import traceback
            print_red("\nGot an exception while trying to store "
                      "skipped test output:\n%s" % traceback.format_exc())
            # Here, we use os._exit() instead of sys.exit() in order to force Python to exit even if
            # this code is invoked from a thread other than the main thread.
            os._exit(-1)
        if skipped_counts != 0:
            LOGGER.info(
                "Finished test(%s): %s (%is) ... %s tests were skipped", pyspark_python, test_name,
                duration, skipped_counts)
        else:
            LOGGER.info(
                "Finished test(%s): %s (%is)", pyspark_python, test_name, duration)

def get_default_python_executables():
    python_execs = [x for x in ["python3.6", "pypy3"] if which(x)]

    if "python3.6" not in python_execs:
        p = which("python3")
        if not p:
            LOGGER.error("No python3 executable found.  Exiting!")
            os._exit(1)
        else:
            python_execs.insert(0, p)
    return python_execs

def parse_opts():
    parser = ArgumentParser(
        prog="run-tests"
    )
    parser.add_argument(
        "--python-executables", type=str, default=','.join(get_default_python_executables()),
        help="A comma-separated list of Python executables to test against (default: %(default)s)"
    )
    parser.add_argument(
        "--modules", type=str,
        default=",".join(sorted(python_modules.keys())),
        help="A comma-separated list of Python modules to test (default: %(default)s)"
    )
    parser.add_argument(
        "-p", "--parallelism", type=int, default=4,
        help="The number of suites to test in parallel (default %(default)d)"
    )
    parser.add_argument(
        "--verbose", action="store_true",
        help="Enable additional debug logging"
    )

    group = parser.add_argument_group("Developer Options")
    group.add_argument(
        "--testnames", type=str,
        default=None,
        help=(
            "A comma-separated list of specific modules, classes and functions of doctest "
            "or unittest to test. "
            "For example, 'pyspark.sql.foo' to run the module as unittests or doctests, "
            "'pyspark.sql.tests FooTests' to run the specific class of unittests, "
            "'pyspark.sql.tests FooTests.test_foo' to run the specific unittest in the class. "
            "'--modules' option is ignored if they are given.")
    )

    args, unknown = parser.parse_known_args()
    if unknown:
        parser.error("Unsupported arguments: %s" % ' '.join(unknown))
    if args.parallelism < 1:
        parser.error("Parallelism cannot be less than 1")
    return args

def _check_coverage(python_exec):
    # Make sure if coverage is installed.
    try:
        subprocess_check_output(
            [python_exec, "-c", "import coverage"],
            stderr=open(os.devnull, 'w'))
    except:
        print_red("Coverage is not installed in Python executable '%s' "
                  "but 'COVERAGE_PROCESS_START' environment variable is set, "
                  "exiting." % python_exec)
        sys.exit(-1)

def main():
    opts = parse_opts()
    if opts.verbose:
        log_level = logging.DEBUG
    else:
        log_level = logging.INFO
    should_test_modules = opts.testnames is None
    logging.basicConfig(stream=sys.stdout, level=log_level, format="%(message)s")
    LOGGER.info("Running PySpark tests. Output is in %s", LOG_FILE)
    if os.path.exists(LOG_FILE):
        os.remove(LOG_FILE)
    python_execs = opts.python_executables.split(',')
    LOGGER.info("Will test against the following Python executables: %s", python_execs)

    if should_test_modules:
        modules_to_test = []
        for module_name in opts.modules.split(','):
            if module_name in python_modules:
                modules_to_test.append(python_modules[module_name])
            else:
                print("Error: unrecognized module '%s'. Supported modules: %s" %
                      (module_name, ", ".join(python_modules)))
                sys.exit(-1)
        LOGGER.info("Will test the following Python modules: %s", [x.name for x in modules_to_test])
    else:
        testnames_to_test = opts.testnames.split(',')
        LOGGER.info("Will test the following Python tests: %s", testnames_to_test)

    task_queue = Queue.PriorityQueue()
    for python_exec in python_execs:
        # Check if the python executable has coverage installed when 'COVERAGE_PROCESS_START'
        # environmental variable is set.
        if "COVERAGE_PROCESS_START" in os.environ:
            _check_coverage(python_exec)

        python_implementation = subprocess_check_output(
            [python_exec, "-c", "import platform; print(platform.python_implementation())"],
            universal_newlines=True).strip()
        LOGGER.info("%s python_implementation is %s", python_exec, python_implementation)
        LOGGER.info("%s version is: %s", python_exec, subprocess_check_output(
            [python_exec, "--version"], stderr=subprocess.STDOUT, universal_newlines=True).strip())
        if should_test_modules:
            for module in modules_to_test:
                if python_implementation not in module.excluded_python_implementations:
                    for test_goal in module.python_test_goals:
                        heavy_tests = ['pyspark.streaming.tests', 'pyspark.mllib.tests',
                                       'pyspark.tests', 'pyspark.sql.tests', 'pyspark.ml.tests',
                                       'pyspark.pandas.tests']
                        if any(map(lambda prefix: test_goal.startswith(prefix), heavy_tests)):
                            priority = 0
                        else:
                            priority = 100
                        task_queue.put((priority, (python_exec, test_goal)))
        else:
            for test_goal in testnames_to_test:
                task_queue.put((0, (python_exec, test_goal)))

    # Create the target directory before starting tasks to avoid races.
    target_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'target'))
    if not os.path.isdir(target_dir):
        os.mkdir(target_dir)

    def process_queue(task_queue):
        while True:
            try:
                (priority, (python_exec, test_goal)) = task_queue.get_nowait()
            except Queue.Empty:
                break
            try:
                run_individual_python_test(target_dir, test_goal, python_exec)
            finally:
                task_queue.task_done()

    start_time = time.time()
    for _ in range(opts.parallelism):
        worker = Thread(target=process_queue, args=(task_queue,))
        worker.daemon = True
        worker.start()
    try:
        task_queue.join()
    except (KeyboardInterrupt, SystemExit):
        print_red("Exiting due to interrupt")
        sys.exit(-1)
    total_duration = time.time() - start_time
    LOGGER.info("Tests passed in %i seconds", total_duration)

    for key, lines in sorted(SKIPPED_TESTS.items()):
        pyspark_python, test_name = key
        LOGGER.info("\nSkipped tests in %s with %s:" % (test_name, pyspark_python))
        for line in lines:
            LOGGER.info("    %s" % line.rstrip())

if __name__ == "__main__":
    SKIPPED_TESTS = Manager().dict()
    main()


# Copyright 2019 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

from __future__ import print_function

import json
import os
import sys
import shutil
import subprocess
import tempfile

CHROMIUM_PATH = os.path.join(os.path.dirname(__file__), '..', '..', '..', '..')
TOOLS_PERF_PATH = os.path.join(CHROMIUM_PATH, 'tools', 'perf')
sys.path.insert(1, TOOLS_PERF_PATH)

from core.external_modules import pandas

RUNS_USED_FOR_LIMIT_UPDATE = 30
CHANGE_PERCENTAGE_LIMIT = 0.01

SWARMING_PATH = os.path.join(
  CHROMIUM_PATH, 'tools', 'swarming_client', 'swarming.py')
UPPER_LIMITS_DATA_DIR = os.path.join(
  CHROMIUM_PATH, 'testing', 'scripts', 'representative_perf_test_data')

def FetchItemIds(tags, limit):
  """Fetches the item id of tasks described by the tags.

  Args:
    tags: The tags which describe the task such as OS and buildername.
    limit: The number of runs to look at.

  Returns:
    A list containing the item Id of the tasks.
  """
  swarming_attributes = (
    'tasks/list?tags=name:rendering_representative_perf_tests&tags=os:{os}'
    '&tags=buildername:{buildername}&tags=master:chromium.gpu.fyi&state='
    'COMPLETED&fields=cursor,items(task_id)').format(**tags)

  query = [
    SWARMING_PATH, 'query', '-S', 'chromium-swarm.appspot.com', '--limit',
    str(limit), swarming_attributes]
  output = json.loads(subprocess.check_output(query))
  return output.get('items')

def FetchItemData(task_id, benchmark, index, temp_dir):
  """Fetches the performance values (AVG & CI ranges) of tasks.

  Args:
    task_id: The list of item Ids to fetch dat for.
    benchmark: The benchmark these task are on (desktop/mobile).
    index: The index field of the data_frame
    temp_dir: The temp directory to store task data in.

  Returns:
    A data_frame containing the averages and confidence interval ranges.
  """
  output_directory = os.path.abspath(
    os.path.join(temp_dir, task_id))
  query = [
    SWARMING_PATH, 'collect', '-S', 'chromium-swarm.appspot.com',
    '--task-output-dir', output_directory, task_id]
  try:
    subprocess.check_output(query)
  except Exception as e:
    print(e)

  result_file_path = os.path.join(
    output_directory, '0', 'rendering.' + benchmark, 'perf_results.csv')

  try:
    df = pandas.read_csv(result_file_path)
    df = df.loc[df['name'] == 'frame_times']
    df = df[['stories', 'avg', 'ci_095']]
    df['index'] = index
    return df
  except:
    print("CSV results were not produced!")

def GetPercentileValues(benchmark, tags, limit, percentile):
  """Get the percentile value of recent runs described by given tags.

  Given the tags, benchmark this function fetches the data of last {limit}
  runs, and find the percentile value for each story.

  Args:
    benchmark: The benchmark these task are on (desktop/mobile).
    tags: The tags which describe the tasks such as OS and buildername.
    limit: The number of runs to look at.
    percentile: the percentile to return.

  Returns:
    A dictionary with averages and confidence interval ranges calculated
    from the percentile of recent runs.
  """
  items = []
  for tag_set in tags:
    items.extend(FetchItemIds(tag_set, limit))

  dfs = []
  try:
    temp_dir = tempfile.mkdtemp('perf_csvs')
    for idx, item in enumerate(items):
      dfs.append(FetchItemData(item['task_id'], benchmark, idx, temp_dir))
      idx += 1
  finally:
    shutil.rmtree(temp_dir)
  data_frame = pandas.concat(dfs, ignore_index=True)

  if not data_frame.empty:
    avg_df = data_frame.pivot(index='stories', columns='index', values='avg')
    upper_limit = avg_df.quantile(percentile, axis = 1)
    ci_df = data_frame.pivot(index='stories', columns='index', values='ci_095')
    upper_limit_ci = ci_df.quantile(percentile, axis = 1)
    results = {}
    for index in avg_df.index:
      results[index] = {
        'avg': round(upper_limit[index], 3),
        'ci_095': round(upper_limit_ci[index], 3)
      }
    return results

def MeasureNewUpperLimit(old_value, new_value, att_name, max_change):
  # There has been an improvement.
  if new_value < old_value:
    # Decrease the limit gradually in case of improvements.
    new_value = (old_value + new_value) / 2.0

  change_pct = 0.0
  if old_value > 0:
    change_pct = (new_value - old_value) / old_value

  print(
    '  {}:\t\t {} -> {} \t({:.2f}%)'.format(
      att_name, old_value, new_value, change_pct * 100))
  if new_value < 0.01:
    print('WARNING: New selected value is close to 0.')
  return (
      round(new_value, 3),
      max(max_change, abs(change_pct))
  )

def RecalculateUpperLimits(data_point_count):
  """Recalculates the upper limits using the data of recent runs.

  This method replaces the existing JSON file which contains the upper limits
  used by representative perf tests if the changes of upper limits are
  significant.

  Args:
    data_point_count: The number of runs to use for recalculation.
  """
  with open(os.path.join(UPPER_LIMITS_DATA_DIR,
            'platform_specific_tags.json')) as tags_data:
    platform_specific_tags = json.load(tags_data)

  with open(
    os.path.join(
      UPPER_LIMITS_DATA_DIR,
      'representatives_frame_times_upper_limit.json')) as current_data:
    current_upper_limits = json.load(current_data)

  max_change = 0.0
  results = {}
  for platform in platform_specific_tags:
    platform_data = platform_specific_tags[platform]
    print('\n- Processing data ({})'.format(platform))
    results[platform] = GetPercentileValues(
      platform_data['benchmark'], platform_data['tags'],
      data_point_count, 0.95)

    # Loop over results and adjust base on current values.
    for story in results[platform]:
      if story in current_upper_limits[platform]:
        print(story, ':')
        new_avg, max_change = MeasureNewUpperLimit(
          current_upper_limits[platform][story]['avg'],
          results[platform][story]['avg'], 'AVG', max_change)
        results[platform][story]['avg'] = new_avg

        new_ci, max_change = MeasureNewUpperLimit(
          current_upper_limits[platform][story]['ci_095'],
          results[platform][story]['ci_095'], 'CI', max_change)
        results[platform][story]['ci_095'] = new_ci

  if max_change > CHANGE_PERCENTAGE_LIMIT:
    with open(
      os.path.join(
        UPPER_LIMITS_DATA_DIR,
        'representatives_frame_times_upper_limit.json'
      ), 'w') as outfile:
      json.dump(results, outfile, separators=(',', ': '), indent=2)
    print(
      'Upper limits were updated on '
      'representatives_frame_times_upper_limit.json')
  else:
    print('Changes are small, no need for new limits')

if __name__ == '__main__':
  sys.exit(RecalculateUpperLimits(RUNS_USED_FOR_LIMIT_UPDATE))


#!/usr/bin/env python
#
# Copyright 2010,2012,2013 Free Software Foundation, Inc.
#
# This file is part of GNU Radio
#
# GNU Radio is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3, or (at your option)
# any later version.
#
# GNU Radio is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with GNU Radio; see the file COPYING.  If not, write to
# the Free Software Foundation, Inc., 51 Franklin Street,
# Boston, MA 02110-1301, USA.
#

from __future__ import print_function
from __future__ import division
from __future__ import unicode_literals
from gnuradio import gr
from gnuradio import blocks
from gnuradio import filter
import sys
import numpy

try:
    from gnuradio import analog
except ImportError:
    sys.stderr.write("Error: Program requires gr-analog.\n")
    sys.exit(1)

try:
    from matplotlib import pyplot
except ImportError:
    sys.stderr.write("Error: Program requires matplotlib (see: matplotlib.sourceforge.net).\n")
    sys.exit(1)

def main():
    N = 1000000
    fs = 8000

    freqs = [100, 200, 300, 400, 500]
    nchans = 7

    sigs = list()
    fmtx = list()
    for fi in freqs:
        s = analog.sig_source_f(fs, analog.GR_SIN_WAVE, fi, 1)
        fm = analog.nbfm_tx(fs, 4*fs, max_dev=10000, tau=75e-6, fh=0.925*(4*fs)/2.0)
        sigs.append(s)
        fmtx.append(fm)

    syntaps = filter.firdes.low_pass_2(len(freqs), fs, fs/float(nchans)/2, 100, 100)
    print("Synthesis Num. Taps = %d (taps per filter = %d)" % (len(syntaps),
                                                               len(syntaps) / nchans))
    chtaps = filter.firdes.low_pass_2(len(freqs), fs, fs/float(nchans)/2, 100, 100)
    print("Channelizer Num. Taps = %d (taps per filter = %d)" % (len(chtaps),
                                                                 len(chtaps) / nchans))
    filtbank = filter.pfb_synthesizer_ccf(nchans, syntaps)
    channelizer = filter.pfb.channelizer_ccf(nchans, chtaps)

    noise_level = 0.01
    head = blocks.head(gr.sizeof_gr_complex, N)
    noise = analog.noise_source_c(analog.GR_GAUSSIAN, noise_level)
    addnoise = blocks.add_cc()
    snk_synth = blocks.vector_sink_c()

    tb = gr.top_block()

    tb.connect(noise, (addnoise,0))
    tb.connect(filtbank, head, (addnoise, 1))
    tb.connect(addnoise, channelizer)
    tb.connect(addnoise, snk_synth)

    snk = list()
    for i,si in enumerate(sigs):
        tb.connect(si, fmtx[i], (filtbank, i))

    for i in range(nchans):
        snk.append(blocks.vector_sink_c())
        tb.connect((channelizer, i), snk[i])

    tb.run()

    if 1:
        channel = 1
        data = snk[channel].data()[1000:]

        f1 = pyplot.figure(1)
        s1 = f1.add_subplot(1,1,1)
        s1.plot(data[10000:10200] )
        s1.set_title(("Output Signal from Channel %d" % channel))

        fftlen = 2048
        winfunc = numpy.blackman
        #winfunc = numpy.hamming

        f2 = pyplot.figure(2)
        s2 = f2.add_subplot(1,1,1)
        s2.psd(data, NFFT=fftlen,
               Fs = nchans*fs,
               noverlap=fftlen / 4,
               window = lambda d: d*winfunc(fftlen))
        s2.set_title(("Output PSD from Channel %d" % channel))

        f3 = pyplot.figure(3)
        s3 = f3.add_subplot(1,1,1)
        s3.psd(snk_synth.data()[1000:], NFFT=fftlen,
               Fs = nchans*fs,
               noverlap=fftlen / 4,
               window = lambda d: d*winfunc(fftlen))
        s3.set_title("Output of Synthesis Filter")

        pyplot.show()

if __name__ == "__main__":
    main()


import numpy
import matplotlib.pyplot as plt
import math
from avoplot import plugins, series, controls, subplots
from avoplot.gui import widgets
import wx

plugin_is_GPL_compatible = True

class TrigFuncSubplot(subplots.AvoPlotXYSubplot):
    def my_init(self):
        """
        When defining your own subplot classes, you should not need to override
        the __init__ method of the base class. Instead you should define a
        my_init() method which takes no args. This will be called automatically
        when the subplot is created. Use this to customise the subplot to suit
        your specific needs - settings titles, axis formatters etc.
        """

        #call the parent class's my_init() method. This is not required, unless
        #you want to make use of any customisation done by the parent class.
        #Note that this includes any control panels defined by the parent class!
        super(TrigFuncSubplot, self).my_init()

        #set up some axis titles
        ax = self.get_mpl_axes()
        ax.set_xlabel(r'$\theta$ (radians)')
        ax.set_ylabel('y')

        #add the units control panel to this subplot to allow the user to change
        #the x-axis units.
        self.add_control_panel(TrigSubplotUnitsCtrl(self))

        #set the initial name of the subplot
        self.set_name("Trig. Function Subplot")

class SineWaveSeries(series.XYDataSeries):
    """
    Define our own data series type for Sine data. Unlike for subplots, when
    defining custom data series, we do override the __init__ method.
    """
    def __init__(self, *args, **kwargs):
        super(SineWaveSeries, self).__init__(*args, **kwargs)

        #add a control for this data series to allow the user to change the
        #frequency of the wave using a slider.
        self.add_control_panel(SineWaveFreqCtrl(self))

    @staticmethod
    def get_supported_subplot_type():
        """
        This is how we restrict which data series can be plotted into which
        types of subplots. Specialised subplots may provide controls for dealing
        with very specific types of data - for example, our TrigFuncSubplot
        allows the x-axis to be switched between degrees and radians, it would
        therefore make no sense to allow time series data to be plotted into it.
        However, it might make sense to allow a SineWaveSeries to be plotted
        into a general AvoPlotXYSuplot, and therefore this is permitted by
        AvoPlot. The rule is as follows:

          A data series may be plotted into a subplot if the subplot is an
          instance of the class returned by its get_supported_subplot_type()
          method or any of its base classes.
        """
        return TrigFuncSubplot

class AdvExamplePlugin(plugins.AvoPlotPluginSimple):
        """
        This class is the same as that used for the Sine wave example, except
        that we use the SineWaveSeries data series class that we defined above
        rather than the generic XYDataSeries class used before.
        """
        def __init__(self):
            super(AdvExamplePlugin, self).__init__("Example Plugin with Controls",
                                                   SineWaveSeries)

            self.set_menu_entry(['Examples', 'Adv. Sine Wave'],
                                "Plot a sine wave with variable frequency")

        def plot_into_subplot(self, subplot):

            x_data = numpy.linspace(0, 7, 500)
            y_data = numpy.sin(x_data)

            data_series = SineWaveSeries("adv sine wave", xdata=x_data,
                                         ydata=y_data)

            subplot.add_data_series(data_series)

            return True

def rad2deg(theta, pos):
    """
    Function for converting radians to degrees for use with matplotlib's
    FuncFormatter object.
    """
    return '%0.2f'%math.degrees(theta)

class TrigSubplotUnitsCtrl(controls.AvoPlotControlPanelBase):
    """
    Control panel for trig function subplots allowing their x axis units
    to be changed from radians to degrees.
    """

    def __init__(self, subplot):

        #call the parent class's __init__ method, passing it the name that we
        #want to appear on the control panels tab.
        super(TrigSubplotUnitsCtrl, self).__init__("Units")

        #store the subplot object that this control panel is associated with,
        #so that we can access it later
        self.subplot = subplot

    def setup(self, parent):
        """
        This is where all the controls get added to the control panel. You
        *must* call the setup method of the parent class before doing any of
        your own setup.
        """
        #call parent class's setup method - do this before anything else
        super(TrigSubplotUnitsCtrl, self).setup(parent)

        #create a choice box for the different units for the x axis
        #we use a avoplot.gui.widgets.ChoiceSetting object which is a
        #thin wrapper around a wx.ChoiceBox, but provides a label and
        #automatically registers the event handler.
        units_choice = widgets.ChoiceSetting(self, "x-axis units:", "Radians",
                                             ["Radians", "Degrees"],
                                             self.on_units_change)

        #add the choice widget to the control panel sizer
        self.Add(units_choice, 0,wx.ALL | wx.ALIGN_CENTER_HORIZONTAL, border=10)

    def on_units_change(self, evnt):
        """
        Event handler for change of x axis units events.
        """
        #get the matplotlib axes object from the subplot
        ax = self.subplot.get_mpl_axes()

        #change the axis labels and label formatting based on the choice of
        #units
        if evnt.GetString() == 'Degrees':
            ax.set_xlabel(r'$\theta$ (degrees)')
            ax.xaxis.set_major_formatter(plt.FuncFormatter(rad2deg))
        else:
            ax.set_xlabel(r'$\theta$ (radians)')
            ax.xaxis.set_major_formatter(plt.ScalarFormatter())

        #draw our changes in the display
        self.subplot.update()

class SineWaveFreqCtrl(controls.AvoPlotControlPanelBase):
    """
    Control panel for sine wave data series allowing their frequency to
    be changed using a slider.
    """
    def __init__(self, series):
        #call the parent class's __init__ method, passing it the name that we
        #want to appear on the control panels tab.
        super(SineWaveFreqCtrl, self).__init__("Freq.")

        #store the data series object that this control panel is associated with,
        #so that we can access it later
        self.series = series

    def setup(self, parent):
        """
        This is where all the controls get added to the control panel. You
        *must* call the setup method of the parent class before doing any of
        your own setup.
        """

        #call parent class's setup method - do this before anything else
        super(SineWaveFreqCtrl, self).setup(parent)

        #create a label for the slider
        label = wx.StaticText(self, wx.ID_ANY, 'Frequency')
        self.Add(label, 0,
                 wx.LEFT | wx.RIGHT | wx.TOP | wx.ALIGN_CENTER_HORIZONTAL,
                 border=10)

        #create a frequency slider
        self.slider = wx.Slider(self, wx.ID_ANY, value=1, minValue=1,
                                maxValue=30, style=wx.SL_LABELS)

        #add the slider to the control panel's sizer
        self.Add(self.slider, 0,
                 wx.ALL | wx.EXPAND | wx.ALIGN_CENTER_HORIZONTAL, border=10)

        #register an event handler for slider change events
        wx.EVT_COMMAND_SCROLL(self, self.slider.GetId(), self.on_slider_change)

    def on_slider_change(self, evnt):
        """
        Event handler for frequency slider change events.
        """

        #change the frequency of the sine wave data accordingly
        f = self.slider.GetValue()
        x_data = numpy.linspace(0, 7, 2000)
        y_data = numpy.sin(x_data * f)

        #change the data in the series object
        self.series.set_xy_data(xdata=x_data, ydata=y_data)

        #draw our changes on the display
        self.series.update()

#register the plugin with AvoPlot
plugins.register(AdvExamplePlugin())