#!/usr/bin/env python
"""Utils to parse the terminal output from bench_compress.sh
"""

from __future__ import print_function, division, absolute_import
import os
import sys
import json
import datetime as dt
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

def parse_elapsed(elapsed):
    """Parse string of elapsed time from output of Unix 'time' command into
    `datetime.timedelta`.

    Parameters
    ----------
    elapsed : string
        Elapsed time field output from Unix 'time' command.
        Format: [HH:]MM:SS[.SSS]

    Returns
    -------
    elapsed_dt : datetime.timedelta
        Elapsed time as ``datetime.timedelta``.
    """
    elapsed_arr = elapsed.split(':')
    if len(elapsed_arr) == 2:
        hours = '0'
        [minutes, seconds] = elapsed_arr
    elif len(elapsed_arr) == 3:
        [hours, minutes, seconds] = elapsed_arr
    else:
        raise AssertionError(("Program error. Elapsed time does not have 2 or 3 fields:\n" +
                              "{ea}").format(ea=elapsed_arr))
    hours_int = int(float(hours))
    minutes_int = int(float(minutes))
    seconds_int = int(float(seconds))
    milliseconds_int = int((float(seconds) - seconds_int) / 0.001)
    elapsed_dt = dt.timedelta(hours=hours_int,
                              minutes=minutes_int,
                              seconds=seconds_int,
                              milliseconds=milliseconds_int)
    return elapsed_dt

def recursive_timedelta_to_totsec(dobj):
    """Recursively convert ``datetime.timedelta`` elements to total seconds
    in a ``dict``.

    Call this function before writing the ``dict`` to JSON.

    Parameters
    ----------
    dobj : dict
        ``dict`` that may contain ``datetime.timedelta`` elements. ``dict`` may
        be nested.

    Returns
    -------
    dobj_converted : dict
        ``dict`` with ``datetime.timedelta`` elements converted to
        total seconds.
    """
    dobj_converted = {}
    for key in dobj:
        if isinstance(dobj[key], dt.timedelta):
            dobj_converted[key] = dobj[key].total_seconds()
        elif isinstance(dobj[key], dict):
            dobj_converted[key] = recursive_timedelta_to_totsec(dobj=dobj[key])
        else:
            dobj_converted[key] = dobj[key]
    return dobj_converted

def parse_compress(fin, fout=None):
    """Parse terminal output from bench_compress.sh

    Parse by filename, file size, compression method, compression ratio, compression and decompression speed.
    Note: This function is rigidly dependent upon bench_compress.sh.

    Parameters
    ----------
    fin : string
        Path to text file with terminal output.
    fout : {None}, string, optional
        Path to output .json file of parsed terminal output.

    Returns
    -------
    parsed : dict
        ``dict`` of parsed terminal output.
    """
    # Check input.
    fpath = os.path.abspath(fin)
    if not os.path.isfile(fpath):
        raise IOError("File does not exist:\n{fpath}".format(fpath=fpath))
    if fout is not None:
        if not os.path.splitext(fout)[1] == '.json':
            raise IOError(("File extension is not '.json':\n" +
                           "{fout}").format(fout=fout))
    # Parse text file into dict.
    parsed = {}
    skip_lines = None
    catch_initial_size = None
    catch_comp_cmd = None
    catch_comp_time = None
    catch_comp_size = None
    catch_decomp_cmd = None
    catch_decomp_time = None
    catch_decomp_size = None
    with open(fpath, 'rb') as fobj:
        for line in fobj:
            line = line.rstrip()
            if line.startswith('Begin processing:'):
                line_arr = line.split(':')
                fname = os.path.splitext(os.path.basename(line_arr[1]))[0]
                parsed[fname] = {}
                continue
            # Note: Typo in original script "Intial". Do not correct.
            elif line.startswith('Intial .fastq size:'):
                catch_initial_size = True
                skip_lines = 1
                continue
            elif catch_initial_size and skip_lines >= 0:
                if skip_lines > 0:
                    skip_lines -= 1
                    continue
                elif skip_lines == 0:
                    line_arr = line.split()
                    parsed[fname]['size_bytes'] = int(line_arr[0])
                    assert os.path.basename(line_arr[1]) == fname
                    catch_initial_size = False
                    skip_lines = None
                    continue
            elif line.startswith('Iteration:'):
                line_arr = line.split(':')
                iteration = int(line_arr[1])
                parsed[fname][iteration] = {}
                continue
            elif line.startswith('Testing'):
                line_arr = line.rstrip(':').split()
                method = line_arr[1]
                parsed[fname][iteration][method] = {}
                catch_comp_cmd = True
                continue
            elif catch_comp_cmd and line.startswith('+ sudo time'):
                parsed[fname][iteration][method]['compress'] = {}
                parsed[fname][iteration][method]['compress']['command'] = line
                catch_comp_cmd = False
                catch_comp_time = True
                continue
            elif catch_comp_time and ('elapsed' in line) and ('CPU' in line):
                line_arr = line.split()
                elapsed = parse_elapsed(elapsed=line_arr[2].strip('elapsed'))
                parsed[fname][iteration][method]['compress']['elapsed_time'] = elapsed
                pct_cpu = line_arr[3].strip('%CPU')
                if pct_cpu == '?':
                    pct_cpu = np.NaN
                else:
                    pct_cpu = float(pct_cpu)
                parsed[fname][iteration][method]['compress']['CPU_percent'] = pct_cpu
                catch_comp_time = False
                catch_comp_size = True
                continue
            elif catch_comp_size:
                if line.startswith('+ du --bytes'):
                    skip_lines = 0
                    continue
                elif skip_lines == 0:
                    line_arr = line.split()
                    parsed[fname][iteration][method]['compress']['size_bytes'] = int(line_arr[0])
                    catch_comp_size = False
                    skip_lines = None
                    catch_decomp_cmd = True
                    continue
            elif catch_decomp_cmd and line.startswith('+ sudo time'):
                parsed[fname][iteration][method]['decompress'] = {}
                parsed[fname][iteration][method]['decompress']['command'] = line
                catch_decomp_cmd = False
                catch_decomp_time = True
                continue
            elif catch_decomp_time and ('elapsed' in line) and ('CPU' in line):
                line_arr = line.split()
                elapsed = parse_elapsed(elapsed=line_arr[2].strip('elapsed'))
                parsed[fname][iteration][method]['decompress']['elapsed_time'] = elapsed
                pct_cpu = line_arr[3].strip('%CPU')
                if pct_cpu == '?':
                    pct_cpu = np.NaN
                else:
                    pct_cpu = float(pct_cpu)
                parsed[fname][iteration][method]['decompress']['CPU_percent'] = pct_cpu
                catch_decomp_time = False
                catch_decomp_size = True
                continue
            elif catch_decomp_size:
                if line.startswith('+ du --bytes'):
                    skip_lines = 0
                    continue
                elif skip_lines == 0:
                    line_arr = line.split()
                    parsed[fname][iteration][method]['decompress']['size_bytes'] = int(line_arr[0])
                    if parsed[fname]['size_bytes'] != parsed[fname][iteration][method]['decompress']['size_bytes']:
                        # noinspection PyPep8
                        print(("WARNING: File size before and after compression test do not match.\n" +
                               "file name = {fname}\n" +
                               "method = {method}\n" +
                               "initial size (bytes) = {init_size}\n" +
                               "final size (bytes) = {finl_size}").format(fname=fname, method=method,
                                                                          init_size=parsed[fname]['size_bytes'],
                                                                          finl_size=parsed[fname][iteration][method]['decompress']['size_bytes']),
                              file=sys.stderr)
                    catch_decomp_size = False
                    skip_lines = None
                    continue
    # Write out dict as JSON.
    if fout is not None:
        parsed_converted = recursive_timedelta_to_totsec(dobj=parsed)
        print("Writing parsed text to: {fout}".format(fout=fout))
        with open(fout, "wb") as fobj:
            json.dump(parsed_converted, fobj, indent=4, sort_keys=True)
    return parsed

def parsed_dict_to_df(parsed_dict):
    """Convert ``dict`` from parse_compress to ``pandas.dataframe``.

    Parameters
    ----------
    parsed_dict : dict
        ``dict`` of parsed terminal output.

    Returns
    -------
    parsed_df : pandas.dataframe
        ``pandas.dataframe`` with heirarchical index by filename, iteration,
        method, quantity.
    """
    # TODO: make recursive method, e.g. http://stackoverflow.com/questions/9538875/recursive-depth-of-python-dictionary
    filename_df_dict = {}
    for filename in parsed_dict:
        iteration_df_dict = {}
        for iteration in parsed_dict[filename]:
            method_df_dict = {}
            # Skip size_bytes for file since not a nested dict.
            if isinstance(parsed_dict[filename][iteration], dict):
                for method in parsed_dict[filename][iteration]:
                    method_df_dict[method] = pd.DataFrame.from_dict(parsed_dict[filename][iteration][method],
                                                                    orient='columns')
                iteration_df_dict[iteration] = pd.concat(method_df_dict, axis=1)
        filename_df_dict[filename] = pd.concat(iteration_df_dict, axis=1)
    parsed_df = pd.concat(filename_df_dict, axis=1)
    parsed_df.index.names = ['quantity']
    parsed_df.columns.names = ['filename', 'iteration', 'method', 'process']
    return parsed_df

def condense_parsed_df(parsed_df, parsed_dict):
    """Condense ``pandas.dataframe`` from parsed terminal output.

    Calculate compression/decompression rate in GB per minute and compression ratio, averaging over iterations and
    taking median of results.

    Parameters
    ----------
    parsed_df : pandas.DataFrame
        ``pandas.DataFrame`` from `parsed_dict_to_df`.
        Index name: quantity
        Heirarchical column names: filename, method, process, iteration
    parsed_dict : dict
        Nested ``dict`` from parse_compress.

    Returns
    -------
    condensed_df : pandas.DataFrame
        Heirarchical index names: method, process, quantity
        Column name: quantity

    See Also
    --------
    parsed_dict_to_df, parse_compress, reduce_condensed_df
    """
    # Calculate compression/decompression rate in GB per minute and compression ratio.
    # Drop quantities except for 'GB_per_minute' and 'compression_ratio'. Drop test files and incomplete tests.
    # Average over iterations. Take median of results.
    condensed_df = parsed_df.stack(['filename', 'method', 'process', 'iteration']).unstack('quantity').copy()
    condensed_df['elapsed_seconds'] = condensed_df['elapsed_time'].apply(
        lambda x: x.total_seconds() if isinstance(x, dt.timedelta) else x)
    condensed_df['elapsed_seconds'] = condensed_df['elapsed_seconds'].apply(lambda x: np.NaN if x == 0.0 else x)
    condensed_df['GB_per_minute'] = np.NaN
    condensed_df['compression_ratio'] = np.NaN
    # TODO: Use .values to vectorize
    for fname in condensed_df.index.levels[0].values:
        # TODO: remove SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame
        condensed_df.loc[fname, 'GB_per_minute'].update(
            (parsed_dict[fname]['size_bytes'] / condensed_df.loc[fname, 'elapsed_seconds']).multiply(60.0 / 1.0E9))
        condensed_df.loc[fname, 'compression_ratio'].update(
            condensed_df.loc[fname, 'size_bytes'].div(parsed_dict[fname]['size_bytes']))
    return condensed_df

def reduce_condensed_df(condensed_df):
    """Reduce ``pandas.DataFrame`` from `condense_parsed_df` by averaging over iterations and taking the median over
    file names.

    Parameters
    ----------
    condensed_df : pandas.DataFrame
        Heirarchical index names: method, process, quantity
        Column name: quantity

    Returns
    -------
    reduced_ser :  pandas.Series'
        ``pandas.Series`` from `condense_parsed_df`.
        Heirarchical index names: method, process, quantity

    See Also
    --------
    condense_parsed_df, plot_rate, plot_ratio
    """
    reduced_ser = condensed_df.stack().unstack(['filename', 'method', 'process', 'quantity']).mean()
    reduced_ser = reduced_ser.unstack(['method', 'process', 'quantity']).median()
    return reduced_ser

def plot_rate(reduced_ser, fout=None):
    """Plot processing rate vs compression method.

    Parameters
    ----------
    reduced_ser : pandas.Series
        ``pandas.Series`` from `reduce_condensed_df`.
        Heirarchical index names: method, process, quantity
    fout : {None}, string, optional
        Path to save plot as image. Extension must be supported by ``matplotlib.pyplot.savefig()``

    Returns
    -------
    None

    See Also
    --------
    reduce_condensed_df, plot_ratio
    """
    plt.figure()
    pd.DataFrame.plot(reduced_ser.unstack(['quantity'])['GB_per_minute'].unstack(['process']),
                      title="Processing rate vs compression method\nmedian results over all files",
                      sort_columns=True, kind='bar')
    legend = plt.legend(loc='best', title="Process")
    legend.get_texts()[0].set_text('Compress')
    legend.get_texts()[1].set_text('Decompress')
    xtick_labels = ('(bzip2, --fast)', '(fqz_comp, default)', '(gzip, --fast)', '(quip, default)')
    plt.xticks(xrange(len(xtick_labels)), xtick_labels, rotation=45)
    plt.xlabel("Compression method with options")
    plt.ylabel("Processing rate (GB per minute)")
    if fout is not None:
        print("Writing plot to: {fout}".format(fout=fout))
        plt.savefig(fout, bbox_inches='tight')
    plt.show()
    return None

def plot_ratio(reduced_ser, fout=None):
    """Plot compression ratio vs compression method.

    Parameters
    ----------
    reduced_ser : pandas.Series
        ``pandas.Series`` from `reduce_condensed_df`.
        Heirarchical index names: method, process, quantity
    fout : {None}, string, optional
        Path to save plot as image. Extension must be supported by ``matplotlib.pyplot.savefig()``

    Returns
    -------
    None

    See Also
    --------
    reduce_condensed_df, plot_rate
    """
    plt.figure()
    pd.Series.plot(reduced_ser.unstack(['quantity'])['compression_ratio'].unstack(['process'])['compress'],
                   title="Compression size ratio vs compression method\nmedian results over all files",
                   sort_columns=True, kind='bar')
    xtick_labels = ('(bzip2, --fast)', '(fqz_comp, default)', '(gzip, --fast)', '(quip, default)')
    plt.xticks(xrange(len(xtick_labels)), xtick_labels, rotation=45)
    plt.xlabel("Compression method with options")
    plt.ylabel("Compression size ratio\n(compressed size / decompressed size)")
    if fout is not None:
        print("Writing plot to: {fout}".format(fout=fout))
        plt.savefig(fout, bbox_inches='tight')
    plt.show()
    return None


import numpy as np
import netCDF4
import os
import sys
import subprocess
import pyroms
from pyroms_toolbox import jday2date
from mpl_toolkits.basemap import Basemap
import numpy as np
import matplotlib.pyplot as plt
from datetime import datetime

# draw line around map projection limb.
# color background of map projection region.
# missing values over land will show up this color.
# plot sst, then ice with pcolor
# add a title.
#year = int(sys.argv[1])
#lst_year = [year]

lst_file = []

#for year in lst_year:
#    year = np.str(year)
#lst = subprocess.getoutput('ls clima/*.nc')
lst = subprocess.getoutput('ls 19800110.ocean_daily.nc')
lst = lst.split()
lst_file = lst_file + lst

#grd = pyroms_toolbox.BGrid_GFDL.get_nc_BGrid_GFDL('prog.nc')
grd = netCDF4.Dataset('sea_ice_geometry.nc', "r")

clat = grd.variables["geolatb"][:]
clon = grd.variables["geolonb"][:]

m = Basemap(llcrnrlon=-121., llcrnrlat=17., urcrnrlon=-125.0, urcrnrlat=53.0,\
            rsphere=(6378137.00,6356752.3142),\
            resolution='h', projection='lcc',\
            lat_0=30., lat_1=40.0, lon_0=-78.)
x, y = m(clon, clat)
levels = np.arange(-.6, 0.6, 0.01)
cmap = plt.cm.get_cmap("seismic")

for file in lst_file:
    print("Plotting "+file)
    nc = netCDF4.Dataset(file, "r")
    time = nc.variables["time"][:]
    ntim = len(time)
#   for it in range(10):
    for it in range(0,ntim,30):
        fig = plt.figure(figsize=(4,9))
        ax = fig.add_subplot(111)
        ax.set_aspect('equal')
#       ax.axis(xmin=-300,xmax=300)
#       m.drawmapboundary(fill_color='0.3')
        m.drawcoastlines()
        ssh = nc.variables["RV"][it,0,:-1,:-1]
        ssh *= 1.e4
        time = nc.variables["time"][it]
        cs = m.contourf(x, y, ssh, levels=levels, cmap=cmap, extend='both')
#       csa = m.contour(x, y, ssh, levels=levels, linewidths=(0.5,))
#       cs = plt.contourf(clon, clat, ssh, levels=levels, cmap=cmap, extend='both')
        plt.title('Surface RV')
#       csa = plt.contour(clon, clat, ssh, levels=levels, linewidths=(0.5,))

        cbaxes = fig.add_axes([0.1, 0.05, 0.8, 0.02])
        plt.colorbar(orientation='horizontal', cax=cbaxes)

        print('printing frame:', it)
        fig.savefig('movie/vort_%(number)04d.png'%{'number': it})
        plt.close()

    nc.close()


# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2016, Numenta, Inc.  Unless you have an agreement
# with Numenta, Inc., for a separate license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU Affero Public License for more details.
#
# You should have received a copy of the GNU Affero Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------

"""
This file is used to run Thing experiments using simulated sensations.
"""
import random
import os
from math import ceil
import numpy as np
import pprint
import matplotlib.pyplot as plt
from sklearn import manifold, random_projection

from htmresearch.frameworks.layers.l2_l4_inference import (
  L4L2Experiment, rerunExperimentFromLogfile)

from htmresearch.frameworks.layers.object_machine_factory import (
  createObjectMachine
)

def getL4Params():
  """
  Returns a good default set of parameters to use in the L4 region.
  """
  return {
    "columnCount": 256,
    "cellsPerColumn": 16,
    "learn": True,
    "learnOnOneCell": False,
    "initialPermanence": 0.51,
    "connectedPermanence": 0.6,
    "permanenceIncrement": 0.1,
    "permanenceDecrement": 0.01,
    "minThreshold": 19,
    "predictedSegmentDecrement": 0.0,
    "activationThreshold": 19,
    "sampleSize": 20,
    "implementation": "etm",
  }

def getL2Params():
  """
  Returns a good default set of parameters to use in the L4 region.
  """
  return {
    "inputWidth": 256 * 16,
    "cellCount": 4096,
    "sdrSize": 40,
    "synPermProximalInc": 0.5,
    "synPermProximalDec": 0.0,
    "initialProximalPermanence": 0.6,
    "minThresholdProximal": 9,
    "sampleSizeProximal": 10,
    "connectedPermanenceProximal": 0.5,
    "synPermDistalInc": 0.1,
    "synPermDistalDec": 0.001,
    "initialDistalPermanence": 0.41,
    "activationThresholdDistal": 13,
    "sampleSizeDistal": 30,
    "connectedPermanenceDistal": 0.5,
    "distalSegmentInhibitionFactor": 1.001,
    "learningMode": True,
  }

def locateConvergencePoint(stats, minOverlap, maxOverlap):
  """
  Walk backwards through stats until you locate the first point that diverges
  from target overlap values.  We need this to handle cases where it might get
  to target values, diverge, and then get back again.  We want the last
  convergence point.
  """
  for i,v in enumerate(stats[::-1]):
    if not (v >= minOverlap and v <= maxOverlap):
      return len(stats)-i + 1

  # Never differs - converged in one iteration
  return 1

def averageConvergencePoint(inferenceStats, prefix, minOverlap, maxOverlap,
                            settlingTime):
  """
  inferenceStats contains activity traces while the system visits each object.

  Given the i'th object, inferenceStats[i] contains activity statistics for
  each column for each region for the entire sequence of sensations.

  For each object, compute the convergence time - the first point when all
  L2 columns have converged.

  Return the average convergence time across all objects.

  Given inference statistics for a bunch of runs, locate all traces with the
  given prefix. For each trace locate the iteration where it finally settles
  on targetValue. Return the average settling iteration across all runs.
  """
  convergenceSum = 0.0

  # For each object
  for stats in inferenceStats:

    # For each L2 column locate convergence time
    convergencePoint = 0.0
    for key in stats.iterkeys():
      if prefix in key:
        columnConvergence = locateConvergencePoint(
          stats[key], minOverlap, maxOverlap)

        # Ensure this column has converged by the last iteration
        # assert(columnConvergence <= len(stats[key]))

        convergencePoint = max(convergencePoint, columnConvergence)

    convergenceSum += ceil(float(convergencePoint)/settlingTime)

  return convergenceSum/len(inferenceStats)

def loadThingObjects(numCorticalColumns=1, objDataPath='./data/'):
  """
  Load simulated sensation data on a number of different objects
  There is one file per object, each row contains one feature, location pairs
  The format is as follows
  [(-33.6705, 75.5003, 2.4207)/10] => [[list of active bits of location],
                                       [list of active bits of feature]]
  The content before "=>" is the true 3D location / sensation
  The number of active bits in the location and feature is listed after "=>".
  @return A simple object machine
  """
  # create empty simple object machine
  objects = createObjectMachine(
    machineType="simple",
    numInputBits=20,
    sensorInputSize=1024,
    externalInputSize=1024,
    numCorticalColumns=numCorticalColumns,
    numFeatures=0,
    numLocations=0,
  )

  for _ in range(numCorticalColumns):
    objects.locations.append([])
    objects.features.append([])

  objFiles = []
  for f in os.listdir(objDataPath):
    if os.path.isfile(os.path.join(objDataPath, f)):
      if '.log' in f:
        objFiles.append(f)

  idx = 0
  OnBitsList = []
  for f in objFiles:
    objName = f.split('.')[0]
    objName = objName[4:]
    objFile = open('{}/{}'.format(objDataPath, f))

    sensationList = []
    for line in objFile.readlines():
      # parse thing data file and extract feature/location vectors
      sense = line.split('=>')[1].strip(' ').strip('\n')
      OnBitsList.append(float(line.split('] =>')[0].split('/')[1]))
      location = sense.split('],[')[0].strip('[')
      feature = sense.split('],[')[1].strip(']')
      location = np.fromstring(location, sep=',', dtype=np.uint8)
      feature = np.fromstring(feature, sep=',', dtype=np.uint8)

      # add the current sensation to object Machine
      sensationList.append((idx, idx))
      for c in range(numCorticalColumns):
        objects.locations[c].append(set(location.tolist()))
        objects.features[c].append(set(feature.tolist()))
      idx += 1
    objects.addObject(sensationList, objName)
    print "load object file: {} object name: {} sensation # {}".format(
      f, objName, len(sensationList))
  OnBitsList
  OnBitsList = np.array(OnBitsList)

  plt.figure()
  plt.hist(OnBitsList)
  return objects, OnBitsList

def trainNetwork(objects, numColumns, l4Params, l2Params, verbose=False):
  print " Training sensorimotor network ..."
  objectNames = objects.objects.keys()
  numObjects = len(objectNames)

  exp = L4L2Experiment("shared_features",
                       L2Overrides=l2Params,
                       L4Overrides=l4Params,
                       numCorticalColumns=numColumns)
  exp.learnObjects(objects.provideObjectsToLearn())

  settlingTime = 1
  L2Representations = exp.objectL2Representations
  # if verbose:
  #   print "Learned object representations:"
  #   pprint.pprint(L2Representations, width=400)
  #   print "=========================="

  # For inference, we will check and plot convergence for each object. For each
  # object, we create a sequence of random sensations for each column.  We will
  # present each sensation for settlingTime time steps to let it settle and
  # ensure it converges.

  maxSensationNumber = 30
  overlapMat = np.zeros((numObjects, numObjects, maxSensationNumber))
  numL2ActiveCells = np.zeros((numObjects, maxSensationNumber))
  for objectIdx in range(numObjects):
    objectId = objectNames[objectIdx]
    obj = objects[objectId]

    # Create sequence of sensations for this object for one column. The total
    # number of sensations is equal to the number of points on the object. No
    # point should be visited more than once.
    objectCopy = [pair for pair in obj]
    random.shuffle(objectCopy)
    exp.sendReset()
    for sensationNumber in range(maxSensationNumber):
      objectSensations = {}
      for c in range(numColumns):
        objectSensations[c] = []

      if sensationNumber >= len(objectCopy):
        pair = objectCopy[-1]
      else:
        pair = objectCopy[sensationNumber]
      if numColumns > 1:
        raise NotImplementedError
      else:
        # stay multiple steps on each sensation
        for _ in xrange(settlingTime):
          objectSensations[0].append(pair)

      inferConfig = {
        "object": objectId,
        "numSteps": len(objectSensations[0]),
        "pairs": objectSensations,
        "includeRandomLocation": False,
      }

      inferenceSDRs = objects.provideObjectToInfer(inferConfig)
      exp.infer(inferenceSDRs, objectName=objectId, reset=False)

      for i in range(numObjects):
        overlapMat[objectIdx, i, sensationNumber] = len(
          exp.getL2Representations()[0] &
          L2Representations[objects.objects.keys()[i]][0])
        # if verbose:
        #   print "Intersection with {}:{}".format(
        #     objectNames[i], overlapMat[objectIdx, i])

      for c in range(numColumns):
        numL2ActiveCells[objectIdx, sensationNumber] += len(
          exp.getL2Representations()[c])

      print "{} # L2 active cells {}: ".format(sensationNumber,
                                               numL2ActiveCells[
                                                 objectIdx, sensationNumber])
    if verbose:
      print "Output for {}: {}".format(objectId, exp.getL2Representations())
      print "Final L2 active cells {}: ".format(
        numL2ActiveCells[objectIdx, sensationNumber])
      print
    exp.sendReset()

  expResult = {'overlapMat': overlapMat,
               'numL2ActiveCells': numL2ActiveCells}
  return expResult

def computeAccuracy(expResult, objects):
  objectNames = objects.objects.keys()
  overlapMat = expResult['overlapMat'][:, :, -1]
  numL2ActiveCells = expResult['numL2ActiveCells'][:, -1]
  numCorrect = 0
  numObjects = overlapMat.shape[0]
  numFound = 0

  percentOverlap = np.zeros(overlapMat.shape)
  for i in range(numObjects):
    for j in range(i, numObjects):
      percentOverlap[i, j] = overlapMat[i, j] # / np.min([numL2ActiveCells[i], numL2ActiveCells[j]])

  objectNames = np.array(objectNames)
  for i in range(numObjects):
    # idx = np.where(overlapMat[i, :]>confuseThresh)[0]
    idx = np.where(percentOverlap[i, :] == np.max(percentOverlap[i, :]))[0]
    print " {}, # sensations {}, best match is {}".format(
      objectNames[i], len(objects[objectNames[i]]), objectNames[idx])

    found = len(np.where(idx == i)[0]) > 0
    numFound += found
    if not found:
      print "<=========== {} was not detected ! ===========>".format(objectNames[i])
    if len(idx) > 1:
      continue

    if idx[0] == i:
      numCorrect += 1

  accuracy = float(numCorrect)/numObjects
  numPerfect = len(np.where(numL2ActiveCells<=40)[0])
  print "accuracy: {} ({}/{}) ".format(accuracy, numCorrect, numObjects)
  print "perfect retrival ratio: {} ({}/{}) ".format(
    float(numPerfect)/numObjects, numPerfect, numObjects)
  print "Object detection ratio {}/{} ".format(numFound, numObjects)
  return accuracy

def runExperimentAccuracyVsL4Thresh():
  accuracyVsThresh = []
  threshList = np.arange(13, 20)
  for thresh in threshList:
    numColumns = 1
    l2Params = getL2Params()
    l4Params = getL4Params()

    l4Params['minThreshold'] = thresh
    l4Params['activationThreshold'] = thresh

    objects = loadThingObjects(1, './data')
    expResult = trainNetwork(objects, numColumns, l4Params, l2Params, True)

    accuracy = computeAccuracy(expResult, objects)
    accuracyVsThresh.append(accuracy)

  plt.figure()
  plt.plot(threshList, accuracyVsThresh, '-o')
  plt.xlabel('L4 distal Threshold')
  plt.ylabel('Classification Accuracy')
  plt.savefig('accuracyVsL4Thresh.pdf')
  return threshList, accuracyVsThresh

if __name__ == "__main__":

  # uncomment to plot accuracy as a function of L4 threshold
  # threshList, accuracyVsThresh = runExperimentAccuracyVsL4Thresh()

  numColumns = 1
  l2Params = getL2Params()
  l4Params = getL4Params()
  verbose = 1
  objects, OnBitsList = loadThingObjects(numColumns, './data')

  expResult = trainNetwork(objects, numColumns, l4Params, l2Params, True)

  accuracy = computeAccuracy(expResult, objects)

  objectNames = objects.objects.keys()
  numObjects = len(objectNames)

  overlapMat = expResult['overlapMat']
  numL2ActiveCells = expResult['numL2ActiveCells']

  objectNames = objects.objects.keys()
  numObjects = len(objectNames)

  plt.figure()
  for sensationNumber in range(10):
    plt.imshow(overlapMat[:, :, sensationNumber])
    plt.xticks(range(numObjects), objectNames, rotation='vertical', fontsize=4)
    plt.yticks(range(numObjects), objectNames, fontsize=4)
    plt.title('pairwise overlap at step {}'.format(sensationNumber))
    plt.xlabel('target representation')
    plt.ylabel('inferred representation')
    plt.tight_layout()
    plt.savefig('plots/overlap_matrix_step_{}.png'.format(sensationNumber))

  # plot number of active cells for each object
  plt.figure()
  objectNamesSort = []
  idx = np.argsort(expResult['numL2ActiveCells'][:, -1])
  for i in idx:
    objectNamesSort.append(objectNames[i])
  plt.plot(numL2ActiveCells[idx, -1])
  plt.xticks(range(numObjects), objectNamesSort, rotation='vertical', fontsize=5)
  plt.tight_layout()
  plt.ylabel('Number of active L2 cells')
  plt.savefig('plots/number_of_active_l2_cells.pdf')
  #


import numpy as np
import pytest

from pandas import DataFrame, NaT, compat, date_range
import pandas.util.testing as tm

@pytest.fixture
def float_frame():
    """
    Fixture for DataFrame of floats with index of unique strings

    Columns are ['A', 'B', 'C', 'D'].
    """
    return DataFrame(tm.getSeriesData())

@pytest.fixture
def float_frame_with_na():
    """
    Fixture for DataFrame of floats with index of unique strings

    Columns are ['A', 'B', 'C', 'D']; some entries are missing
    """
    df = DataFrame(tm.getSeriesData())
    # set some NAs
    df.loc[5:10] = np.nan
    df.loc[15:20, -2:] = np.nan
    return df

@pytest.fixture
def float_frame2():
    """
    Fixture for DataFrame of floats with index of unique strings

    Columns are ['D', 'C', 'B', 'A']
    """
    return DataFrame(tm.getSeriesData(), columns=['D', 'C', 'B', 'A'])

@pytest.fixture
def bool_frame_with_na():
    """
    Fixture for DataFrame of booleans with index of unique strings

    Columns are ['A', 'B', 'C', 'D']; some entries are missing
    """
    df = DataFrame(tm.getSeriesData()) > 0
    df = df.astype(object)
    # set some NAs
    df.loc[5:10] = np.nan
    df.loc[15:20, -2:] = np.nan
    return df

@pytest.fixture
def int_frame():
    """
    Fixture for DataFrame of ints with index of unique strings

    Columns are ['A', 'B', 'C', 'D']
    """
    df = DataFrame({k: v.astype(int)
                   for k, v in compat.iteritems(tm.getSeriesData())})
    # force these all to int64 to avoid platform testing issues
    return DataFrame({c: s for c, s in compat.iteritems(df)}, dtype=np.int64)

@pytest.fixture
def datetime_frame():
    """
    Fixture for DataFrame of floats with DatetimeIndex

    Columns are ['A', 'B', 'C', 'D']
    """
    return DataFrame(tm.getTimeSeriesData())

@pytest.fixture
def float_string_frame():
    """
    Fixture for DataFrame of floats and strings with index of unique strings

    Columns are ['A', 'B', 'C', 'D', 'foo'].
    """
    df = DataFrame(tm.getSeriesData())
    df['foo'] = 'bar'
    return df

@pytest.fixture
def mixed_float_frame():
    """
    Fixture for DataFrame of different float types with index of unique strings

    Columns are ['A', 'B', 'C', 'D'].
    """
    df = DataFrame(tm.getSeriesData())
    df.A = df.A.astype('float32')
    df.B = df.B.astype('float32')
    df.C = df.C.astype('float16')
    df.D = df.D.astype('float64')
    return df

@pytest.fixture
def mixed_float_frame2():
    """
    Fixture for DataFrame of different float types with index of unique strings

    Columns are ['A', 'B', 'C', 'D'].
    """
    df = DataFrame(tm.getSeriesData())
    df.D = df.D.astype('float32')
    df.C = df.C.astype('float32')
    df.B = df.B.astype('float16')
    df.D = df.D.astype('float64')
    return df

@pytest.fixture
def mixed_int_frame():
    """
    Fixture for DataFrame of different int types with index of unique strings

    Columns are ['A', 'B', 'C', 'D'].
    """
    df = DataFrame({k: v.astype(int)
                   for k, v in compat.iteritems(tm.getSeriesData())})
    df.A = df.A.astype('int32')
    df.B = np.ones(len(df.B), dtype='uint64')
    df.C = df.C.astype('uint8')
    df.D = df.C.astype('int64')
    return df

@pytest.fixture
def mixed_type_frame():
    """
    Fixture for DataFrame of float/int/string columns with RangeIndex

    Columns are ['a', 'b', 'c', 'float32', 'int32'].
    """
    return DataFrame({'a': 1., 'b': 2, 'c': 'foo',
                      'float32': np.array([1.] * 10, dtype='float32'),
                      'int32': np.array([1] * 10, dtype='int32')},
                     index=np.arange(10))

@pytest.fixture
def timezone_frame():
    """
    Fixture for DataFrame of date_range Series with different time zones

    Columns are ['A', 'B', 'C']; some entries are missing
    """
    df = DataFrame({'A': date_range('20130101', periods=3),
                    'B': date_range('20130101', periods=3,
                                    tz='US/Eastern'),
                    'C': date_range('20130101', periods=3,
                                    tz='CET')})
    df.iloc[1, 1] = NaT
    df.iloc[1, 2] = NaT
    return df

@pytest.fixture
def empty_frame():
    """
    Fixture for empty DataFrame
    """
    return DataFrame({})

@pytest.fixture
def datetime_series():
    """
    Fixture for Series of floats with DatetimeIndex
    """
    return tm.makeTimeSeries(nper=30)

@pytest.fixture
def datetime_series_short():
    """
    Fixture for Series of floats with DatetimeIndex
    """
    return tm.makeTimeSeries(nper=30)[5:]

@pytest.fixture
def simple_frame():
    """
    Fixture for simple 3x3 DataFrame

    Columns are ['one', 'two', 'three'], index is ['a', 'b', 'c'].
    """
    arr = np.array([[1., 2., 3.],
                    [4., 5., 6.],
                    [7., 8., 9.]])

    return DataFrame(arr, columns=['one', 'two', 'three'],
                     index=['a', 'b', 'c'])

@pytest.fixture
def frame_of_index_cols():
    """
    Fixture for DataFrame of columns that can be used for indexing

    Columns are ['A', 'B', 'C', 'D', 'E', ('tuple', 'as', 'label')];
    'A' & 'B' contain duplicates (but are jointly unique), the rest are unique.
    """
    df = DataFrame({'A': ['foo', 'foo', 'foo', 'bar', 'bar'],
                    'B': ['one', 'two', 'three', 'one', 'two'],
                    'C': ['a', 'b', 'c', 'd', 'e'],
                    'D': np.random.randn(5),
                    'E': np.random.randn(5),
                    ('tuple', 'as', 'label'): np.random.randn(5)})
    return df


import sys
from sklearn.externals.six.moves import cStringIO as StringIO
import numpy as np
import scipy.sparse as sp
from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_almost_equal
from sklearn.utils.testing import assert_less
from sklearn.utils.testing import assert_raises_regexp
from sklearn.utils import check_random_state
from sklearn.manifold.t_sne import _joint_probabilities
from sklearn.manifold.t_sne import _kl_divergence
from sklearn.manifold.t_sne import _gradient_descent
from sklearn.manifold.t_sne import trustworthiness
from sklearn.manifold.t_sne import TSNE
from sklearn.manifold._utils import _binary_search_perplexity
from scipy.optimize import check_grad
from scipy.spatial.distance import pdist
from scipy.spatial.distance import squareform

def test_gradient_descent_stops():
    # Test stopping conditions of gradient descent.
    class ObjectiveSmallGradient:
        def __init__(self):
            self.it = -1

        def __call__(self, _):
            self.it += 1
            return (10 - self.it) / 10.0, np.array([1e-5])

    def flat_function(_):
        return 0.0, np.ones(1)

    # Gradient norm
    old_stdout = sys.stdout
    sys.stdout = StringIO()
    try:
        _, error, it = _gradient_descent(
            ObjectiveSmallGradient(), np.zeros(1), 0, n_iter=100,
            n_iter_without_progress=100, momentum=0.0, learning_rate=0.0,
            min_gain=0.0, min_grad_norm=1e-5, min_error_diff=0.0, verbose=2)
    finally:
        out = sys.stdout.getvalue()
        sys.stdout.close()
        sys.stdout = old_stdout
    assert_equal(error, 1.0)
    assert_equal(it, 0)
    assert("gradient norm" in out)

    # Error difference
    old_stdout = sys.stdout
    sys.stdout = StringIO()
    try:
        _, error, it = _gradient_descent(
            ObjectiveSmallGradient(), np.zeros(1), 0, n_iter=100,
            n_iter_without_progress=100, momentum=0.0, learning_rate=0.0,
            min_gain=0.0, min_grad_norm=0.0, min_error_diff=0.2, verbose=2)
    finally:
        out = sys.stdout.getvalue()
        sys.stdout.close()
        sys.stdout = old_stdout
    assert_equal(error, 0.9)
    assert_equal(it, 1)
    assert("error difference" in out)

    # Maximum number of iterations without improvement
    old_stdout = sys.stdout
    sys.stdout = StringIO()
    try:
        _, error, it = _gradient_descent(
            flat_function, np.zeros(1), 0, n_iter=100,
            n_iter_without_progress=10, momentum=0.0, learning_rate=0.0,
            min_gain=0.0, min_grad_norm=0.0, min_error_diff=-1.0, verbose=2)
    finally:
        out = sys.stdout.getvalue()
        sys.stdout.close()
        sys.stdout = old_stdout
    assert_equal(error, 0.0)
    assert_equal(it, 11)
    assert("did not make any progress" in out)

    # Maximum number of iterations
    old_stdout = sys.stdout
    sys.stdout = StringIO()
    try:
        _, error, it = _gradient_descent(
            ObjectiveSmallGradient(), np.zeros(1), 0, n_iter=11,
            n_iter_without_progress=100, momentum=0.0, learning_rate=0.0,
            min_gain=0.0, min_grad_norm=0.0, min_error_diff=0.0, verbose=2)
    finally:
        out = sys.stdout.getvalue()
        sys.stdout.close()
        sys.stdout = old_stdout
    assert_equal(error, 0.0)
    assert_equal(it, 10)
    assert("Iteration 10" in out)

def test_binary_search():
    # Test if the binary search finds Gaussians with desired perplexity.
    random_state = check_random_state(0)
    distances = random_state.randn(50, 2)
    distances = distances.dot(distances.T)
    np.fill_diagonal(distances, 0.0)
    desired_perplexity = 25.0
    P = _binary_search_perplexity(distances, desired_perplexity, verbose=0)
    P = np.maximum(P, np.finfo(np.double).eps)
    mean_perplexity = np.mean([np.exp(-np.sum(P[i] * np.log(P[i])))
                               for i in range(P.shape[0])])
    assert_almost_equal(mean_perplexity, desired_perplexity, decimal=3)

def test_gradient():
    # Test gradient of Kullback-Leibler divergence.
    random_state = check_random_state(0)

    n_samples = 50
    n_features = 2
    n_components = 2
    alpha = 1.0

    distances = random_state.randn(n_samples, n_features)
    distances = distances.dot(distances.T)
    np.fill_diagonal(distances, 0.0)
    X_embedded = random_state.randn(n_samples, n_components)

    P = _joint_probabilities(distances, desired_perplexity=25.0,
                             verbose=0)
    fun = lambda params: _kl_divergence(params, P, alpha, n_samples,
                                        n_components)[0]
    grad = lambda params: _kl_divergence(params, P, alpha, n_samples,
                                         n_components)[1]
    assert_almost_equal(check_grad(fun, grad, X_embedded.ravel()), 0.0,
                        decimal=5)

def test_trustworthiness():
    # Test trustworthiness score.
    random_state = check_random_state(0)

    # Affine transformation
    X = random_state.randn(100, 2)
    assert_equal(trustworthiness(X, 5.0 + X / 10.0), 1.0)

    # Randomly shuffled
    X = np.arange(100).reshape(-1, 1)
    X_embedded = X.copy()
    random_state.shuffle(X_embedded)
    assert_less(trustworthiness(X, X_embedded), 0.6)

    # Completely different
    X = np.arange(5).reshape(-1, 1)
    X_embedded = np.array([[0], [2], [4], [1], [3]])
    assert_almost_equal(trustworthiness(X, X_embedded, n_neighbors=1), 0.2)

def test_preserve_trustworthiness_approximately():
    # Nearest neighbors should be preserved approximately.
    random_state = check_random_state(0)
    X = random_state.randn(100, 2)
    for init in ('random', 'pca'):
        tsne = TSNE(n_components=2, perplexity=10, learning_rate=100.0,
                    init=init, random_state=0)
        X_embedded = tsne.fit_transform(X)
        assert_almost_equal(trustworthiness(X, X_embedded, n_neighbors=1), 1.0,
                            decimal=1)

def test_fit_csr_matrix():
    # X can be a sparse matrix.
    random_state = check_random_state(0)
    X = random_state.randn(100, 2)
    X[(np.random.randint(0, 100, 50), np.random.randint(0, 2, 50))] = 0.0
    X_csr = sp.csr_matrix(X)
    tsne = TSNE(n_components=2, perplexity=10, learning_rate=100.0,
                random_state=0)
    X_embedded = tsne.fit_transform(X_csr)
    assert_almost_equal(trustworthiness(X_csr, X_embedded, n_neighbors=1), 1.0,
                        decimal=1)

def test_preserve_trustworthiness_approximately_with_precomputed_distances():
    # Nearest neighbors should be preserved approximately.
    random_state = check_random_state(0)
    X = random_state.randn(100, 2)
    D = squareform(pdist(X), "sqeuclidean")
    tsne = TSNE(n_components=2, perplexity=10, learning_rate=100.0,
                metric="precomputed", random_state=0)
    X_embedded = tsne.fit_transform(D)
    assert_almost_equal(trustworthiness(D, X_embedded, n_neighbors=1,
                                        precomputed=True), 1.0, decimal=1)

def test_early_exaggeration_too_small():
    # Early exaggeration factor must be >= 1.
    tsne = TSNE(early_exaggeration=0.99)
    assert_raises_regexp(ValueError, "early_exaggeration .*",
                         tsne.fit_transform, np.array([[0.0]]))

def test_too_few_iterations():
    # Number of gradient descent iterations must be at least 200.
    tsne = TSNE(n_iter=199)
    assert_raises_regexp(ValueError, "n_iter .*", tsne.fit_transform,
                         np.array([[0.0]]))

def test_non_square_precomputed_distances():
    # Precomputed distance matrices must be square matrices.
    tsne = TSNE(metric="precomputed")
    assert_raises_regexp(ValueError, ".* square distance matrix",
                         tsne.fit_transform, np.array([[0.0], [1.0]]))

def test_init_not_available():
    # 'init' must be 'pca' or 'random'.
    assert_raises_regexp(ValueError, "'init' must be either 'pca' or 'random'",
                         TSNE, init="not available")

def test_distance_not_available():
    # 'metric' must be valid.
    tsne = TSNE(metric="not available")
    assert_raises_regexp(ValueError, "Unknown metric not available.*",
                         tsne.fit_transform, np.array([[0.0], [1.0]]))

def test_pca_initialization_not_compatible_with_precomputed_kernel():
    # Precomputed distance matrices must be square matrices.
    tsne = TSNE(metric="precomputed", init="pca")
    assert_raises_regexp(ValueError, "The parameter init=\"pca\" cannot be "
                         "used with metric=\"precomputed\".",
                         tsne.fit_transform, np.array([[0.0], [1.0]]))

def test_verbose():
    random_state = check_random_state(0)
    tsne = TSNE(verbose=2)
    X = random_state.randn(5, 2)

    old_stdout = sys.stdout
    sys.stdout = StringIO()
    try:
        tsne.fit_transform(X)
    finally:
        out = sys.stdout.getvalue()
        sys.stdout.close()
        sys.stdout = old_stdout

    assert("[t-SNE]" in out)
    assert("Computing pairwise distances" in out)
    assert("Computed conditional probabilities" in out)
    assert("Mean sigma" in out)
    assert("Finished" in out)
    assert("early exaggeration" in out)
    assert("Finished" in out)

def test_chebyshev_metric():
    # t-SNE should allow metrics that cannot be squared (issue #3526).
    random_state = check_random_state(0)
    tsne = TSNE(metric="chebyshev")
    X = random_state.randn(5, 2)
    tsne.fit_transform(X)

def test_reduction_to_one_component():
    # t-SNE should allow reduction to one component (issue #4154).
    random_state = check_random_state(0)
    tsne = TSNE(n_components=1)
    X = random_state.randn(5, 2)
    X_embedded = tsne.fit(X).embedding_
    assert(np.all(np.isfinite(X_embedded)))


#!/usr/bin/python

import sys
import os
import re
import fnmatch
import string
import matplotlib.pyplot as plt

def print_period(stat):
    # use to profile the application running solo.
    # stat is an iterator or array
    i = 0
    for item in stat:
        plt.plot(item, label=str(i))
        plt.legend()
        i = i + 1

def print_double_array (x):
	for x_i in x:
		sys.stdout.write(str("%.2f" % x_i) + ' ')
	print "\n"
	sys.stdout.flush()

def print_int_array (x):
	for x_i in x:
		sys.stdout.write(str(x_i) + ' ')
	print "\n"
	sys.stdout.flush()

def print_stat_dict (my_stat):
	for key, value in iter(sorted(my_stat.iteritems())):
		if type(value) is not list:
			print key.ljust(20), value.ljust(20)
#		else:
#			for element in value:
#				print element

def print_power (stat):
	output_str = '\n\n#############    Power Distribution  ################\n\n'
	output_str = output_str + ''.ljust(15) + 'Static'.ljust(20) + 'Dynamic'.ljust(20) + 'Overall'.ljust(20) + '\n'
	## print BLESS
	static_percent = "{:.2f}".format(stat[0]/stat[2]*100)
	dynamic_percent = "{:.2f}".format(stat[1]/stat[2]*100)
	output_str = output_str + 'BLESS'.ljust(15) + ('%s (%s%%)'%("{:.2f}".format(stat[0]),static_percent)).ljust(20) + ('%s (%s%%)'%("{:.2f}".format(stat[1]),dynamic_percent)).ljust(20) + str(stat[2]).ljust(20) + '\n'
	# print MBNoC
	static_percent = "{:.2f}".format(stat[3]/stat[5]*100)
	dynamic_percent = "{:.2f}".format(stat[4]/stat[5]*100)
	output_str = output_str + 'MBNoC'.ljust(15) + ('%s (%s%%)'%("{:.2f}".format(stat[3]),static_percent)).ljust(20) + ('%s (%s%%)'%("{:.2f}".format(stat[4]),dynamic_percent)).ljust(20) + str(stat[5]).ljust(20)
	output_str = output_str + '\n'

	print output_str

def print_power_breakdown (stat):
	output_str = '\n\n#############    Power Breakdown   ################\n\n'
	output_str = output_str + ''.ljust(15) + 'Static'.ljust(20) + 'Dynamic'.ljust(20) + 'Overall'.ljust(20) + '\n'
	output_str = output_str + 'Component'.ljust(15) + 'BLESS'.ljust(10) + 'MBNoC'.ljust(10) + 'BLESS'.ljust(10) + 'MBNoC'.ljust(10) + 'BLESS'.ljust(10) + 'MBNoC'.ljust(10) + '\n'
	print_order = ['DFF', 'portAlloc', 'RC', 'Xbar', 'Local', 'permNet', 'link']
	for component in range (0, 7):
		output_str = output_str + print_order[component].ljust(15)
		for metric in stat:
			output_str = output_str + str(metric[component+1]).ljust(10)
		output_str = output_str + '\n'

	print output_str

def print_final_stat (stat):
	output_str = '\n\n#############    Overall    ################\n\n'
	output_str = output_str + ''.ljust(20) + 'weighted_speedup'.ljust(20) + 'Energy'.ljust(20) + 'Throughput'.ljust(20) + 'Defection Rate'.ljust(20) + '\n'
	output_str = output_str + 'Load'.ljust(10) + 'Count'.ljust(10)
	for i in range (0, 4):
		output_str = output_str + 'BLESS'.ljust(10) + 'MBNoC'.ljust(10)
	output_str = output_str + '\n' + 'Low'.ljust(10)
	for metric in stat[0]:
		output_str = output_str + str(metric).ljust(10)
	output_str = output_str + '\n'
	output_str = output_str + 'Medium'.ljust(10)
	for metric in stat[1]:
		output_str = output_str + str(metric).ljust(10)
	output_str = output_str + '\n'
	output_str = output_str + 'High'.ljust(10)
	for metric in stat[2]:
		output_str = output_str + str(metric).ljust(10)
	output_str = output_str + '\n'
	output_str = output_str + 'Average'.ljust(10)
	for metric in stat[3]:
		output_str = output_str + str(metric).ljust(10)
	output_str = output_str + '\n'
	print output_str
	return output_str

def print_for_plot (stat):
	output_str = '\n\n#############    Print for plot    ################\n\n'
	output_str = output_str + 'Baseline of each metrics of interest is 1.\nEach metric is normailized to BLESS with the same network size.\n\n'
	output_str = output_str + 'Load'.ljust(8) + 'Count'.ljust(8) + 'ws'.ljust(8) + '4x4'.ljust(8) + '8x8'.ljust(8) + '16x6'.ljust(8) + 'engy'.ljust(8) + '4x4'.ljust(8) + '8x8'.ljust(8) + '16x16'.ljust(8) + 'th'.ljust(8) + '4x4'.ljust(8) + '8x8'.ljust(8) + '16x16'.ljust(8) + 'defl'.ljust(8) + '4x4'.ljust(8) + '8x8'.ljust(8) + '16x16'.ljust(8) + '\n'
	groups = ['Low','Medium','High','Average']
	i = 0
	for element in stat:
		output_str = output_str + groups[i].ljust(8)
		for metric in element:
			output_str = output_str + str(metric).ljust(8)
		i = i + 1
		output_str = output_str + '\n'

	print output_str
	return output_str

def print_synth (stat, design):
	traffic = str(stat.pop(0))
	network = str(stat.pop(0))
	#output_str = '\n\n#############    ' + "Traffic = " + traffic.ljust(20) + "Network = " + network.ljust(20) + '   ################\n\n'
	#output_str = output_str + 'Inject_rate'.ljust(20) + 'Energy'.ljust(20) + 'Latency'.ljust(20) + 'Deflect_rate'.ljust(20) + 'Throughput'.ljust(20) + '\n\n'
	#output_str = output_str + 'BLESS'.ljust(10) + 'MBNoC'.ljust(10) + 'BLESS'.ljust(10) + 'MBNoC'.ljust(10) + 'BLESS'.ljust(10) + 'MBNoC'.ljust(10) + 'BLESS'.ljust(10) + 'MBNoC'.ljust(10) + 'BLESS'.ljust(10) + 'MBNoC'.ljust(10) + '\n'

	output_str = '\n\n#############    ' + 'Traffic = ' + traffic.ljust(20) + 'Network = ' + network.ljust(20) + '   ################\n\n'
	type_stat = len(stat) / len(design)
	#for i in range (0, type_stat):
	space = (len(design)+1)*10
	output_str = output_str + 'Energy'.ljust(space) + 'Latency'.ljust(space) +  'Throughput'.ljust(space) + 'Deflect_rate'.ljust(space) + '\n\n'

	for i in range (1, 80, 1):
		load = "{:.2f}".format(float(i)/100)
		for j in range (0, len(stat)):
			if j % len(design) is 0:
				output_str = output_str + load.ljust(10)
			if load in stat[j]:
				output_str = output_str + str(stat[j][load]).ljust(10)
			else:
				output_str = output_str + '-'.ljust(10)
		output_str = output_str + '\n'

	#for i in range (0, len(stat[0])):
	#	for j in range (0, len(stat)):
	#		output_str = output_str + str(stat[j][i]).ljust(10)
	#	output_str = output_str + '\n'
	output_str = output_str + '********* Based on %u data points ************' % len(stat[0])
	print output_str

def print_synth_wrt_load (stat, design):
	traffic = str(stat.pop(0))
	network = str(stat.pop(0))

	output_str = '\n\n#############    ' + 'Traffic = ' + traffic.ljust(20) + 'Network = ' + network.ljust(20) + '   ################\n\n'
	type_stat = len(stat) / len(design)
	#for i in range (0, type_stat):
	space = (len(design)+1)*10
	output_str = output_str + 'Latency'.ljust(space) +  'Throughput'.ljust(space) + 'Deflect_rate'.ljust(space) + '\n\n'

	for i in range (0, type_stat):
	 	output_str = output_str + 'InjRate'.ljust(10)
		for element in design:
			output_str = output_str + element.ljust(10)
	output_str = output_str + '\n'

	for i in range (1, 80, 1):
		load = "{:.2f}".format(float(i)/100)
		for j in range (0, len(stat)):
			if j % len(design) is 0:
				output_str = output_str + load.ljust(10)
			if load in stat[j]:
				output_str = output_str + str(stat[j][load]).ljust(10)
			else:
				output_str = output_str + '-'.ljust(10)
		output_str = output_str + '\n'

	output_str = output_str + '********* Based on %u data points ************' % len(stat[0])
	print output_str

def print_synth_avg_reduction (stat, design):
	output_str = ''
	for element in design:
		output_str = output_str + element.ljust(10)
	baseline = stat[0]
	output_str = output_str + '\n' + '1'.ljust(10)
	stat.pop(0)
	for element in stat:
		reduction = ''
		if baseline > 0: reduction = "{:.2f}".format((baseline - element) / baseline)
		output_str = output_str + reduction.ljust(10)
	output_str = output_str + '\n'
	print output_str

def print_synth_avg_gain (stat, design):
	output_str = ''
	for element in design:
		output_str = output_str + element.ljust(10)
	baseline = stat[0]
	output_str = output_str + '\n' + '1'.ljust(10)
	stat.pop(0)
	for element in stat:
		reduction = ''
		if baseline > 0: reduction = "{:.2f}".format((element - baseline) / baseline)
		output_str = output_str + reduction.ljust(10)
	output_str = output_str + '\n'
	print output_str

def print_final (stat, design):
	output_str = ''
	for element in design:
		output_str = output_str + element.ljust(10)
	output_str = output_str + '\n'
	for element in stat:
		output_str = output_str + "{:.2f}".format(float(element)).ljust(10)

	output_str = output_str + '\n'
	print output_str


import numpy as np
import pytest

import pandas as pd
import pandas._testing as tm
from pandas.core.arrays import FloatingArray

@pytest.mark.parametrize("ufunc", [np.abs, np.sign])
# np.sign emits a warning with nans, <https://github.com/numpy/numpy/issues/15127>
@pytest.mark.filterwarnings("ignore:invalid value encountered in sign")
def test_ufuncs_single_int(ufunc):
    a = pd.array([1, 2, -3, np.nan])
    result = ufunc(a)
    expected = pd.array(ufunc(a.astype(float)), dtype="Int64")
    tm.assert_extension_array_equal(result, expected)

    s = pd.Series(a)
    result = ufunc(s)
    expected = pd.Series(pd.array(ufunc(a.astype(float)), dtype="Int64"))
    tm.assert_series_equal(result, expected)

@pytest.mark.parametrize("ufunc", [np.log, np.exp, np.sin, np.cos, np.sqrt])
def test_ufuncs_single_float(ufunc):
    a = pd.array([1, 2, -3, np.nan])
    with np.errstate(invalid="ignore"):
        result = ufunc(a)
        expected = FloatingArray(ufunc(a.astype(float)), mask=a._mask)
    tm.assert_extension_array_equal(result, expected)

    s = pd.Series(a)
    with np.errstate(invalid="ignore"):
        result = ufunc(s)
    expected = pd.Series(expected)
    tm.assert_series_equal(result, expected)

@pytest.mark.parametrize("ufunc", [np.add, np.subtract])
def test_ufuncs_binary_int(ufunc):
    # two IntegerArrays
    a = pd.array([1, 2, -3, np.nan])
    result = ufunc(a, a)
    expected = pd.array(ufunc(a.astype(float), a.astype(float)), dtype="Int64")
    tm.assert_extension_array_equal(result, expected)

    # IntegerArray with numpy array
    arr = np.array([1, 2, 3, 4])
    result = ufunc(a, arr)
    expected = pd.array(ufunc(a.astype(float), arr), dtype="Int64")
    tm.assert_extension_array_equal(result, expected)

    result = ufunc(arr, a)
    expected = pd.array(ufunc(arr, a.astype(float)), dtype="Int64")
    tm.assert_extension_array_equal(result, expected)

    # IntegerArray with scalar
    result = ufunc(a, 1)
    expected = pd.array(ufunc(a.astype(float), 1), dtype="Int64")
    tm.assert_extension_array_equal(result, expected)

    result = ufunc(1, a)
    expected = pd.array(ufunc(1, a.astype(float)), dtype="Int64")
    tm.assert_extension_array_equal(result, expected)

def test_ufunc_binary_output():
    a = pd.array([1, 2, np.nan])
    result = np.modf(a)
    expected = np.modf(a.to_numpy(na_value=np.nan, dtype="float"))
    expected = (pd.array(expected[0]), pd.array(expected[1]))

    assert isinstance(result, tuple)
    assert len(result) == 2

    for x, y in zip(result, expected):
        tm.assert_extension_array_equal(x, y)

@pytest.mark.parametrize("values", [[0, 1], [0, None]])
def test_ufunc_reduce_raises(values):
    a = pd.array(values)
    msg = r"The 'reduce' method is not supported."
    with pytest.raises(NotImplementedError, match=msg):
        np.add.reduce(a)

@pytest.mark.parametrize(
    "pandasmethname, kwargs",
    [
        ("var", {"ddof": 0}),
        ("var", {"ddof": 1}),
        ("kurtosis", {}),
        ("skew", {}),
        ("sem", {}),
    ],
)
def test_stat_method(pandasmethname, kwargs):
    s = pd.Series(data=[1, 2, 3, 4, 5, 6, np.nan, np.nan], dtype="Int64")
    pandasmeth = getattr(s, pandasmethname)
    result = pandasmeth(**kwargs)
    s2 = pd.Series(data=[1, 2, 3, 4, 5, 6], dtype="Int64")
    pandasmeth = getattr(s2, pandasmethname)
    expected = pandasmeth(**kwargs)
    assert expected == result

def test_value_counts_na():
    arr = pd.array([1, 2, 1, pd.NA], dtype="Int64")
    result = arr.value_counts(dropna=False)
    expected = pd.Series([2, 1, 1], index=[1, 2, pd.NA], dtype="Int64")
    tm.assert_series_equal(result, expected)

    result = arr.value_counts(dropna=True)
    expected = pd.Series([2, 1], index=[1, 2], dtype="Int64")
    tm.assert_series_equal(result, expected)

def test_value_counts_empty():
    # https://github.com/pandas-dev/pandas/issues/33317
    s = pd.Series([], dtype="Int64")
    result = s.value_counts()
    # TODO: The dtype of the index seems wrong (it's int64 for non-empty)
    idx = pd.Index([], dtype="object")
    expected = pd.Series([], index=idx, dtype="Int64")
    tm.assert_series_equal(result, expected)

def test_value_counts_with_normalize():
    # GH 33172
    s = pd.Series([1, 2, 1, pd.NA], dtype="Int64")
    result = s.value_counts(normalize=True)
    expected = pd.Series([2, 1], index=[1, 2], dtype="Float64") / 3
    tm.assert_series_equal(result, expected)

@pytest.mark.parametrize("skipna", [True, False])
@pytest.mark.parametrize("min_count", [0, 4])
def test_integer_array_sum(skipna, min_count, any_nullable_int_dtype):
    dtype = any_nullable_int_dtype
    arr = pd.array([1, 2, 3, None], dtype=dtype)
    result = arr.sum(skipna=skipna, min_count=min_count)
    if skipna and min_count == 0:
        assert result == 6
    else:
        assert result is pd.NA

@pytest.mark.parametrize("skipna", [True, False])
@pytest.mark.parametrize("method", ["min", "max"])
def test_integer_array_min_max(skipna, method, any_nullable_int_dtype):
    dtype = any_nullable_int_dtype
    arr = pd.array([0, 1, None], dtype=dtype)
    func = getattr(arr, method)
    result = func(skipna=skipna)
    if skipna:
        assert result == (0 if method == "min" else 1)
    else:
        assert result is pd.NA

@pytest.mark.parametrize("skipna", [True, False])
@pytest.mark.parametrize("min_count", [0, 9])
def test_integer_array_prod(skipna, min_count, any_nullable_int_dtype):
    dtype = any_nullable_int_dtype
    arr = pd.array([1, 2, None], dtype=dtype)
    result = arr.prod(skipna=skipna, min_count=min_count)
    if skipna and min_count == 0:
        assert result == 2
    else:
        assert result is pd.NA

@pytest.mark.parametrize(
    "values, expected", [([1, 2, 3], 6), ([1, 2, 3, None], 6), ([None], 0)]
)
def test_integer_array_numpy_sum(values, expected):
    arr = pd.array(values, dtype="Int64")
    result = np.sum(arr)
    assert result == expected

@pytest.mark.parametrize("op", ["sum", "prod", "min", "max"])
def test_dataframe_reductions(op):
    # https://github.com/pandas-dev/pandas/pull/32867
    # ensure the integers are not cast to float during reductions
    df = pd.DataFrame({"a": pd.array([1, 2], dtype="Int64")})
    result = df.max()
    assert isinstance(result["a"], np.int64)

# TODO(jreback) - these need testing / are broken

# shift

# set_index (destroys type)


# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Implementation of k-means clustering on top of `Estimator` API.

This module is deprecated. Please use
@{tf.contrib.factorization.KMeansClustering} instead of
@{tf.contrib.learn.KMeansClustering}. It has a similar interface, but uses the
@{tf.estimator.Estimator} API instead of @{tf.contrib.learn.Estimator}.
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import time
import numpy as np

from tensorflow.contrib.factorization.python.ops import clustering_ops
from tensorflow.python.training import training_util
from tensorflow.contrib.learn.python.learn.estimators import estimator
from tensorflow.contrib.learn.python.learn.estimators.model_fn import ModelFnOps
from tensorflow.python.framework import ops
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import math_ops
from tensorflow.python.ops import state_ops
from tensorflow.python.ops.control_flow_ops import with_dependencies
from tensorflow.python.platform import tf_logging as logging
from tensorflow.python.summary import summary
from tensorflow.python.training import session_run_hook
from tensorflow.python.training.session_run_hook import SessionRunArgs
from tensorflow.python.util.deprecation import deprecated

_USE_TF_CONTRIB_FACTORIZATION = (
    'Please use tf.contrib.factorization.KMeansClustering instead of'
    ' tf.contrib.learn.KMeansClustering. It has a similar interface, but uses'
    ' the tf.estimator.Estimator API instead of tf.contrib.learn.Estimator.')

class _LossRelativeChangeHook(session_run_hook.SessionRunHook):
  """Stops when the change in loss goes below a tolerance."""

  def __init__(self, tolerance):
    """Initializes _LossRelativeChangeHook.

    Args:
      tolerance: A relative tolerance of change between iterations.
    """
    self._tolerance = tolerance
    self._prev_loss = None

  def begin(self):
    self._loss_tensor = ops.get_default_graph().get_tensor_by_name(
        KMeansClustering.LOSS_OP_NAME + ':0')
    assert self._loss_tensor is not None

  def before_run(self, run_context):
    del run_context
    return SessionRunArgs(
        fetches={KMeansClustering.LOSS_OP_NAME: self._loss_tensor})

  def after_run(self, run_context, run_values):
    loss = run_values.results[KMeansClustering.LOSS_OP_NAME]
    assert loss is not None
    if self._prev_loss is not None:
      relative_change = (abs(loss - self._prev_loss) /
                         (1 + abs(self._prev_loss)))
      if relative_change < self._tolerance:
        run_context.request_stop()
    self._prev_loss = loss

class _InitializeClustersHook(session_run_hook.SessionRunHook):
  """Initializes clusters or waits for cluster initialization."""

  def __init__(self, init_op, is_initialized_op, is_chief):
    self._init_op = init_op
    self._is_chief = is_chief
    self._is_initialized_op = is_initialized_op

  def after_create_session(self, session, _):
    assert self._init_op.graph == ops.get_default_graph()
    assert self._is_initialized_op.graph == self._init_op.graph
    while True:
      try:
        if session.run(self._is_initialized_op):
          break
        elif self._is_chief:
          session.run(self._init_op)
        else:
          time.sleep(1)
      except RuntimeError as e:
        logging.info(e)

def _parse_tensor_or_dict(features):
  """Helper function to parse features."""
  if isinstance(features, dict):
    keys = sorted(features.keys())
    with ops.colocate_with(features[keys[0]]):
      features = array_ops.concat([features[k] for k in keys], 1)
  return features

def _kmeans_clustering_model_fn(features, labels, mode, params, config):
  """Model function for KMeansClustering estimator."""
  assert labels is None, labels
  (all_scores, model_predictions, losses,
   is_initialized, init_op, training_op) = clustering_ops.KMeans(
       _parse_tensor_or_dict(features),
       params.get('num_clusters'),
       initial_clusters=params.get('training_initial_clusters'),
       distance_metric=params.get('distance_metric'),
       use_mini_batch=params.get('use_mini_batch'),
       mini_batch_steps_per_iteration=params.get(
           'mini_batch_steps_per_iteration'),
       random_seed=params.get('random_seed'),
       kmeans_plus_plus_num_retries=params.get(
           'kmeans_plus_plus_num_retries')).training_graph()
  incr_step = state_ops.assign_add(training_util.get_global_step(), 1)
  loss = math_ops.reduce_sum(losses, name=KMeansClustering.LOSS_OP_NAME)
  summary.scalar('loss/raw', loss)
  training_op = with_dependencies([training_op, incr_step], loss)
  predictions = {
      KMeansClustering.ALL_SCORES: all_scores[0],
      KMeansClustering.CLUSTER_IDX: model_predictions[0],
  }
  eval_metric_ops = {KMeansClustering.SCORES: loss}
  training_hooks = [_InitializeClustersHook(
      init_op, is_initialized, config.is_chief)]
  relative_tolerance = params.get('relative_tolerance')
  if relative_tolerance is not None:
    training_hooks.append(_LossRelativeChangeHook(relative_tolerance))
  return ModelFnOps(
      mode=mode,
      predictions=predictions,
      eval_metric_ops=eval_metric_ops,
      loss=loss,
      train_op=training_op,
      training_hooks=training_hooks)

# TODO(agarwal,ands): support sharded input.
class KMeansClustering(estimator.Estimator):
  """An Estimator for K-Means clustering."""
  SQUARED_EUCLIDEAN_DISTANCE = clustering_ops.SQUARED_EUCLIDEAN_DISTANCE
  COSINE_DISTANCE = clustering_ops.COSINE_DISTANCE
  RANDOM_INIT = clustering_ops.RANDOM_INIT
  KMEANS_PLUS_PLUS_INIT = clustering_ops.KMEANS_PLUS_PLUS_INIT
  SCORES = 'scores'
  CLUSTER_IDX = 'cluster_idx'
  CLUSTERS = 'clusters'
  ALL_SCORES = 'all_scores'
  LOSS_OP_NAME = 'kmeans_loss'

  @deprecated(None, _USE_TF_CONTRIB_FACTORIZATION)
  def __init__(self,
               num_clusters,
               model_dir=None,
               initial_clusters=RANDOM_INIT,
               distance_metric=SQUARED_EUCLIDEAN_DISTANCE,
               random_seed=0,
               use_mini_batch=True,
               mini_batch_steps_per_iteration=1,
               kmeans_plus_plus_num_retries=2,
               relative_tolerance=None,
               config=None):
    """Creates a model for running KMeans training and inference.

    Args:
      num_clusters: number of clusters to train.
      model_dir: the directory to save the model results and log files.
      initial_clusters: specifies how to initialize the clusters for training.
        See clustering_ops.kmeans for the possible values.
      distance_metric: the distance metric used for clustering.
        See clustering_ops.kmeans for the possible values.
      random_seed: Python integer. Seed for PRNG used to initialize centers.
      use_mini_batch: If true, use the mini-batch k-means algorithm. Else assume
        full batch.
      mini_batch_steps_per_iteration: number of steps after which the updated
        cluster centers are synced back to a master copy. See clustering_ops.py
        for more details.
      kmeans_plus_plus_num_retries: For each point that is sampled during
        kmeans++ initialization, this parameter specifies the number of
        additional points to draw from the current distribution before selecting
        the best. If a negative value is specified, a heuristic is used to
        sample O(log(num_to_sample)) additional points.
      relative_tolerance: A relative tolerance of change in the loss between
        iterations.  Stops learning if the loss changes less than this amount.
        Note that this may not work correctly if use_mini_batch=True.
      config: See Estimator
    """
    params = {}
    params['num_clusters'] = num_clusters
    params['training_initial_clusters'] = initial_clusters
    params['distance_metric'] = distance_metric
    params['random_seed'] = random_seed
    params['use_mini_batch'] = use_mini_batch
    params['mini_batch_steps_per_iteration'] = mini_batch_steps_per_iteration
    params['kmeans_plus_plus_num_retries'] = kmeans_plus_plus_num_retries
    params['relative_tolerance'] = relative_tolerance
    super(KMeansClustering, self).__init__(
        model_fn=_kmeans_clustering_model_fn,
        params=params,
        model_dir=model_dir,
        config=config)

  @deprecated(None, _USE_TF_CONTRIB_FACTORIZATION)
  def predict_cluster_idx(self, input_fn=None):
    """Yields predicted cluster indices."""
    key = KMeansClustering.CLUSTER_IDX
    results = super(KMeansClustering, self).predict(
        input_fn=input_fn, outputs=[key])
    for result in results:
      yield result[key]

  @deprecated(None, _USE_TF_CONTRIB_FACTORIZATION)
  def score(self, input_fn=None, steps=None):
    """Predict total sum of distances to nearest clusters.

    Note that this function is different from the corresponding one in sklearn
    which returns the negative of the sum of distances.

    Args:
      input_fn: see predict.
      steps: see predict.

    Returns:
      Total sum of distances to nearest clusters.
    """
    return np.sum(
        self.evaluate(
            input_fn=input_fn, steps=steps)[KMeansClustering.SCORES])

  @deprecated(None, _USE_TF_CONTRIB_FACTORIZATION)
  def transform(self, input_fn=None, as_iterable=False):
    """Transforms each element to distances to cluster centers.

    Note that this function is different from the corresponding one in sklearn.
    For SQUARED_EUCLIDEAN distance metric, sklearn transform returns the
    EUCLIDEAN distance, while this function returns the SQUARED_EUCLIDEAN
    distance.

    Args:
      input_fn: see predict.
      as_iterable: see predict

    Returns:
      Array with same number of rows as x, and num_clusters columns, containing
      distances to the cluster centers.
    """
    key = KMeansClustering.ALL_SCORES
    results = super(KMeansClustering, self).predict(
        input_fn=input_fn,
        outputs=[key],
        as_iterable=as_iterable)
    if not as_iterable:
      return results[key]
    else:
      return results

  @deprecated(None, _USE_TF_CONTRIB_FACTORIZATION)
  def clusters(self):
    """Returns cluster centers."""
    return super(KMeansClustering, self).get_variable_value(self.CLUSTERS)


#!/usr/bin/env python
"""Calculate scores for Heinz.

This script transform a p-value into a score:
    1. Use alpha and lambda to calculate a threshold P-value.
    2. Calculate a score based on each P-value by alpha and the threshold.

For more details, please refer to the paper doi:10.1093/bioinformatics/btn161

Input:
    P-values from DESeq2 result: first column: names, second column P-values
Output:
    Scores, which will be used as the input of Heinz.
    First column: names, second column: scores.

Python 3 is required.
"""
# Implemented by: Chao (Cico) Zhang
# Homepage: https://Hi-IT.org
# Date: 14 Mar 2017
# Last modified: 23 May 2018

import argparse
import sys

import numpy as np
import pandas as pd

parser = argparse.ArgumentParser(description='Transform a P-value into a '
                                 'score which can be used as the input of '
                                 'Heinz')
parser.add_argument('-n', '--node', required=True, dest='nodes',
                    metavar='nodes_pvalue.txt', type=str,
                    help='Input file of nodes with P-values')
parser.add_argument('-f', '--fdr', required=True, dest='fdr',
                    metavar='0.007', type=float, help='Choose a value of FDR')
parser.add_argument('-m', '--model', required=False, dest='param_file',
                    metavar='param.txt', type=str,
                    help='A txt file contains model params as input')
parser.add_argument('-a', '--alpha', required=False, dest='alpha',
                    metavar='0.234', type=float, default=0.5,
                    help='Single parameter alpha as input if txt input is '
                    'not provided')
parser.add_argument('-l', '--lambda', required=False, dest='lam',
                    metavar='0.345', type=float, default=0.5,
                    help='Single parameter lambda as input if txt input is '
                    'not provided')
parser.add_argument('-o', '--output', required=True, dest='output',
                    metavar='scores.txt', type=str,
                    help='The output file to store the calculated scores')
args = parser.parse_args()

# Check if the parameters are complete
if args.output is None:
    sys.exit('Output file is not designated.')

if args.nodes is None:
    sys.exit('Nodes with p-values must be provided.')

if args.fdr is None:
    sys.exit('FDR must be provided')

if args.fdr >= 1 or args.fdr <= 0:
    sys.exit('FDR must greater than 0 and smaller than 1')

# run heinz-print according to the input type
if args.param_file is not None:  # if BUM output is provided
    with open(args.param_file) as p:
        params = p.readlines()
        lam = float(params[0])  # Maybe this is a bug
        alpha = float(params[1])  # Maybe this is a bug
# if BUM output is not provided
elif args.alpha is not None and args.lam is not None:
    lam = args.lam
    alpha = args.alpha
else:  # The input is not complete
    sys.exit('The parameters of the model are incomplete.')

# Calculate the threshold P-value
pie = lam + (1 - lam) * alpha
p_threshold = np.power((pie - lam * args.fdr) / (args.fdr - lam * args.fdr),
                       1 / (alpha - 1))
print(p_threshold)
# Calculate the scores
input_pvalues = pd.read_csv(args.nodes, sep='\t', names=['node', 'pvalue'])
input_pvalues.loc[:, 'score'] = input_pvalues.pvalue.apply(lambda x:
                                                           (alpha - 1) * (np.log(x) - np.log(p_threshold)))
# print(input_pvalues.loc[:, ['node', 'score']])
input_pvalues.loc[:, ['node', 'score']].to_csv(args.output, sep='\t',
                                               index=False, header=False)


"""
===================================
Simple 1D Kernel Density Estimation
===================================
This example uses the :class:`sklearn.neighbors.KernelDensity` class to
demonstrate the principles of Kernel Density Estimation in one dimension.

The first plot shows one of the problems with using histograms to visualize
the density of points in 1D. Intuitively, a histogram can be thought of as a
scheme in which a unit "block" is stacked above each point on a regular grid.
As the top two panels show, however, the choice of gridding for these blocks
can lead to wildly divergent ideas about the underlying shape of the density
distribution.  If we instead center each block on the point it represents, we
get the estimate shown in the bottom left panel.  This is a kernel density
estimation with a "top hat" kernel.  This idea can be generalized to other
kernel shapes: the bottom-right panel of the first figure shows a Gaussian
kernel density estimate over the same distribution.

Scikit-learn implements efficient kernel density estimation using either
a Ball Tree or KD Tree structure, through the
:class:`sklearn.neighbors.KernelDensity` estimator.  The available kernels
are shown in the second figure of this example.

The third figure compares kernel density estimates for a distribution of 100
samples in 1 dimension.  Though this example uses 1D distributions, kernel
density estimation is easily and efficiently extensible to higher dimensions
as well.
"""
# Author: Jake Vanderplas <jakevdp@cs.washington.edu>
#
import numpy as np
import matplotlib.pyplot as plt
from scipy.stats import norm
from sklearn.neighbors import KernelDensity

#----------------------------------------------------------------------
# Plot the progression of histograms to kernels
np.random.seed(1)
N = 20
X = np.concatenate((np.random.normal(0, 1, int(0.3 * N)),
                    np.random.normal(5, 1, int(0.7 * N))))[:, np.newaxis]
X_plot = np.linspace(-5, 10, 1000)[:, np.newaxis]
bins = np.linspace(-5, 10, 10)

fig, ax = plt.subplots(2, 2, sharex=True, sharey=True)
fig.subplots_adjust(hspace=0.05, wspace=0.05)

# histogram 1
ax[0, 0].hist(X[:, 0], bins=bins, fc='#AAAAFF', normed=True)
ax[0, 0].text(-3.5, 0.31, "Histogram")

# histogram 2
ax[0, 1].hist(X[:, 0], bins=bins + 0.75, fc='#AAAAFF', normed=True)
ax[0, 1].text(-3.5, 0.31, "Histogram, bins shifted")

# tophat KDE
kde = KernelDensity(kernel='tophat', bandwidth=0.75).fit(X)
log_dens = kde.score_samples(X_plot)
ax[1, 0].fill(X_plot[:, 0], np.exp(log_dens), fc='#AAAAFF')
ax[1, 0].text(-3.5, 0.31, "Tophat Kernel Density")

# Gaussian KDE
kde = KernelDensity(kernel='gaussian', bandwidth=0.75).fit(X)
log_dens = kde.score_samples(X_plot)
ax[1, 1].fill(X_plot[:, 0], np.exp(log_dens), fc='#AAAAFF')
ax[1, 1].text(-3.5, 0.31, "Gaussian Kernel Density")

for axi in ax.ravel():
    axi.plot(X[:, 0], np.zeros(X.shape[0]) - 0.01, '+k')
    axi.set_xlim(-4, 9)
    axi.set_ylim(-0.02, 0.34)

for axi in ax[:, 0]:
    axi.set_ylabel('Normalized Density')

for axi in ax[1, :]:
    axi.set_xlabel('x')

#----------------------------------------------------------------------
# Plot all available kernels
X_plot = np.linspace(-6, 6, 1000)[:, None]
X_src = np.zeros((1, 1))

fig, ax = plt.subplots(2, 3, sharex=True, sharey=True)
fig.subplots_adjust(left=0.05, right=0.95, hspace=0.05, wspace=0.05)

def format_func(x, loc):
    if x == 0:
        return '0'
    elif x == 1:
        return 'h'
    elif x == -1:
        return '-h'
    else:
        return '%ih' % x

for i, kernel in enumerate(['gaussian', 'tophat', 'epanechnikov',
                            'exponential', 'linear', 'cosine']):
    axi = ax.ravel()[i]
    log_dens = KernelDensity(kernel=kernel).fit(X_src).score_samples(X_plot)
    axi.fill(X_plot[:, 0], np.exp(log_dens), '-k', fc='#AAAAFF')
    axi.text(-2.6, 0.95, kernel)

    axi.xaxis.set_major_formatter(plt.FuncFormatter(format_func))
    axi.xaxis.set_major_locator(plt.MultipleLocator(1))
    axi.yaxis.set_major_locator(plt.NullLocator())

    axi.set_ylim(0, 1.05)
    axi.set_xlim(-2.9, 2.9)

ax[0, 1].set_title('Available Kernels')

#----------------------------------------------------------------------
# Plot a 1D density example
N = 100
np.random.seed(1)
X = np.concatenate((np.random.normal(0, 1, int(0.3 * N)),
                    np.random.normal(5, 1, int(0.7 * N))))[:, np.newaxis]

X_plot = np.linspace(-5, 10, 1000)[:, np.newaxis]

true_dens = (0.3 * norm(0, 1).pdf(X_plot[:, 0])
             + 0.7 * norm(5, 1).pdf(X_plot[:, 0]))

fig, ax = plt.subplots()
ax.fill(X_plot[:, 0], true_dens, fc='black', alpha=0.2,
        label='input distribution')

for kernel in ['gaussian', 'tophat', 'epanechnikov']:
    kde = KernelDensity(kernel=kernel, bandwidth=0.5).fit(X)
    log_dens = kde.score_samples(X_plot)
    ax.plot(X_plot[:, 0], np.exp(log_dens), '-',
            label="kernel = '{0}'".format(kernel))

ax.text(6, 0.38, "N={0} points".format(N))

ax.legend(loc='upper left')
ax.plot(X[:, 0], -0.005 - 0.01 * np.random.random(X.shape[0]), '+k')

ax.set_xlim(-4, 9)
ax.set_ylim(-0.02, 0.4)
plt.show()


"""
Benchmark scikit-learn's Ward implement compared to SciPy's
"""

import time

import numpy as np
from scipy.cluster import hierarchy
import pylab as pl

from sklearn.cluster import AgglomerativeClustering

ward = AgglomerativeClustering(n_clusters=3, linkage='ward')

n_samples = np.logspace(.5, 3, 9)
n_features = np.logspace(1, 3.5, 7)
N_samples, N_features = np.meshgrid(n_samples,
                                    n_features)
scikits_time = np.zeros(N_samples.shape)
scipy_time = np.zeros(N_samples.shape)

for i, n in enumerate(n_samples):
    for j, p in enumerate(n_features):
        X = np.random.normal(size=(n, p))
        t0 = time.time()
        ward.fit(X)
        scikits_time[j, i] = time.time() - t0
        t0 = time.time()
        hierarchy.ward(X)
        scipy_time[j, i] = time.time() - t0

ratio = scikits_time / scipy_time

pl.figure("scikit-learn Ward's method benchmark results")
pl.imshow(np.log(ratio), aspect='auto', origin="lower")
pl.colorbar()
pl.contour(ratio, levels=[1, ], colors='k')
pl.yticks(range(len(n_features)), n_features.astype(np.int))
pl.ylabel('N features')
pl.xticks(range(len(n_samples)), n_samples.astype(np.int))
pl.xlabel('N samples')
pl.title("Scikit's time, in units of scipy time (log)")
pl.show()


# coding: utf-8
# python 3.5
import sys
import os
sys.path.append('/Users/ooki/git/research_dr/python/MLEM2')
sys.path.append(os.path.dirname(os.path.abspath("__file__"))+'/../MLEM2')
from sklearn.metrics import accuracy_score
import copy
import importlib
import mlem2
import LERS
importlib.reload(mlem2)
importlib.reload(LERS)
from rules_stat import getNumRulesClass
from rules_stat import getRulesValueCount

# =====================================
# 公正配慮すべき属性list_sをdecision_tableから削除する
# =====================================
def delDiscriminativeAttributes(decision_table, list_s):
    return(decision_table.drop(list_s, axis=1))

# =====================================
# Rules のうち 属性attr / 基本条件 e(属性attrの値v) を含むルールセットの数を返す
# =====================================
def getNumRulesIncludeAttr(list_rules, attr) :
    rules = [r for r in list_rules if attr in r.getKey()]
    return(len(rules))

def getNumRulesIncludeE(list_rules, attr, v) :
    rules = [r for r in list_rules if r.getValue(attr) == v]
    return(len(rules))

def getNumRulesClassIncludeAttr(list_rules, attr, cls) :
    rules = [r for r in list_rules if (attr in r.getKey()) and r.getConsequent() == cls]
    return(len(rules))

def getNumRulesClassIncludeE(list_rules, attr, v, cls) :
    rules = [r for r in list_rules if r.getValue(attr) == v and r.getConsequent() == cls]
    return(len(rules))

def getNumRulesIncludeMultipleE(list_rules,  dict_attribute_value):
    tmp_rules = list_rules
    for attr in dict_attribute_value.keys():
        for v in dict_attribute_value[attr] :
            tmp_rules = [r for r in tmp_rules if r.getValue(attr) == v]
    return(len(tmp_rules))

def getNumRulesClassIncludeMultipleE(list_rules,  dict_attribute_value, cls):
    tmp_rules = list_rules
    for attr in dict_attribute_value.keys():
        for v in dict_attribute_value[attr] :
            tmp_rules = [r for r in tmp_rules if r.getValue(attr) == v and r.getConsequent() == cls]
    return(len(tmp_rules))

# ======================================
# 分割表a, b, c, d を返す
# ======================================
def getContingencyTable(list_rules, dict_attribute_value, CLASSES):
    N = len(list_rules)
    n1 = getNumRulesClass(list_rules, CLASSES["bad"])
    n2 = getNumRulesClass(list_rules, CLASSES["good"])
    a = getNumRulesClassIncludeMultipleE(list_rules, dict_attribute_value, CLASSES["bad"])
    b = n1 - a
    c = getNumRulesClassIncludeMultipleE(list_rules, dict_attribute_value, CLASSES["good"])
    d = n2 - c
    return(a,b,c,d)

# =====================================
# Rules のうち 属性attr / 基本条件 e(属性attrの値v) を含むルールセットを返す
# =====================================
def getRulesIncludeAttr(list_rules, attr) :
    rules = [r for r in list_rules if attr in r.getKey()]
    return(rules)

def getRulesIncludeE(list_rules, attr, v) :
    rules = [r for r in list_rules if r.getValue(attr) == v]
    return(rules)

# =====================================
# Rules のうち 属性attr / 基本条件e を 含まないルールセットを返す
# =====================================
def getRulesExcludeAttr(list_rules, attr) :
    rules = [r for r in list_rules if not attr in r.getKey()]
    return(rules)

def getRulesExcludeE(list_rules, attr, v) :
    rules = [r for r in list_rules if r.getValue(attr) != v]
    return(rules)

# =====================================
# Rules のうち 属性attr / 基本条件e を 削除したルールセットを返す
# Rule の 属性attr / 基本条件　e を削除したルールを返す
# =====================================
def getRulesDelAttr(list_rules, attr) :
    rules = [delAttrFromRule(r, attr) for r in list_rules]
    return(rules)

def getRulesDelE(list_rules, attr, v) :
    rules = [delEFromRule(r, attr, v) for r in list_rules]
    return(rules)

def delAttrFromRule(rule, attr) :
    rule_new = copy.deepcopy(rule)
    rule_new.delKey(attr)
    return(rule_new)

def delEFromRule(rule, attr, v) :
    if rule.getValue(attr) == v : return(delAttrFromRule(rule, attr))
    else : return(rule)

# =====================================
# alpha差別的な Rule を含まないルールセットを返す
# alpha差別的な Rule の 基本条件　e を削除したルールを返す
# =====================================
def getAlphaRulesExcludeE(list_rules, attr, v, decision_table, list_judgeNominal, alpha = 0) :
    rules = [r for r in list_rules if getElift(r, attr, v, decision_table, list_judgeNominal) <= alpha ]
    return(rules)

def getAlphaRulesDelE(list_rules, attr, v, decision_table, list_judgeNominal, alpha = 0) :
    rules = [delEFromAlphaRule(r, attr, v, decision_table, list_judgeNominal, alpha = 0) for r in list_rules]
    return(rules)

def delEFromAlphaRule(rule, attr, v, decision_table, list_judgeNominal, alpha = 0):
    if rule.getValue(attr) == v :
        elift = getElift(rule, attr, v, decision_table, list_judgeNominal)
        if elift > alpha : return(delAttrFromRule(rule, attr))
        else : return(rule)
    else :
        return(rule)

# =====================================
# M差別的な Rule の を含まない / 基本条件　e を削除したルールセットを返す
# =====================================
def getMRulesFUN(list_rules, attr, v, target_cls, DELFUN, m = 0) :
    num_target_cls, num_other_cls, list_num_other_cls = 0, 0, []
    classes = mlem2.getEstimatedClass(list_rules)
    for cls in classes :
        if cls == target_cls :
            num_target_cls = getNumRulesClassIncludeE(list_rules, attr, v, cls)
        else :
            list_num_other_cls.append(getNumRulesClassIncludeE(list_rules, attr, v, cls))
    num_other_cls = sum(list_num_other_cls) / len(list_num_other_cls) #複数クラスの場合を考慮
    if (num_target_cls / (num_target_cls + num_other_cls)) > m : #m保護なら
        return(list_rules)
    else :
        return(DELFUN(list_rules, attr, v))

# =====================================
# 配慮変数sをもつ対象だけの決定表を作る
# =====================================
def createDTSuppoterdbyRule(list_rules, attr, v, cls, decision_table):
    target_indice = []
    target_rules = [r for r in list_rules if r.getValue(attr) == v and r.getConsequent() == cls]
    for rule in target_rules:
        target_indice.extend(rule.getSupport())
    target_indice = list(set(target_indice))
    target_indice = sorted(target_indice)
    new_decision_table = decision_table_train.ix[target_indice]
    new_decision_class = new_decision_table[new_decision_table.columns[-1]].values.tolist()
    return(new_decision_table, new_decision_class)

# 有利な決定クラスのルールを減らす関数 配慮変数sを

# =====================================
# Rule の 配慮変数s での decision_tableにおける　elift
# =====================================
def getElift(rule, attr, v, decision_table, list_judgeNominal):
    supp, conf = LERS.getSupportConfidence(rule, decision_table, list_judgeNominal)
    rule_s = delEFromRule(rule, attr, v)
    supp_s, conf_s = LERS.getSupportConfidence(rule_s, decision_table, list_judgeNominal)
    if conf_s == 0: elift = 999
    else : elift = conf / conf_s
    return(elift)

# =====================================
# Rule の 配慮変数s での decision_tableにおける　slift
# =====================================
def getSlift(rule, s, decision_table, operator):
    conditions = mlem2.getConditionValues(decision_table, s)
    clifts = [getClift(rule, s, c, decision_table) for c in conditions]
    slift = operator(clifts)
    return(slift)

# =====================================
# Rule の 配慮変数s と 代替する変数c での decision_tableにおける　clift
# =====================================
def getClift(rule, s, c, decision_table, list_judgeNominal):
    supp, conf = LERS.getSupportConfidence(rule, decision_table,list_judgeNominal)
    rule_c = mlem2.delEfromRule(rule,s)
    rule_c = rule_c.setValue(s,c)
    supp_c, conf_c = LERS.getSupportConfidence(rule_c, decision_table, list_judgeNominal)
    clift = conf / conf_c
    return(clift)

# ====================================
# Attribute Value dict を stringにして返す
# ====================================
def strAttributeValue(ATTRIBUTE_VALUE) :
    list_string = []
    for i in ATTRIBUTE_VALUE :
        list_string.append(i+"-".join(ATTRIBUTE_VALUE[i]))
    return("+".join(list_string))

# ====================================
# Attribute Value dict を stringにして返す
# ====================================
def getItemSet(rule_value) :
    itemset = set()
    for attr in rule_value :
        itemset.add(attr+"-".join(rule_value[attr]))
    return(itemset)

def jaccard(set1, set2):
    set_and = set1 & set2
    set_or = set1 | set2
    if len(set_or) == 0 :
        return(0)
    else :
        return(len(set_and)/len(set_or))

# ========================================
# main
# ========================================
if __name__ == "__main__":

    # 設定
    DIR_UCI = '/mnt/data/uci/'
    FILENAME = 'german_credit_categorical'
    iter1 = 1
    iter2 = 1

    # rule induction
    rules = mlem2.getRulesByMLEM2(FILENAME, iter1, iter2)

    # test data
    filepath = DIR_UCI+FILENAME+'/'+FILENAME+'-test'+str(iter1)+'-'+str(iter2)+'.tsv'
    decision_table_test = mlem2.getDecisionTable(filepath)
    decision_table_test = decision_table_test.dropna()
    decision_class = decision_table_test[decision_table_test.columns[-1]].values.tolist()

    # nominal data
    filepath = DIR_UCI+FILENAME+'/'+FILENAME+'.nominal'
    list_nominal = mlem2.getNominalList(filepath)
    list_judgeNominal = mlem2.getJudgeNominal(decision_table_test, list_nominal)

    # predict by LERS
    predictions = LERS.predictByLERS(rules, decision_table_test, list_judgeNominal)

    # 正答率を求める
    accuracy_score(decision_class, predictions)

    # rules の数を求める
    num = len(rules)
    # 平均の長さを求める
    mean_length = mlem2.getMeanLength(rules)

    # train data setup
    decision_table_train, decision_class = getData(FILENAME, iter1, iter2, T = "train")
    list_judgeNominal = getJudgeNominal(decision_table_train, FILENAME)

    # 平均支持度と平均確信度を求める
    mean_support, mean_conf = LERS.getSupportConfidenceRules(rules, decision_table_train, list_judgeNominal)
    # AccとRecallを求める
    acc_recall = LERS.getAccurayRecall(rules, decision_table_train, list_judgeNominal)
    for i,c in enumerate(mlem2.getEstimatedClass(rules)):
        print(str(acc_recall[i][0])+","+str(acc_recall[i][1]))

    ###### 公正配慮のテスト

    # 基本条件を含むルールセット
    rules_sex_2 = mlem2.getRulesIncludeE(rules, "Sex_Marital_Status", "2.0")
    rules_sex_4 = mlem2.getRulesIncludeE(rules, "Sex_Marital_Status", "4.0")
    # 条件を含まないルールセット
    rules_exclude_sex = mlem2.getRulesExcludeAttr(rules, "Sex_Marital_Status")
    # 基本条件を含まないルールセット
    rules_exclude_sex_1 = mlem2.getRulesExcludeE(rules, "Sex_Marital_Status", "1.0")
    # 条件を削除したルールセット
    rules_del_value = mlem2.getRulesDelAttr(rules, "Value_Savings_Stocks")
    # 基本条件を削除したルールセット
    rules_del_value_1 = mlem2.getRulesDelE(rules, "Value_Savings_Stocks", "1.0")

    # 条件を1つ削除する例
    rule = mlem2.delAttrFromRule(rules[12],'No_of_dependents')
    rule = mlem2.delAttrFromRule(rules[12],'Concurrent_Credits')

    # ====

    # read data
    filepath = '/mnt/data/uci/'+FILENAME+'/'+FILENAME+'-train'+str(iter1)+'-'+str(iter2)+'.tsv'
    decision_table = mlem2.getDecisionTable(filepath)
    decision_table = decision_table.dropna()
    decision_table.index = range(decision_table.shape[0])

    # read nominal
    filepath = '/mnt/data/uci/'+'/'+FILENAME+'/'+FILENAME+'.nominal'
    list_nominal = mlem2.getNominalList(filepath)

    # ルールを満たすやつ ほとんどないな。。
    match_objects = decision_table.apply(lambda obj: isExplainRule(obj, rules[12], list_judgeNominal), axis=1)

    # confidence
    getConfidence(rule, decision_table, list_judgeNominal)

    rules_sex_2 = mlem2.getRulesIncludeE(rules, "Sex_Marital_Status","2.0")