# SPDX-License-Identifier: Apache-2.0
#
# Copyright (C) 2015, ARM Limited and contributors.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

""" Frequency Analysis Module """

import matplotlib.gridspec as gridspec
import matplotlib.pyplot as plt
import pandas as pd
import pylab as pl
import operator
from trappy.utils import listify
from devlib.utils.misc import memoized
from collections import namedtuple

from analysis_module import AnalysisModule

# Configure logging
import logging

NON_IDLE_STATE = 4294967295

ResidencyTime = namedtuple('ResidencyTime', ['total', 'active'])
ResidencyData = namedtuple('ResidencyData', ['label', 'residency'])

class FrequencyAnalysis(AnalysisModule):
    """
    Support for plotting Frequency Analysis data

    :param trace: input Trace object
    :type trace: :mod:`libs.utils.Trace`
    """

    def __init__(self, trace):
        super(FrequencyAnalysis, self).__init__(trace)

###############################################################################
# DataFrame Getter Methods
###############################################################################

    def _dfg_cpu_frequency_residency(self, cpu, total=True):
        """
        Get per-CPU frequency residency, i.e. amount of
        time CPU `cpu` spent at each frequency.

        :param cpu: CPU ID
        :type cpu: int

        :param total: if true returns the "total" time, otherwise the "active"
                      time is returned
        :type total: bool

        :returns: :mod:`pandas.DataFrame` - "total" or "active" time residency
                  at each frequency.
        """
        residency = self._getCPUFrequencyResidency(cpu)
        if not residency:
            return None
        if total:
            return residency.total
        return residency.active

    def _dfg_cluster_frequency_residency(self, cluster, total=True):
        """
        Get per-Cluster frequency residency, i.e. amount of time CLUSTER
        `cluster` spent at each frequency.

        :param cluster: this can be either a single CPU ID or a list of CPU IDs
            belonging to a cluster or the cluster name as specified in the
            platform description
        :type cluster: str or int or list(int)

        :param total: if true returns the "total" time, otherwise the "active"
                      time is returned
        :type total: bool

        :returns: :mod:`pandas.DataFrame` - "total" or "active" time residency
                  at each frequency.
        """
        residency = self._getClusterFrequencyResidency(cluster)
        if not residency:
            return None
        if total:
            return residency.total
        return residency.active

###############################################################################
# Plotting Methods
###############################################################################

    def plotClusterFrequencies(self, title='Clusters Frequencies'):
        """
        Plot frequency trend for all clusters. If sched_overutilized events are
        available, the plots will also show the intervals of time where the
        cluster was overutilized.

        :param title: user-defined plot title
        :type title: str
        """
        if not self._trace.hasEvents('cpu_frequency'):
            logging.warn('Events [cpu_frequency] not found, plot DISABLED!')
            return
        df = self._dfg_trace_event('cpu_frequency')

        pd.options.mode.chained_assignment = None

        # Extract LITTLE and big clusters frequencies
        # and scale them to [MHz]
        if len(self._platform['clusters']['little']):
            lfreq = df[df.cpu == self._platform['clusters']['little'][-1]]
            lfreq['frequency'] = lfreq['frequency']/1e3
        else:
            lfreq = []
        if len(self._platform['clusters']['big']):
            bfreq = df[df.cpu == self._platform['clusters']['big'][-1]]
            bfreq['frequency'] = bfreq['frequency']/1e3
        else:
            bfreq = []

        # Compute AVG frequency for LITTLE cluster
        avg_lfreq = 0
        if len(lfreq) > 0:
            lfreq['timestamp'] = lfreq.index
            lfreq['delta'] = (lfreq['timestamp'] -lfreq['timestamp'].shift()).fillna(0).shift(-1)
            lfreq['cfreq'] = (lfreq['frequency'] * lfreq['delta']).fillna(0)
            timespan = lfreq.iloc[-1].timestamp - lfreq.iloc[0].timestamp
            avg_lfreq = lfreq['cfreq'].sum()/timespan

        # Compute AVG frequency for big cluster
        avg_bfreq = 0
        if len(bfreq) > 0:
            bfreq['timestamp'] = bfreq.index
            bfreq['delta'] = (bfreq['timestamp'] - bfreq['timestamp'].shift()).fillna(0).shift(-1)
            bfreq['cfreq'] = (bfreq['frequency'] * bfreq['delta']).fillna(0)
            timespan = bfreq.iloc[-1].timestamp - bfreq.iloc[0].timestamp
            avg_bfreq = bfreq['cfreq'].sum()/timespan

        pd.options.mode.chained_assignment = 'warn'

        # Setup a dual cluster plot
        fig, pltaxes = plt.subplots(2, 1, figsize=(16, 8))
        plt.suptitle(title, y=.97, fontsize=16, horizontalalignment='center')

        # Plot Cluster frequencies
        axes = pltaxes[0]
        axes.set_title('big Cluster')
        if avg_bfreq > 0:
            axes.axhline(avg_bfreq, color='r', linestyle='--', linewidth=2)
        axes.set_ylim(
                (self._platform['freqs']['big'][0] - 100000)/1e3,
                (self._platform['freqs']['big'][-1] + 100000)/1e3
        )
        if len(bfreq) > 0:
            bfreq['frequency'].plot(style=['r-'], ax=axes,
                                    drawstyle='steps-post', alpha=0.4)
        else:
            logging.warn('NO big CPUs frequency events to plot')
        axes.set_xlim(self._trace.x_min, self._trace.x_max)
        axes.set_ylabel('MHz')
        axes.grid(True)
        axes.set_xticklabels([])
        axes.set_xlabel('')
        self._trace.analysis.status.plotOverutilized(axes)

        axes = pltaxes[1]
        axes.set_title('LITTLE Cluster')
        if avg_lfreq > 0:
            axes.axhline(avg_lfreq, color='b', linestyle='--', linewidth=2)
        axes.set_ylim(
                (self._platform['freqs']['little'][0] - 100000)/1e3,
                (self._platform['freqs']['little'][-1] + 100000)/1e3
        )
        if len(lfreq) > 0:
            lfreq['frequency'].plot(style=['b-'], ax=axes,
                                    drawstyle='steps-post', alpha=0.4)
        else:
            logging.warn('NO LITTLE CPUs frequency events to plot')
        axes.set_xlim(self._trace.x_min, self._trace.x_max)
        axes.set_ylabel('MHz')
        axes.grid(True)
        self._trace.analysis.status.plotOverutilized(axes)

        # Save generated plots into datadir
        figname = '{}/{}cluster_freqs.png'\
                  .format(self._trace.plots_dir, self._trace.plots_prefix)
        pl.savefig(figname, bbox_inches='tight')

        logging.info('LITTLE cluster average frequency: %.3f GHz',
                     avg_lfreq/1e3)
        logging.info('big    cluster average frequency: %.3f GHz',
                     avg_bfreq/1e3)

        return (avg_lfreq/1e3, avg_bfreq/1e3)

    def plotCPUFrequencyResidency(self, cpus=None, pct=False, active=False):
        """
        Plot per-CPU frequency residency. big CPUs are plotted first and then
        LITTLEs.

        Requires the following trace events:
            - cpu_frequency
            - cpu_idle

        :param cpus: List of cpus. By default plot all CPUs
        :type cpus: list(str)

        :param pct: plot residencies in percentage
        :type pct: bool

        :param active: for percentage plot specify whether to plot active or
            total time. Default is TOTAL time
        :type active: bool
        """
        if not self._trace.hasEvents('cpu_frequency'):
            logging.warn('Events [cpu_frequency] not found, plot DISABLED!')
            return
        if not self._trace.hasEvents('cpu_idle'):
            logging.warn('Events [cpu_idle] not found, plot DISABLED!')
            return

        if cpus is None:
            # Generate plots only for available CPUs
            cpufreq_data = self._dfg_trace_event('cpu_frequency')
            _cpus = range(cpufreq_data.cpu.max()+1)
        else:
            _cpus = listify(cpus)

        # Split between big and LITTLE CPUs ordered from higher to lower ID
        _cpus.reverse()
        big_cpus = [c for c in _cpus if c in self._platform['clusters']['big']]
        little_cpus = [c for c in _cpus if c in
                       self._platform['clusters']['little']]
        _cpus = big_cpus + little_cpus

        # Precompute active and total time for each CPU
        residencies = []
        xmax = 0.0
        for cpu in _cpus:
            res = self._getCPUFrequencyResidency(cpu)
            residencies.append(ResidencyData('CPU{}'.format(cpu), res))

            max_time = res.total.max().values[0]
            if xmax < max_time:
                xmax = max_time

        self._plotFrequencyResidency(residencies, 'cpu', xmax, pct, active)

    def plotClusterFrequencyResidency(self, clusters=None,
                                      pct=False, active=False):
        """
        Plot the frequency residency in a given cluster, i.e. the amount of
        time cluster `cluster` spent at frequency `f_i`. By default, both 'big'
        and 'LITTLE' clusters data are plotted.

        Requires the following trace events:
            - cpu_frequency
            - cpu_idle

        :param clusters: name of the clusters to be plotted (all of them by
            default)
        :type clusters: str ot list(str)

        :param pct: plot residencies in percentage
        :type pct: bool

        :param active: for percentage plot specify whether to plot active or
            total time. Default is TOTAL time
        :type active: bool
        """
        if not self._trace.hasEvents('cpu_frequency'):
            logging.warn('Events [cpu_frequency] not found, plot DISABLED!')
            return
        if not self._trace.hasEvents('cpu_idle'):
            logging.warn('Events [cpu_idle] not found, plot DISABLED!')
            return

        # Assumption: all CPUs in a cluster run at the same frequency, i.e. the
        # frequency is scaled per-cluster not per-CPU. Hence, we can limit the
        # cluster frequencies data to a single CPU
        if not self._trace.freq_coherency:
            logging.warn('Cluster frequency is not coherent, plot DISABLED!')
            return

        # Sanitize clusters
        if clusters is None:
            _clusters = self._platform['clusters'].keys()
        else:
            _clusters = listify(clusters)

        # Precompute active and total time for each cluster
        residencies = []
        xmax = 0.0
        for cluster in _clusters:
            res = self._getClusterFrequencyResidency(
                self._platform['clusters'][cluster.lower()])
            residencies.append(ResidencyData('{} Cluster'.format(cluster),
                                             res))

            max_time = res.total.max().values[0]
            if xmax < max_time:
                xmax = max_time

        self._plotFrequencyResidency(residencies, 'cluster', xmax, pct, active)

###############################################################################
# Utility Methods
###############################################################################

    @memoized
    def _getCPUActiveSignal(self, cpu):
        """
        Build a square wave representing the active (i.e. non-idle) CPU time,
        i.e.:
            cpu_active[t] == 1 if at least one CPU is reported to be
                               non-idle by CPUFreq at time t
            cpu_active[t] == 0 otherwise

        :param cpu: CPU ID
        :type cpu: int
        """
        if not self._trace.hasEvents('cpu_idle'):
            logging.warn('Events [cpu_idle] not found, '
                         'cannot compute CPU active signal!')
            return None

        idle_df = self._dfg_trace_event('cpu_idle')
        cpu_df = idle_df[idle_df.cpu_id == cpu]

        cpu_active = cpu_df.state.apply(
            lambda s: 1 if s == NON_IDLE_STATE else 0
        )

        start_time = 0.0
        if not self._trace.ftrace.normalized_time:
            start_time = self._trace.ftrace.basetime
        if cpu_active.index[0] != start_time:
            entry_0 = pd.Series(cpu_active.iloc[0] ^ 1, index=[start_time])
            cpu_active = pd.concat([entry_0, cpu_active])

        return cpu_active

    @memoized
    def _getClusterActiveSignal(self, cluster):
        """
        Build a square wave representing the active (i.e. non-idle) cluster
        time, i.e.:
            cluster_active[t] == 1 if at least one CPU is reported to be
                                   non-idle by CPUFreq at time t
            cluster_active[t] == 0 otherwise

        :param cluster: list of CPU IDs belonging to a cluster
        :type cluster: list(int)
        """
        cpu_active = {}
        for cpu in cluster:
            cpu_active[cpu] = self._getCPUActiveSignal(cpu)

        active = pd.DataFrame(cpu_active)
        active.fillna(method='ffill', inplace=True)

        # Cluster active is the OR between the actives on each CPU
        # belonging to that specific cluster
        cluster_active = reduce(
            operator.or_,
            [cpu_active.astype(int) for _, cpu_active in
             active.iteritems()]
        )

        return cluster_active

    @memoized
    def _getClusterFrequencyResidency(self, cluster):
        """
        Get a DataFrame with per cluster frequency residency, i.e. amount of
        time spent at a given frequency in each cluster.

        :param cluster: this can be either a single CPU ID or a list of CPU IDs
            belonging to a cluster or the cluster name as specified in the
            platform description
        :type cluster: str or int or list(int)

        :returns: namedtuple(ResidencyTime) - tuple of total and active time
            dataframes

        :raises: KeyError
        """
        if not self._trace.hasEvents('cpu_frequency'):
            logging.warn('Events [cpu_frequency] not found, '
                         'frequency residency computation not possible!')
            return None
        if not self._trace.hasEvents('cpu_idle'):
            logging.warn('Events [cpu_idle] not found, '
                         'frequency residency computation not possible!')
            return None

        if isinstance(cluster, str):
            try:
                _cluster = self._platform['clusters'][cluster.lower()]
            except KeyError:
                logging.warn('%s cluster not found!', cluster)
                return None
        else:
            _cluster = listify(cluster)

        freq_df = self._dfg_trace_event('cpu_frequency')
        # Assumption: all CPUs in a cluster run at the same frequency, i.e. the
        # frequency is scaled per-cluster not per-CPU. Hence, we can limit the
        # cluster frequencies data to a single CPU. This assumption is verified
        # by the Trace module when parsing the trace.
        if len(_cluster) > 1 and not self._trace.freq_coherency:
            logging.warn('Cluster frequency is NOT coherent,'
                         'cannot compute residency!')
            return None
        cluster_freqs = freq_df[freq_df.cpu == _cluster[0]]

        # Compute TOTAL Time
        time_intervals = cluster_freqs.index[1:] - cluster_freqs.index[:-1]
        total_time = pd.DataFrame({
            'time': time_intervals,
            'frequency': [f/1000.0 for f in cluster_freqs.iloc[:-1].frequency]
        })
        total_time = total_time.groupby(['frequency']).sum()

        # Compute ACTIVE Time
        cluster_active = self._getClusterActiveSignal(_cluster)

        # In order to compute the active time spent at each frequency we
        # multiply 2 square waves:
        # - cluster_active, a square wave of the form:
        #     cluster_active[t] == 1 if at least one CPU is reported to be
        #                            non-idle by CPUFreq at time t
        #     cluster_active[t] == 0 otherwise
        # - freq_active, square wave of the form:
        #     freq_active[t] == 1 if at time t the frequency is f
        #     freq_active[t] == 0 otherwise
        available_freqs = sorted(cluster_freqs.frequency.unique())
        new_idx = sorted(cluster_freqs.index.tolist() +
                         cluster_active.index.tolist())
        cluster_freqs = cluster_freqs.reindex(new_idx, method='ffill')
        cluster_active = cluster_active.reindex(new_idx, method='ffill')
        nonidle_time = []
        for f in available_freqs:
            freq_active = cluster_freqs.frequency.apply(
                lambda x: 1 if x == f else 0
            )
            active_t = cluster_active * freq_active
            # Compute total time by integrating the square wave
            nonidle_time.append(self._trace.integrate_square_wave(active_t))

        active_time = pd.DataFrame({'time': nonidle_time},
                                   index=[f/1000.0 for f in available_freqs])
        active_time.index.name = 'frequency'
        return ResidencyTime(total_time, active_time)

    def _getCPUFrequencyResidency(self, cpu):
        """
        Get a DataFrame with per-CPU frequency residency, i.e. amount of
        time CPU `cpu` spent at each frequency. Both total and active times
        will be computed.

        :param cpu: CPU ID
        :type cpu: int

        :returns: namedtuple(ResidencyTime) - tuple of total and active time
            dataframes
        """
        return self._getClusterFrequencyResidency(cpu)

    def _plotFrequencyResidencyAbs(self, axes, residency, n_plots,
                                   is_first, is_last, xmax, title=''):
        """
        Private method to generate frequency residency plots.

        :param axes: axes over which to generate the plot
        :type axes: matplotlib.axes.Axes

        :param residency: tuple of total and active time dataframes
        :type residency: namedtuple(ResidencyTime)

        :param n_plots: total number of plots
        :type n_plots: int

        :param is_first: if True this is the first plot
        :type is_first: bool

        :param is_last: if True this is the last plot
        :type is_last: bool

        :param xmax: x-axes higher bound
        :param xmax: double

        :param title: title of this subplot
        :type title: str
        """
        yrange = 0.4 * max(6, len(residency.total)) * n_plots
        residency.total.plot.barh(ax=axes, color='g',
                                  legend=False, figsize=(16, yrange))
        residency.active.plot.barh(ax=axes, color='r',
                                   legend=False, figsize=(16, yrange))

        axes.set_xlim(0, 1.05*xmax)
        axes.set_ylabel('Frequency [MHz]')
        axes.set_title(title)
        axes.grid(True)
        if is_last:
            axes.set_xlabel('Time [s]')
        else:
            axes.set_xticklabels([])

        if is_first:
            # Put title on top of the figure. As of now there is no clean way
            # to make the title appear always in the same position in the
            # figure because figure heights may vary between different
            # platforms (different number of OPPs). Hence, we use annotation
            legend_y = axes.get_ylim()[1]
            axes.annotate('OPP Residency Time', xy=(0, legend_y),
                          xytext=(-50, 45), textcoords='offset points',
                          fontsize=18)
            axes.annotate('GREEN: Total', xy=(0, legend_y),
                          xytext=(-50, 25), textcoords='offset points',
                          color='g', fontsize=14)
            axes.annotate('RED: Active', xy=(0, legend_y),
                          xytext=(50, 25), textcoords='offset points',
                          color='r', fontsize=14)

    def _plotFrequencyResidencyPct(self, axes, residency_df, label,
                                   n_plots, is_first, is_last, res_type):
        """
        Private method to generate PERCENTAGE frequency residency plots.

        :param axes: axes over which to generate the plot
        :type axes: matplotlib.axes.Axes

        :param residency_df: residency time dataframe
        :type residency_df: :mod:`pandas.DataFrame`

        :param label: label to be used for percentage residency dataframe
        :type label: str

        :param n_plots: total number of plots
        :type n_plots: int

        :param is_first: if True this is the first plot
        :type is_first: bool

        :param is_first: if True this is the last plot
        :type is_first: bool

        :param res_type: type of residency, either TOTAL or ACTIVE
        :type title: str
        """
        # Compute sum of the time intervals
        duration = residency_df.time.sum()
        residency_pct = pd.DataFrame(
            {label: residency_df.time.apply(lambda x: x*100/duration)},
            index=residency_df.index
        )
        yrange = 3 * n_plots
        residency_pct.T.plot.barh(ax=axes, stacked=True, figsize=(16, yrange))

        axes.legend(loc='lower center', ncol=7)
        axes.set_xlim(0, 100)
        axes.grid(True)
        if is_last:
            axes.set_xlabel('Residency [%]')
        else:
            axes.set_xticklabels([])
        if is_first:
            legend_y = axes.get_ylim()[1]
            axes.annotate('OPP {} Residency Time'.format(res_type),
                          xy=(0, legend_y), xytext=(-50, 35),
                          textcoords='offset points', fontsize=18)

    def _plotFrequencyResidency(self, residencies, entity_name, xmax,
                                pct, active):
        """
        Generate Frequency residency plots for the given entities.

        :param residencies:
        :type residencies: namedtuple(ResidencyData) - tuple containing:
            1) as first element, a label to be used as subplot title
            2) as second element, a namedtuple(ResidencyTime)

        :param entity_name: name of the entity ('cpu' or 'cluster') used in the
            figure name
        :type entity_name: str

        :param xmax: upper bound of x-axes
        :type xmax: double

        :param pct: plot residencies in percentage
        :type pct: bool

        :param active: for percentage plot specify whether to plot active or
            total time. Default is TOTAL time
        :type active: bool
        """
        n_plots = len(residencies)
        gs = gridspec.GridSpec(n_plots, 1)
        fig = plt.figure()

        figtype = ""
        for idx, data in enumerate(residencies):
            if data.residency is None:
                plt.close(fig)
                return

            axes = fig.add_subplot(gs[idx])
            is_first = idx == 0
            is_last = idx+1 == n_plots
            if pct and active:
                self._plotFrequencyResidencyPct(axes, data.residency.active,
                                                data.label, n_plots,
                                                is_first, is_last,
                                                'ACTIVE')
                figtype = "_pct_active"
                continue
            if pct:
                self._plotFrequencyResidencyPct(axes, data.residency.total,
                                                data.label, n_plots,
                                                is_first, is_last,
                                                'TOTAL')
                figtype = "_pct_total"
                continue

            self._plotFrequencyResidencyAbs(axes, data.residency,
                                            n_plots, is_first,
                                            is_last, xmax,
                                            title=data.label)

        figname = '{}/{}{}_freq_residency{}.png'\
                  .format(self._trace.plots_dir,
                          self._trace.plots_prefix,
                          entity_name, figtype)
        pl.savefig(figname, bbox_inches='tight')

# vim :set tabstop=4 shiftwidth=4 expandtab


#!/usr/bin/env python
# -*- coding: UTF-8 -*-
'''Produce a custom twist drill plot'''

import numpy as np
import matplotlib as mpl
import matplotlib.pyplot as plt
plt.rc('text', usetex=True)

# set some rcParams
mpl.rcParams['font.weight'] = 'bold'

mpl.rcParams['xtick.major.pad'] = 10
mpl.rcParams['xtick.direction'] = 'inout'
mpl.rcParams['xtick.labelsize'] = 26

mpl.rcParams['ytick.direction'] = 'inout'
mpl.rcParams['ytick.labelsize'] = 20

# define the constants for our chart
materials = [
             ('Acrylic'    , 650 , 'c'          , '-' ) ,
             ('Aluminum'   , 300 , 'b'          , '-' ) ,
             ('Brass'      , 200 , 'g'          , '-' ) ,
             ('LC Steel'   , 110 , 'k'          , '-' ) ,
             ('Wood'       , 100 , 'brown'      , '-' ) ,
             ('MC Steel'   , 80  , 'darkgray'   , '-' ) ,
             ('HC Steel'   , 60  , 'lightgray'  , '-' ) ,
             ('Stainless'  , 50  , 'purple'     , '-' ) ,
             ]
drill_speeds = [250, 340, 390, 510, 600, 650, 990, 1550, 1620, 1900, 2620, 3100] #rpm

speed_lims = (200., 4000.) # rpm

max_in = 1. # in.
incr = 1./16. # in.

im_sz = 25. # in.
ratio = 8.5/11.

fig = plt.figure(figsize=(im_sz,ratio * im_sz), dpi=600)
fig.patch.set_alpha(0)

# generate a vector of drill bit diameter
x = np.array([float(i) * incr for i in range(1,int(max_in/incr) + 1)]) # in.

# calculate the drill speed curve for each material type and plot the curve
for name, speed, color, linestyle in materials:
    plt.loglog(x, 12/np.pi/x*speed, label=name, linewidth=5, color=color, linestyle=linestyle)

ax = plt.gca()

# adjust the axis tick locators to match drill press speeds
ax.yaxis.set_major_locator(mpl.ticker.FixedLocator(drill_speeds))
ax.yaxis.set_major_formatter(mpl.ticker.FormatStrFormatter('%4d'))
ax.yaxis.set_minor_locator(mpl.ticker.NullLocator())
ax.set_ylim(speed_lims)

# set the drill diameter locators and format the ticks with LaTeX
ax.xaxis.set_major_locator(mpl.ticker.MultipleLocator(base=incr))
ax.xaxis.set_minor_locator(mpl.ticker.NullLocator())
ax.set_xlim((incr, max_in))
ticks = ['0', r'$$\frac{1}{16}$$'  , r'$$\frac{1}{8}$$'  , r'$$\frac{3}{16}$$'  , r'$$\frac{1}{4}$$' ,
              r'$$\frac{5}{16}$$'  , r'$$\frac{3}{8}$$'  , r'$$\frac{7}{16}$$'  , r'$$\frac{1}{2}$$' ,
              r'$$\frac{9}{16}$$'  , r'$$\frac{5}{8}$$'  , r'$$\frac{11}{16}$$' , r'$$\frac{3}{4}$$' ,
              r'$$\frac{13}{16}$$' , r'$$\frac{7}{8}$$'  , r'$$\frac{15}{16}$$' , r'$$1$$'     ]
ax.xaxis.set_ticklabels(ticks)

# Add the Texts
plt.xlabel('Bit Diameter (in.)', fontsize=26)
plt.ylabel('Drill Speed (rpm)' , fontsize=26)
plt.title('Twist Drill Speeds' , fontsize=50)
plt.legend(ncol=2, loc=3, fontsize=40)

plt.grid('on')
plt.savefig('drill_speed_chart.png')


#!/usr/bin/python
# -*- coding: utf-8 -*-

"""
=========================================================
PCA example with Iris Data-set
=========================================================

Principal Component Analysis applied to the Iris dataset.

See `here <https://en.wikipedia.org/wiki/Iris_flower_data_set>`_ for more
information on this dataset.

"""
print(__doc__)

# Code source: Gaël Varoquaux
# License: BSD 3 clause

import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D

from sklearn import decomposition
from sklearn import datasets

np.random.seed(5)

centers = [[1, 1], [-1, -1], [1, -1]]
iris = datasets.load_iris()
X = iris.data
y = iris.target

fig = plt.figure(1, figsize=(4, 3))
plt.clf()
ax = Axes3D(fig, rect=[0, 0, .95, 1], elev=48, azim=134)

plt.cla()
pca = decomposition.PCA(n_components=3)
pca.fit(X)
X = pca.transform(X)

for name, label in [('Setosa', 0), ('Versicolour', 1), ('Virginica', 2)]:
    ax.text3D(X[y == label, 0].mean(),
              X[y == label, 1].mean() + 1.5,
              X[y == label, 2].mean(), name,
              horizontalalignment='center',
              bbox=dict(alpha=.5, edgecolor='w', facecolor='w'))
# Reorder the labels to have colors matching the cluster results
y = np.choose(y, [1, 2, 0]).astype(np.float)
ax.scatter(X[:, 0], X[:, 1], X[:, 2], c=y, cmap=plt.cm.spectral,
           edgecolor='k')

ax.w_xaxis.set_ticklabels([])
ax.w_yaxis.set_ticklabels([])
ax.w_zaxis.set_ticklabels([])

plt.show()


"""
====================================
Outlier detection on a real data set
====================================

This example illustrates the need for robust covariance estimation
on a real data set. It is useful both for outlier detection and for
a better understanding of the data structure.

We selected two sets of two variables from the Boston housing data set
as an illustration of what kind of analysis can be done with several
outlier detection tools. For the purpose of visualization, we are working
with two-dimensional examples, but one should be aware that things are
not so trivial in high-dimension, as it will be pointed out.

In both examples below, the main result is that the empirical covariance
estimate, as a non-robust one, is highly influenced by the heterogeneous
structure of the observations. Although the robust covariance estimate is
able to focus on the main mode of the data distribution, it sticks to the
assumption that the data should be Gaussian distributed, yielding some biased
estimation of the data structure, but yet accurate to some extent.
The One-Class SVM algorithm

First example
-------------
The first example illustrates how robust covariance estimation can help
concentrating on a relevant cluster when another one exists. Here, many
observations are confounded into one and break down the empirical covariance
estimation.
Of course, some screening tools would have pointed out the presence of two
clusters (Support Vector Machines, Gaussian Mixture Models, univariate
outlier detection, ...). But had it been a high-dimensional example, none
of these could be applied that easily.

Second example
--------------
The second example shows the ability of the Minimum Covariance Determinant
robust estimator of covariance to concentrate on the main mode of the data
distribution: the location seems to be well estimated, although the covariance
is hard to estimate due to the banana-shaped distribution. Anyway, we can
get rid of some outlying observations.
The One-Class SVM is able to capture the real data structure, but the
difficulty is to adjust its kernel bandwidth parameter so as to obtain
a good compromise between the shape of the data scatter matrix and the
risk of over-fitting the data.

"""
print(__doc__)

# Author: Virgile Fritsch <virgile.fritsch@inria.fr>
# License: BSD 3 clause

import numpy as np
from sklearn.covariance import EllipticEnvelope
from sklearn.svm import OneClassSVM
import matplotlib.pyplot as plt
import matplotlib.font_manager
from sklearn.datasets import load_boston

# Get data
X1 = load_boston()['data'][:, [8, 10]]  # two clusters
X2 = load_boston()['data'][:, [5, 12]]  # "banana"-shaped

# Define "classifiers" to be used
classifiers = {
    "Empirical Covariance": EllipticEnvelope(support_fraction=1.,
                                             contamination=0.261),
    "Robust Covariance (Minimum Covariance Determinant)":
    EllipticEnvelope(contamination=0.261),
    "OCSVM": OneClassSVM(nu=0.261, gamma=0.05)}
colors = ['m', 'g', 'b']
legend1 = {}
legend2 = {}

# Learn a frontier for outlier detection with several classifiers
xx1, yy1 = np.meshgrid(np.linspace(-8, 28, 500), np.linspace(3, 40, 500))
xx2, yy2 = np.meshgrid(np.linspace(3, 10, 500), np.linspace(-5, 45, 500))
for i, (clf_name, clf) in enumerate(classifiers.items()):
    plt.figure(1)
    clf.fit(X1)
    Z1 = clf.decision_function(np.c_[xx1.ravel(), yy1.ravel()])
    Z1 = Z1.reshape(xx1.shape)
    legend1[clf_name] = plt.contour(
        xx1, yy1, Z1, levels=[0], linewidths=2, colors=colors[i])
    plt.figure(2)
    clf.fit(X2)
    Z2 = clf.decision_function(np.c_[xx2.ravel(), yy2.ravel()])
    Z2 = Z2.reshape(xx2.shape)
    legend2[clf_name] = plt.contour(
        xx2, yy2, Z2, levels=[0], linewidths=2, colors=colors[i])

legend1_values_list = list( legend1.values() )
legend1_keys_list = list( legend1.keys() )

# Plot the results (= shape of the data points cloud)
plt.figure(1)  # two clusters
plt.title("Outlier detection on a real data set (boston housing)")
plt.scatter(X1[:, 0], X1[:, 1], color='black')
bbox_args = dict(boxstyle="round", fc="0.8")
arrow_args = dict(arrowstyle="->")
plt.annotate("several confounded points", xy=(24, 19),
             xycoords="data", textcoords="data",
             xytext=(13, 10), bbox=bbox_args, arrowprops=arrow_args)
plt.xlim((xx1.min(), xx1.max()))
plt.ylim((yy1.min(), yy1.max()))
plt.legend((legend1_values_list[0].collections[0],
            legend1_values_list[1].collections[0],
            legend1_values_list[2].collections[0]),
           (legend1_keys_list[0], legend1_keys_list[1], legend1_keys_list[2]),
           loc="upper center",
           prop=matplotlib.font_manager.FontProperties(size=12))
plt.ylabel("accessibility to radial highways")
plt.xlabel("pupil-teacher ratio by town")

legend2_values_list = list( legend2.values() )
legend2_keys_list = list( legend2.keys() )

plt.figure(2)  # "banana" shape
plt.title("Outlier detection on a real data set (boston housing)")
plt.scatter(X2[:, 0], X2[:, 1], color='black')
plt.xlim((xx2.min(), xx2.max()))
plt.ylim((yy2.min(), yy2.max()))
plt.legend((legend2_values_list[0].collections[0],
            legend2_values_list[1].collections[0],
            legend2_values_list[2].collections[0]),
           (legend2_values_list[0], legend2_values_list[1], legend2_values_list[2]),
           loc="upper center",
           prop=matplotlib.font_manager.FontProperties(size=12))
plt.ylabel("% lower status of the population")
plt.xlabel("average number of rooms per dwelling")

plt.show()


import tensorflow as tf
import numpy as np
import sklearn.metrics as metrics

#from databatcher import DataBatcher
import nucconvmodel
#import dubiotools as dbt

import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import pprint
from itertools import cycle

import os
import sys

#Logging imports
from logger import Logger

from nucinference import NucInference

from collections import OrderedDict

class NucBinaryClassifier(NucInference):

    use_onehot_labels = True

    def __init__(self,
                 sess,
                 train_batcher,
                 test_batcher,
                 num_epochs,
                 learning_rate,
                 batch_size,
                 seq_len,
                 save_dir,
                 keep_prob=0.5,
                 beta1=0.9,
                 concat_revcom_input=False,
                 nn_method_key="inferenceA",
                 pos_index=1):

        """NucBinaryClassifier encapsulates training and data
        evaluation for

        :param sess: tf.Session() object
        :param train_batcher: DataBatcher object for training set
        :param test_batcher: DataBatcher object for test set
        :param num_epochs: Number of epoch cycles to perform training
        :param learning_rate: Learning rate
        :param batch_size: Mini-batch pull size
        :param seq_len: Sequence length
        :param save_dir: Root save directory for binary classification model
        :param keep_prob: Probability of keeping weight for dropout
        regularization
        :param beta1: Beta1 parameter for AdamOptimizer
        :param concat_revcom_input: If true, concatenate reverse
        complement of nucleotide sequence to input vector
        :param nn_method_key: Dictionary key for inference
        method found in nucconvmodels.py file. Determines which model
        to use. Example: "inferenceA" will run nucconvmodels.inferenceA
        :param pos_index: The index to use for the positive class
        (defaults to 1)
        :returns: a NucBinaryClassifier object
        :rtype: NucBinaryClassifier

        """

        super(NucBinaryClassifier, self).__init__(sess,
                                    train_batcher,
                                    test_batcher,
                                    num_epochs,
                                    learning_rate,
                                    batch_size,
                                    seq_len,
                                    save_dir,
                                    keep_prob,
                                    beta1,
                                    concat_revcom_input,
                                    nn_method_key="inferenceA")

        if self.train_batcher.num_classes != 2:
            print "Error, more than two classes detected in train batcher"
        else:
            self.num_classes = 2

        #The index for the label that should be considered the positive class
        self.pos_index=pos_index
        self.save_on_epoch = 5

    def build_model(self):

        self.dna_seq_placeholder = tf.placeholder(tf.float32,
                                          shape=[None,self.seq_len,4],
                                          name="dna_seq")

        self.labels_placeholder = tf.placeholder(tf.float32,
                                            shape=[None, self.num_classes],
                                            name="labels")

        self.keep_prob_placeholder = tf.placeholder(tf.float32,name="keep_prob")

        self.logits, self.network = self.nn_method(self.dna_seq_placeholder,
                                                   self.keep_prob_placeholder,
                                                   self.num_classes)

        self.probs = tf.nn.softmax(self.logits)

        self.loss = tf.reduce_mean(
                           tf.nn.softmax_cross_entropy_with_logits(labels=self.labels_placeholder,
                                                                   logits=self.logits))

        '''
        Calculate metrics. num_true positives is the number of true positives for the current batch

        Table below shows index if tf.argmax is applied

        +-----+-----------+---------+
        |     | Classifier|  Label  |
        +-----+-----------+---------+
        | TP  |     1     |  1      |
        +-----+-----------+---------+
        | FP  |     1     |  0      |
        +-----+-----------+---------+
        | TN  |     0     |  0      |
        +-----+-----------+---------+
        | FN  |     0     |  1      |
        +-----+-----------+---------+

        Precision = TP/(TP+FP)
        Recall = TP/(TP+FN)
        F1-score = 2*(Prec*Rec)/(Prec+Rec)

        # Note: I ended up not using the tp,fp,tn,fn ops because I ended up calculating
        # these metrics using sklearn.
        '''

        #correct  = TN+TP     #Used for calculating accuracy
        self.logits_ind = tf.argmax(self.logits,1)
        self.labels_ind = tf.argmax(self.labels_placeholder,1)

        #Create max_mask of logits (ie: [-.5,.5] --> [0 1]. Note logits have
        # shape [batch_size * num_classes= 2]
        #self.inverse_logits_col = tf.ones_like(self.logits_ind) - self.logits_ind
        #self.max_mask_logits = tf.concat([self.inverse_logits_col,self.logits_ind],1)

        #True positives where logits_ind+labels_ind == 2
        #True negatives where logits_ind+labels_ind == 0
        self.sum_ind = tf.add(self.logits_ind,self.labels_ind)

        self.true_positives = tf.equal(self.sum_ind,2*tf.ones_like(self.sum_ind)) #bool
        self.num_true_positives =tf.reduce_sum(tf.cast(self.true_positives, tf.int32))

        #For FP classifier index > label index
        self.false_positives=tf.greater(self.logits_ind,self.labels_ind)
        self.num_false_positives = tf.reduce_sum(tf.cast(self.false_positives, tf.int32))

        self.true_negatives = tf.equal(self.sum_ind,tf.zeros_like(self.sum_ind)) #bool
        self.num_true_negatives= tf.reduce_sum(tf.cast(self.true_negatives,tf.int32))

        #For FN classifier index < label index
        self.false_negatives=tf.less(self.logits_ind,self.labels_ind)
        self.num_false_negatives = tf.reduce_sum(tf.cast(self.false_negatives,tf.int32))

        #num correct can be used to calculate accuracy
        self.correct = tf.equal(self.logits_ind,self.labels_ind)
        self.num_correct= tf.reduce_sum(tf.cast(self.correct, tf.int32))

        self.relevance =self.network.relevance_backprop(tf.multiply(self.logits,
                                                               self.labels_placeholder))

        '''Write and consolidate summaries'''
        self.loss_summary = tf.summary.scalar('loss',self.loss)

        self.summary_writer = tf.summary.FileWriter(self.summary_dir,self.sess.graph)

        self.summary_op = tf.summary.merge([self.loss_summary])

        #Note: Do not use tf.summary.merge_all() here. This will break encapsulation for
        # cross validation and lead to crashes when training multiple models

        # Add gradient ops to graph with learning rate
        self.train_op = tf.train.AdamOptimizer(self.learning_rate,
                                               beta1=self.beta1).minimize(self.loss)

        self.vars = tf.trainable_variables()
        self.var_names = [var.name for var in self.vars]
        #print "Trainable variables:\n"
        #for vname in self.var_names:
        #    print vname

        self.saver = tf.train.Saver()
        self.init_op = tf.global_variables_initializer()
        #Important note: Restoring model does not require init_op.
        #In fact calling tf.global_variables_initializer() after loading a model
        #will overwrite loaded weights
        self.sess.run(self.init_op)
        self.load(self.checkpoint_dir)

    def eval_model_metrics(self,
                           batcher,
                           save_plots=False,
                           image_name ='metrics.png',
                           eval_batch_size=50):

        """
        Note: This method only works for binary classification
        as auPRC and auROC graphs only apply to binary classificaton problems.

        TODO: Modify this code to perform auROC generation
        for one-vs-all in the case of multiclass classification.

        """

        #Ref: http://scikit-learn.org/stable/modules/model_evaluation.html#roc-metrics
        ##auROC calculations

        #Keep batch size at 1 for now to ensure 1 full epoch is evaluated

        all_labels = np.zeros((batcher.num_records,self.num_classes), dtype = np.float32)
        all_probs = np.zeros((batcher.num_records,self.num_classes), dtype = np.float32)

        #num_correct = 0 #counts number of correct predictions
        num_whole_pulls =  batcher.num_records//eval_batch_size
        num_single_pulls = batcher.num_records%eval_batch_size
        num_steps = num_whole_pulls+num_single_pulls
        for i in range(num_steps):
            if i<num_whole_pulls:
                batch_size=eval_batch_size
            else:
                batch_size=1

            labels_batch, dna_seq_batch = batcher.pull_batch(batch_size)
            feed_dict = {
                          self.dna_seq_placeholder:dna_seq_batch,
                          self.labels_placeholder:labels_batch,
                          self.keep_prob_placeholder:1.0
                        }

            cur_prob= self.sess.run(self.probs,feed_dict=feed_dict)

            #Fill labels array
            if batch_size > 1:
                start_ind = batch_size*i
            elif batch_size == 1:
                start_ind = num_whole_pulls*eval_batch_size+(i-num_whole_pulls)
            else:
                print "Never reach this condition"

            all_labels[start_ind:start_ind+batch_size,:] =  labels_batch
            all_probs[start_ind:start_ind+batch_size,:]  = cur_prob

        #Calculate metrics and save results in a dict
        md = self.calc_classifier_metrics(all_labels,all_probs)
        md["epoch"]=self.epoch
        md["step"]=self.step

        #print "Testing accuracy",float(num_correct)/float(batcher.num_records)

        print 'Num examples: %d  Num correct: %d  Accuracy: %0.04f' % \
                  (batcher.num_records, md["num_correct"], md["accuracy"])+'\n'

        if save_plots:
            ###Plot some metrics
            plot_colors = cycle(['cyan','blue','orange','teal'])

            #print "Labels shape",all_labels.shape
            #print "Probs shape",all_probs.shape
            #print "Preds shape",all_preds.shape

            #Generate auROC plot axes
            fig1,ax1  = plt.subplots(2)
            fig1.subplots_adjust(bottom=0.2)

            ax1[0].plot([0,1],[0,1],color='navy',lw=2,linestyle='--')
            ax1[0].set_xbound(0.0,1.0)
            ax1[0].set_ybound(0.0,1.05)
            ax1[0].set_xlabel('False Positive Rate')
            ax1[0].set_ylabel('True Positive Rate')
            ax1[0].set_title('auROC')
            #plt.legend(loc='lower right')

            ax1[0].plot(md["fpr"],md["tpr"],color=plot_colors.next(),
                        lw=2,linestyle='-',label='auROC curve (area=%0.2f)' % md["auroc"] )

            #Generate auPRC plot axes
            #ax1[1].plot([0,1],[1,1],color='royalblue',lw=2,linestyle='--')
            ax1[1].set_xlabel('Precision')
            ax1[1].set_ylabel('Recall')
            ax1[1].set_title('auPRC')
            ax1[1].plot(md["thresh_precision"],md["thresh_recall"],color=plot_colors.next(),
                        lw=2,linestyle='-',label='auPRC curve (area=%0.2f)' % md["auprc"] )

            ax1[1].set_xbound(0.0,1.0)
            ax1[1].set_ybound(0.0,1.05)

            #Note: avg prec score is the area under the prec recall curve

            #Note: Presumably class 1 (pos examples) should be the only f1 score we focus on
            #print "F1 score for class",i,"is",f1_score
            plt.tight_layout()

            plt_fname = self.save_dir+os.sep+image_name
            print "Saving auROC image to",plt_fname
            fig1.savefig(plt_fname)

        #Return metrics dictionary
        return md

    def calc_classifier_metrics(self,all_labels,all_probs):
        """Calculate some metrics for the dataset
           return dictionary with metrics

        :param all_probs: nx2 prob values
        :param all_labels: nx2 labels
        :returns: dictionary of metrics
        :rtype: dict()

        """
        num_records = all_probs.shape[0]
        all_preds = np.zeros((num_records, self.num_classes),dtype = np.float32)
        all_preds[np.arange(num_records),all_probs.argmax(1)] = 1

        #Calculate accuracy
        num_correct = metrics.accuracy_score(all_labels[:,self.pos_index],all_preds[:,self.pos_index],normalize=False)
        accuracy = num_correct/float(all_preds.shape[0])

        ###Calculate auROC
        #http://scikit-learn.org/stable/auto_examples/model_selection/plot_roc.html
        #metrics.roc_curve(y_true, y_score[, ...]) #y_score is probs

        fpr,tpr,_ = metrics.roc_curve(all_labels[:,self.pos_index],
                                      all_probs[:,self.pos_index],
                                      pos_label=self.pos_index)
        auroc = metrics.auc(fpr,tpr)

        thresh_precision,thresh_recall,prc_thresholds = metrics.precision_recall_curve(
                                                        all_labels[:,self.pos_index],
                                                        all_probs[:,self.pos_index])

        #Calculate precision, recall, and f1-score for threshold = 0.5
        #confusion_matrix = metrics.confusion_matrix(all_labels[:,self.pos_index],all_probs[:,self.pos_index])
        precision, recall, f1_score, support = metrics.precision_recall_fscore_support(
                                                all_labels[:,self.pos_index],
                                                all_preds[:,self.pos_index],
                                                pos_label=self.pos_index)

        precision = precision[self.pos_index]
        recall = recall[self.pos_index]
        f1_score = f1_score[self.pos_index]
        support = support[self.pos_index]

        auprc = metrics.average_precision_score(all_labels[:,self.pos_index],
                                                all_probs[:,self.pos_index])

        return OrderedDict([
                ("num_correct",num_correct),
                ("accuracy",accuracy),
                ("auroc",auroc),
                ("auprc",auprc),
                ("fpr",fpr),
                ("tpr",tpr),
                ("precision",precision),
                ("recall",recall),
                ("f1_score",f1_score),
                ("support",support),
                ("thresh_precision",thresh_precision),
                ("thresh_recall",thresh_recall),
                ("prc_thresholds",prc_thresholds)
                 ])


"""
    Tests for the pandas.io.common functionalities
"""
from pandas.compat import StringIO
import os
from os.path import isabs

import nose
import pandas.util.testing as tm

from pandas.io import common

try:
    from pathlib import Path
except ImportError:
    pass

try:
    from py.path import local as LocalPath
except ImportError:
    pass

class TestCommonIOCapabilities(tm.TestCase):

    def test_expand_user(self):
        filename = '~/sometest'
        expanded_name = common._expand_user(filename)

        self.assertNotEqual(expanded_name, filename)
        self.assertTrue(isabs(expanded_name))
        self.assertEqual(os.path.expanduser(filename), expanded_name)

    def test_expand_user_normal_path(self):
        filename = '/somefolder/sometest'
        expanded_name = common._expand_user(filename)

        self.assertEqual(expanded_name, filename)
        self.assertEqual(os.path.expanduser(filename), expanded_name)

    def test_stringify_path_pathlib(self):
        tm._skip_if_no_pathlib()

        rel_path = common._stringify_path(Path('.'))
        self.assertEqual(rel_path, '.')
        redundant_path = common._stringify_path(Path('foo//bar'))
        self.assertEqual(redundant_path, os.path.join('foo', 'bar'))

    def test_stringify_path_localpath(self):
        tm._skip_if_no_localpath()

        path = os.path.join('foo', 'bar')
        abs_path = os.path.abspath(path)
        lpath = LocalPath(path)
        self.assertEqual(common._stringify_path(lpath), abs_path)

    def test_get_filepath_or_buffer_with_path(self):
        filename = '~/sometest'
        filepath_or_buffer, _, _ = common.get_filepath_or_buffer(filename)
        self.assertNotEqual(filepath_or_buffer, filename)
        self.assertTrue(isabs(filepath_or_buffer))
        self.assertEqual(os.path.expanduser(filename), filepath_or_buffer)

    def test_get_filepath_or_buffer_with_buffer(self):
        input_buffer = StringIO()
        filepath_or_buffer, _, _ = common.get_filepath_or_buffer(input_buffer)
        self.assertEqual(filepath_or_buffer, input_buffer)