import networkx as nx
import numpy as np
from numpy.lib.stride_tricks import as_strided
from scipy import ndimage as ndi
import math
from ... import draw, measure, segmentation, util, color
try:
    from matplotlib import colors
    from matplotlib import cm
except ImportError:
    pass

def min_weight(graph, src, dst, n):
    """Callback to handle merging nodes by choosing minimum weight.

    Returns either the weight between (`src`, `n`) or (`dst`, `n`)
    in `graph` or the minimum of the two when both exist.

    Parameters
    ----------
    graph : RAG
        The graph under consideration.
    src, dst : int
        The verices in `graph` to be merged.
    n : int
        A neighbor of `src` or `dst` or both.

    Returns
    -------
    weight : float
        The weight between (`src`, `n`) or (`dst`, `n`) in `graph` or the
        minimum of the two when both exist.

    """

    # cover the cases where n only has edge to either `src` or `dst`
    default = {'weight': np.inf}
    w1 = graph[n].get(src, default)['weight']
    w2 = graph[n].get(dst, default)['weight']
    return min(w1, w2)

def _add_edge_filter(values, graph):
    """Create edge in `graph` between central element of `values` and the rest.

    Add an edge between the middle element in `values` and
    all other elements of `values` into `graph`.  ``values[len(values) // 2]``
    is expected to be the central value of the footprint used.

    Parameters
    ----------
    values : array
        The array to process.
    graph : RAG
        The graph to add edges in.

    Returns
    -------
    0 : float
        Always returns 0. The return value is required so that `generic_filter`
        can put it in the output array, but it is ignored by this filter.
    """
    values = values.astype(int)
    center = values[len(values) // 2]
    for value in values:
        if value != center and not graph.has_edge(center, value):
            graph.add_edge(center, value)
    return 0.

class RAG(nx.Graph):

    """
    The Region Adjacency Graph (RAG) of an image, subclasses
    `networx.Graph <http://networkx.github.io/documentation/latest/reference/classes.graph.html>`_

    Parameters
    ----------
    label_image : array of int
        An initial segmentation, with each region labeled as a different
        integer. Every unique value in ``label_image`` will correspond to
        a node in the graph.
    connectivity : int in {1, ..., ``label_image.ndim``}, optional
        The connectivity between pixels in ``label_image``. For a 2D image,
        a connectivity of 1 corresponds to immediate neighbors up, down,
        left, and right, while a connectivity of 2 also includes diagonal
        neighbors. See `scipy.ndimage.generate_binary_structure`.
    data : networkx Graph specification, optional
        Initial or additional edges to pass to the NetworkX Graph
        constructor. See `networkx.Graph`. Valid edge specifications
        include edge list (list of tuples), NumPy arrays, and SciPy
        sparse matrices.
    **attr : keyword arguments, optional
        Additional attributes to add to the graph.
    """

    def __init__(self, label_image=None, connectivity=1, data=None, **attr):

        super(RAG, self).__init__(data, **attr)
        if self.number_of_nodes() == 0:
            self.max_id = 0
        else:
            self.max_id = max(self.nodes_iter())

        if label_image is not None:
            fp = ndi.generate_binary_structure(label_image.ndim, connectivity)
            ndi.generic_filter(
                label_image,
                function=_add_edge_filter,
                footprint=fp,
                mode='nearest',
                output=as_strided(np.empty((1,), dtype=np.float_),
                                  shape=label_image.shape,
                                  strides=((0,) * label_image.ndim)),
                extra_arguments=(self,))

    def merge_nodes(self, src, dst, weight_func=min_weight, in_place=True,
                    extra_arguments=[], extra_keywords={}):
        """Merge node `src` and `dst`.

        The new combined node is adjacent to all the neighbors of `src`
        and `dst`. `weight_func` is called to decide the weight of edges
        incident on the new node.

        Parameters
        ----------
        src, dst : int
            Nodes to be merged.
        weight_func : callable, optional
            Function to decide edge weight of edges incident on the new node.
            For each neighbor `n` for `src and `dst`, `weight_func` will be
            called as follows: `weight_func(src, dst, n, *extra_arguments,
            **extra_keywords)`. `src`, `dst` and `n` are IDs of vertices in the
            RAG object which is in turn a subclass of
            `networkx.Graph`.
        in_place : bool, optional
            If set to `True`, the merged node has the id `dst`, else merged
            node has a new id which is returned.
        extra_arguments : sequence, optional
            The sequence of extra positional arguments passed to
            `weight_func`.
        extra_keywords : dictionary, optional
            The dict of keyword arguments passed to the `weight_func`.

        Returns
        -------
        id : int
            The id of the new node.

        Notes
        -----
        If `in_place` is `False` the resulting node has a new id, rather than
        `dst`.
        """
        src_nbrs = set(self.neighbors(src))
        dst_nbrs = set(self.neighbors(dst))
        neighbors = (src_nbrs | dst_nbrs) - set([src, dst])

        if in_place:
            new = dst
        else:
            new = self.next_id()
            self.add_node(new)

        for neighbor in neighbors:
            w = weight_func(self, src, new, neighbor, *extra_arguments,
                            **extra_keywords)
            self.add_edge(neighbor, new, weight=w)

        self.node[new]['labels'] = (self.node[src]['labels'] +
                                    self.node[dst]['labels'])
        self.remove_node(src)

        if not in_place:
            self.remove_node(dst)

        return new

    def add_node(self, n, attr_dict=None, **attr):
        """Add node `n` while updating the maximum node id.

        .. seealso:: :func:`networkx.Graph.add_node`."""
        super(RAG, self).add_node(n, attr_dict, **attr)
        self.max_id = max(n, self.max_id)

    def add_edge(self, u, v, attr_dict=None, **attr):
        """Add an edge between `u` and `v` while updating max node id.

        .. seealso:: :func:`networkx.Graph.add_edge`."""
        super(RAG, self).add_edge(u, v, attr_dict, **attr)
        self.max_id = max(u, v, self.max_id)

    def copy(self):
        """Copy the graph with its max node id.

        .. seealso:: :func:`networkx.Graph.copy`."""
        g = super(RAG, self).copy()
        g.max_id = self.max_id
        return g

    def next_id(self):
        """Returns the `id` for the new node to be inserted.

        The current implementation returns one more than the maximum `id`.

        Returns
        -------
        id : int
            The `id` of the new node to be inserted.
        """
        return self.max_id + 1

    def _add_node_silent(self, n):
        """Add node `n` without updating the maximum node id.

        This is a convenience method used internally.

        .. seealso:: :func:`networkx.Graph.add_node`."""
        super(RAG, self).add_node(n)

def rag_mean_color(image, labels, connectivity=2, mode='distance',
                   sigma=255.0):
    """Compute the Region Adjacency Graph using mean colors.

    Given an image and its initial segmentation, this method constructs the
    corresponding Region Adjacency Graph (RAG). Each node in the RAG
    represents a set of pixels within `image` with the same label in `labels`.
    The weight between two adjacent regions represents how similar or
    dissimilar two regions are depending on the `mode` parameter.

    Parameters
    ----------
    image : ndarray, shape(M, N, [..., P,] 3)
        Input image.
    labels : ndarray, shape(M, N, [..., P,])
        The labelled image. This should have one dimension less than
        `image`. If `image` has dimensions `(M, N, 3)` `labels` should have
        dimensions `(M, N)`.
    connectivity : int, optional
        Pixels with a squared distance less than `connectivity` from each other
        are considered adjacent. It can range from 1 to `labels.ndim`. Its
        behavior is the same as `connectivity` parameter in
        `scipy.ndimage.generate_binary_structure`.
    mode : {'distance', 'similarity'}, optional
        The strategy to assign edge weights.

            'distance' : The weight between two adjacent regions is the
            :math:`|c_1 - c_2|`, where :math:`c_1` and :math:`c_2` are the mean
            colors of the two regions. It represents the Euclidean distance in
            their average color.

            'similarity' : The weight between two adjacent is
            :math:`e^{-d^2/sigma}` where :math:`d=|c_1 - c_2|`, where
            :math:`c_1` and :math:`c_2` are the mean colors of the two regions.
            It represents how similar two regions are.
    sigma : float, optional
        Used for computation when `mode` is "similarity". It governs how
        close to each other two colors should be, for their corresponding edge
        weight to be significant. A very large value of `sigma` could make
        any two colors behave as though they were similar.

    Returns
    -------
    out : RAG
        The region adjacency graph.

    Examples
    --------
    >>> from skimage import data, segmentation
    >>> from skimage.future import graph
    >>> img = data.astronaut()
    >>> labels = segmentation.slic(img)
    >>> rag = graph.rag_mean_color(img, labels)

    References
    ----------
    .. [1] Alain Tremeau and Philippe Colantoni
           "Regions Adjacency Graph Applied To Color Image Segmentation"
           http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.11.5274

    """
    graph = RAG(labels, connectivity=connectivity)

    for n in graph:
        graph.node[n].update({'labels': [n],
                              'pixel count': 0,
                              'total color': np.array([0, 0, 0],
                                                      dtype=np.double)})

    for index in np.ndindex(labels.shape):
        current = labels[index]
        graph.node[current]['pixel count'] += 1
        graph.node[current]['total color'] += image[index]

    for n in graph:
        graph.node[n]['mean color'] = (graph.node[n]['total color'] /
                                       graph.node[n]['pixel count'])

    for x, y, d in graph.edges_iter(data=True):
        diff = graph.node[x]['mean color'] - graph.node[y]['mean color']
        diff = np.linalg.norm(diff)
        if mode == 'similarity':
            d['weight'] = math.e ** (-(diff ** 2) / sigma)
        elif mode == 'distance':
            d['weight'] = diff
        else:
            raise ValueError("The mode '%s' is not recognised" % mode)

    return graph

def draw_rag(labels, rag, img, border_color=None, node_color='#ffff00',
             edge_color='#00ff00', colormap=None, thresh=np.inf,
             desaturate=False, in_place=True):
    """Draw a Region Adjacency Graph on an image.

    Given a labelled image and its corresponding RAG, draw the nodes and edges
    of the RAG on the image with the specified colors. Nodes are marked by
    the centroids of the corresponding regions.

    Parameters
    ----------
    labels : ndarray, shape (M, N)
        The labelled image.
    rag : RAG
        The Region Adjacency Graph.
    img : ndarray, shape (M, N, 3)
        Input image.
    border_color : colorspec, optional
        Any matplotlib colorspec.
    node_color : colorspec, optional
        Any matplotlib colorspec. Yellow by default.
    edge_color : colorspec, optional
        Any matplotlib colorspec. Green by default.
    colormap : colormap, optional
        Any matplotlib colormap. If specified the edges are colormapped with
        the specified color map.
    thresh : float, optional
        Edges with weight below `thresh` are not drawn, or considered for color
        mapping.
    desaturate : bool, optional
        Convert the image to grayscale before displaying. Particularly helps
        visualization when using the `colormap` option.
    in_place : bool, optional
        If set, the RAG is modified in place. For each node `n` the function
        will set a new attribute ``rag.node[n]['centroid']``.

    Returns
    -------
    out : ndarray, shape (M, N, 3)
        The image with the RAG drawn.

    Examples
    --------
    >>> from skimage import data, segmentation
    >>> from skimage.future import graph
    >>> img = data.coffee()
    >>> labels = segmentation.slic(img)
    >>> g =  graph.rag_mean_color(img, labels)
    >>> out = graph.draw_rag(labels, g, img)
    """
    if not in_place:
        rag = rag.copy()

    if desaturate:
        img = color.rgb2gray(img)
        img = color.gray2rgb(img)

    out = util.img_as_float(img, force_copy=True)
    cc = colors.ColorConverter()

    edge_color = cc.to_rgb(edge_color)
    node_color = cc.to_rgb(node_color)

    # Handling the case where one node has multiple labels
    # offset is 1 so that regionprops does not ignore 0
    offset = 1
    map_array = np.arange(labels.max() + 1)
    for n, d in rag.nodes_iter(data=True):
        for label in d['labels']:
            map_array[label] = offset
        offset += 1

    rag_labels = map_array[labels]
    regions = measure.regionprops(rag_labels)

    for (n, data), region in zip(rag.nodes_iter(data=True), regions):
        data['centroid'] = region['centroid']

    if border_color is not None:
        border_color = cc.to_rgb(border_color)
        out = segmentation.mark_boundaries(out, rag_labels, color=border_color)

    if colormap is not None:
        edge_weight_list = [d['weight'] for x, y, d in
                            rag.edges_iter(data=True) if d['weight'] < thresh]
        norm = colors.Normalize()
        norm.autoscale(edge_weight_list)
        smap = cm.ScalarMappable(norm, colormap)

    for n1, n2, data in rag.edges_iter(data=True):

        if data['weight'] >= thresh:
            continue
        r1, c1 = map(int, rag.node[n1]['centroid'])
        r2, c2 = map(int, rag.node[n2]['centroid'])
        line = draw.line(r1, c1, r2, c2)

        if colormap is not None:
            out[line] = smap.to_rgba([data['weight']])[0][:-1]
        else:
            out[line] = edge_color

        circle = draw.circle(r1, c1, 2)
        out[circle] = node_color

    return out


import argparse
import matplotlib
matplotlib.use('agg')
import csv
import json
import multiprocessing as mp
import os
import random
import re
import sys
from functools import partial
from operator import attrgetter, itemgetter
import networkx as nx
import numpy as np
import pandas as pd
import time
from sofa_aisi import *
from sofa_common import *
from sofa_config import *
from sofa_print import *
from matplotlib import pyplot as plt
import grpc
import potato_pb2
import potato_pb2_grpc
import socket
import random
import subprocess
from sofa_ml import hsg_v2

def random_generate_color():
    rand = lambda: random.randint(0, 255)
    return '#%02X%02X%02X' % (64, rand(), rand())

def get_top_k_events(cfg, df, topk):
    topk_events=[]
    gby = df.groupby(['name'])
    df_agg = gby.aggregate(np.sum)
    df_agg_sorted = df_agg.sort_values(by=['duration'],ascending=False)
    #memcpy = ['copyKind_1_','copyKind_2_','copyKind_8_']
    if cfg.verbose:
        print("Top %d Events: "%topk)
        print(df_agg_sorted[['duration']][0:topk])
    eventName = df_agg_sorted[df_agg_sorted.columns[0:0]].head(topk).index.values.tolist()
    return eventName

# input: pfv(performance feature vector), Pandas.DataFrame
# output: hint, docker_image
def get_hint(potato_server, features):

    if len(features) > 0:
        pfv = potato_pb2.PerformanceFeatureVector()
        for i in range(len(features)):
            name = features.iloc[i]['name']
            value = features.iloc[i]['value']
            #print('%s%s%s' % (str(i).ljust(10), name.ljust(30), ('%.3lf'%value).ljust(20)))
            pfv.name.append(name)
            pfv.value.append(value)

        #print('Wait for response from POTATO server...')
        myhostname = socket.gethostname()
        channel = grpc.insecure_channel(potato_server)
        stub = potato_pb2_grpc.HintStub(channel)
        request = potato_pb2.HintRequest( hostname = myhostname,
                                          pfv = pfv)
        response = stub.Hint(request)
        hint = response.hint
        docker_image = response.docker_image
    else:
        hint = 'There is no pfv to get hints.'
        docker_image = 'NA'

    return hint, docker_image

def concurrency_breakdown(logdir, cfg, df_mpstat, df_cpu, df_gpu, df_nvsmi, df_bandwidth, features):
    if cfg.verbose:
        print_title('Concurrency Breakdown Analysis')

    total_elapsed_time = {'usr':0, 'sys':0, 'gpu':0, 'iow':0, 'idl':0}
    elapsed_time_ratio = {'usr':0, 'sys':0, 'gpu':0, 'iow':0, 'idl':0}
    total_interval_vector = []
    total_performace_vector = []

    if len(df_mpstat) == 0:
        print_warning(cfg, 'no mpstat and perf traces!')
        return features

    t_begin = df_mpstat.iloc[0]['timestamp']
    t_end = df_mpstat.iloc[-1]['timestamp']
    t = t_begin
    sample_time = (1 / float(cfg.sys_mon_rate))
    while t < t_end:
        t = t + sample_time
        if cfg.roi_end > 0 and (t < cfg.roi_begin or t > cfg.roi_end):
            continue

        window_begin = t - sample_time
        window_end = t

        if len(df_cpu) > 0:
            if df_cpu.iloc[0].timestamp > window_end:
                continue
            cond1 = (df_cpu['timestamp'] > window_begin)
            cond2 = (df_cpu['timestamp'] <= window_end)
            df_cpu_interval = df_cpu[ cond1 & cond2 ]

        num_gpus = len(list(set(df_nvsmi['deviceId'])))
        cond1 = (df_nvsmi['timestamp'] > window_begin)
        cond2 = (df_nvsmi['timestamp'] <= window_end)
        sm = df_nvsmi['event'] == int(0)
        df_nvsmi_interval = df_nvsmi[ cond1 & cond2 & sm ]

        cond1 = (df_mpstat['timestamp'] > window_begin)
        cond2 = (df_mpstat['timestamp'] <= window_end)
        df_mpstat_interval = df_mpstat[ cond1 & cond2 ]

        cond1 = (df_bandwidth['timestamp'] > window_begin)
        cond2 = (df_bandwidth['timestamp'] <= window_end)
        tx = df_bandwidth['event'] == float(0)
        rx = df_bandwidth['event'] == float(1)
        df_tx_interval = df_bandwidth[ cond1 & cond2 & tx ]
        df_rx_interval = df_bandwidth[ cond1 & cond2 & rx ]

        mp_usr = []
        mp_sys = []
        mp_idl = []
        mp_iow = []

        usr = []
        sys = []
        irq = []

        cpu_max = 0
        cpu_min = 100
        for i in range(len(df_mpstat_interval)):
            ratios = df_mpstat_interval.iloc[i]['name'].split(':')[1].split('|')

            #print(ratios)
            mp_usr.append(sample_time*int(ratios[1])/100.0)
            mp_sys.append(sample_time*int(ratios[2])/100.0)
            mp_idl.append(sample_time*int(ratios[3])/100.0)
            mp_iow.append(sample_time*int(ratios[4])/100.0)

            usr.append(int(ratios[1]))
            sys.append(int(ratios[2]))
            irq.append(int(ratios[5]))

            cpu_tmp = int(ratios[1]) + int(ratios[2]) + int(ratios[5])
            if cpu_tmp > cpu_max:
                cpu_max = cpu_tmp
            if cpu_tmp < cpu_min:
                cpu_min = cpu_tmp
        mp_usr = np.asarray(mp_usr)
        mp_sys = np.asarray(mp_sys)
        mp_idl = np.asarray(mp_idl)
        mp_iow = np.asarray(mp_iow)

        usr = np.asarray(usr)
        sys = np.asarray(sys)
        irq = np.asarray(irq)

        elapsed_time = {'usr':0, 'sys':0, 'gpu':0, 'iow':0, 'idl':0}

        if len(df_mpstat_interval) > 0:
            elapsed_time['usr'] = mp_usr.max()
            elapsed_time['sys'] = mp_sys.max()
            elapsed_time['gpu'] = df_nvsmi_interval['duration'].max() * 0.01 * sample_time
            elapsed_time['iow'] = mp_iow.max()
            #print('gput,usrt = ', elapsed_time['gpu'], elapsed_time['usr'])
            dominator = max(elapsed_time, key=elapsed_time.get)
            #if elapsed_time['gpu'] > 0.1 :
            #    dominator = 'gpu'
            if elapsed_time[dominator] > sample_time * int(cfg.is_idle_threshold)/100:
                total_elapsed_time[dominator] = total_elapsed_time[dominator] + sample_time
            else:
                total_elapsed_time['idl'] += sample_time

            if num_gpus > 0:
                time_gpu_avg = df_nvsmi_interval['duration'].sum() * 0.01 * sample_time / num_gpus
            else:
                time_gpu_avg = 0

            interval_vector = [mp_usr.max(),
                               mp_sys.max(),
                               mp_iow.max(),
                               mp_idl.max(),
                               time_gpu_avg,
                               df_tx_interval['bandwidth'].sum(),
                               df_rx_interval['bandwidth'].sum()]
            total_interval_vector.append(tuple(interval_vector))
            if num_gpus > 0:
                sm_avg = df_nvsmi_interval['duration'].sum() / int(len(list(set(df_nvsmi_interval['deviceId']))))
            else:
                sm_avg = 0
            performace_vector = [window_end,
                                 df_nvsmi_interval['duration'].max(),
                                 sm_avg,
                                 df_nvsmi_interval['duration'].min(),
                                 round((usr.mean() + sys.mean() + irq.mean()), 0),
                                 cpu_max,
                                 cpu_min]
            total_performace_vector.append(tuple(performace_vector))

    total_all_elapsed_time = sum(total_elapsed_time.values())
    if total_all_elapsed_time > 0 :
        elapsed_time_ratio['usr'] = 100 * total_elapsed_time['usr'] / total_all_elapsed_time
        elapsed_time_ratio['sys'] = 100 * total_elapsed_time['sys'] / total_all_elapsed_time
        elapsed_time_ratio['gpu'] = 100 * total_elapsed_time['gpu'] / total_all_elapsed_time
        elapsed_time_ratio['idl'] = 100 * total_elapsed_time['idl'] / total_all_elapsed_time
        elapsed_time_ratio['iow'] = 100 * total_elapsed_time['iow'] / total_all_elapsed_time
        if cfg.verbose:
            print('Elapsed Time = %.1lf ' % total_all_elapsed_time)
            print('USR = %.1lf %%' % elapsed_time_ratio['usr'])
            print('SYS = %.1lf %%' % elapsed_time_ratio['sys'])
            if num_gpus > 0:
                print('GPU = %.1lf %%' % elapsed_time_ratio['gpu'])
            print('IDL = %.1lf %%' % elapsed_time_ratio['idl'])
            print('IOW = %.1lf %%' % elapsed_time_ratio['iow'])
        if cfg.spotlight_gpu:
            elapsed_hotspot_time = cfg.roi_end - cfg.roi_begin
        else:
            elapsed_hotspot_time = 0

        df = pd.DataFrame({ 'name':['elapsed_usr_time_ratio', 'elapsed_sys_time_ratio', 'elapsed_gpu_time_ratio',
                            'elapsed_iow_time_ratio', 'elapsed_hotspot_time'],
                            'value':[elapsed_time_ratio['usr'], elapsed_time_ratio['sys'], elapsed_time_ratio['gpu'],
                            elapsed_time_ratio['iow'], elapsed_hotspot_time ] },
                            columns=['name','value'])

        features = pd.concat([features, df])

    if len(total_performace_vector) > 0:
        performance_table = pd.DataFrame(total_performace_vector, columns = ['time', 'max_gpu_util', 'avg_gpu_util', 'min_gpu_util', 'cpu_util', 'cpu_max', 'cpu_min'])
        performance_table.to_csv('%s/performance.csv' % logdir)
        vector_table = pd.DataFrame(total_interval_vector, columns = ['usr' , 'sys', 'iow', 'idl','gpu', 'net_tx', 'net_rx'])
        pearson = vector_table.corr(method ='pearson').round(2)
        if cfg.verbose:
            print('Correlation Table :')
            print(pearson)
        df = pd.DataFrame({ 'name':['corr_gpu_usr', 'corr_gpu_sys', 'corr_gpu_iow', 'corr_gpu_ntx', 'corr_gpu_nrx'], 'value':[pearson['gpu'].usr, pearson['gpu'].sys, pearson['gpu'].iow, pearson['gpu'].net_tx, pearson['gpu'].net_rx]}, columns=['name','value'])
        features = pd.concat([features, df])
    return features

def payload_sum(df):
    print((len(df)))

class Event:

    def __init__(self, name, ttype, timestamp, duration):
        self.name = name
        self.ttype = ttype  # 0 for begin, 1 for end
        self.timestamp = timestamp
        self.duration = duration

    def __repr__(self):
        return repr((self.name, self.ttype, self.timestamp, self.duration))

def nvsmi_profile(logdir, cfg, df_nvsmi, features):
    if not cfg.cluster_ip and cfg.verbose:
        print_title('SM & MEM & ENCODE/DECODE Profiling')

    if cfg.spotlight_gpu:
        if cfg.roi_end == 0 :
            print_warning(cfg, 'spotlight_gpu has no effects.')
        else:
            cond1 = (df_nvsmi['timestamp'] > cfg.roi_begin)
            cond2 = (df_nvsmi['timestamp'] <= cfg.roi_end)
            df_nvsmi = df_nvsmi[ cond1 & cond2 ]

    sm_start = df_nvsmi.iloc[0].timestamp
    sm_end = df_nvsmi.iloc[-1].timestamp
    SM_time = sm_end - sm_start
    result = df_nvsmi.groupby(['deviceId','event'])['duration'].mean()
    result = result.astype(int)

    gpu_sm_util = df_nvsmi.groupby(['event'])['duration'].mean()[0]
    gpu_mem_util = df_nvsmi.groupby(['event'])['duration'].mean()[1]
    if cfg.nvsmi_data:
        gpu_enc_util = df_nvsmi.groupby(['event'])['duration'].mean()[2]
        gpu_dec_util = df_nvsmi.groupby(['event'])['duration'].mean()[3]
    else:
        gpu_enc_util = 0
        gpu_dec_util = 0

    sm = df_nvsmi['event'] == int(0)
    mem = df_nvsmi['event'] == int(1)
    enc = df_nvsmi['event'] == int(2)
    dec = df_nvsmi['event'] == int(3)
    gpunum = list(set(df_nvsmi['deviceId']))
    res = pd.DataFrame([], columns=['sm', 'mem', 'enc', 'dec'])
    sm_q = pd.DataFrame([], columns=['Q1', 'Q2', 'Q3', 'Avg'])
    mem_q = pd.DataFrame([], columns=['Q1', 'Q2', 'Q3', 'Avg'])
    for i in gpunum:
        gpuid = df_nvsmi['deviceId'] == int(i)
        gpudata = [round(df_nvsmi[sm & gpuid]['duration'].mean(), 2),
                   round(df_nvsmi[mem & gpuid]['duration'].mean(), 2),
                   round(df_nvsmi[enc & gpuid]['duration'].mean(), 2),
                   round(df_nvsmi[dec & gpuid]['duration'].mean(), 2)]
        smdata = [round(df_nvsmi[sm & gpuid]['duration'].quantile(0.25), 2),
                  round(df_nvsmi[sm & gpuid]['duration'].quantile(0.5), 2),
                  round(df_nvsmi[sm & gpuid]['duration'].quantile(0.75), 2),
                  round(df_nvsmi[sm & gpuid]['duration'].mean(), 2)]
        memdata = [round(df_nvsmi[mem & gpuid]['duration'].quantile(0.25), 2),
                   round(df_nvsmi[mem & gpuid]['duration'].quantile(0.5), 2),
                   round(df_nvsmi[mem & gpuid]['duration'].quantile(0.75), 2),
                   round(df_nvsmi[mem & gpuid]['duration'].mean(), 2)]
        gpu_tmp = pd.DataFrame([gpudata], columns=['sm', 'mem', 'enc', 'dec'], index=[i])
        sm_tmp = pd.DataFrame([smdata], columns=['Q1', 'Q2', 'Q3', 'Avg'], index=[i])
        mem_tmp = pd.DataFrame([memdata], columns=['Q1', 'Q2', 'Q3', 'Avg'], index=[i])
        res = pd.concat([res, gpu_tmp])
        sm_q = pd.concat([sm_q, sm_tmp])
        mem_q = pd.concat([mem_q, mem_tmp])
    res.index.name = 'gpu_id'
    sm_q.index.name = 'gpu_id'
    mem_q.index.name = 'gpu_id'

    if not cfg.cluster_ip and cfg.verbose:
        print('GPU Utilization (%):')
        print(res)
        print('\nGPU SM Quartile (%):')
        print(sm_q)
        print('\nGPU MEM Quartile (%):')
        print(mem_q)
        print('Overall Average SM Utilization (%): ', int(gpu_sm_util))
        print('Overall Average MEM Utilization (%): ', int(gpu_mem_util))
        print('Overall Average ENC Utilization (%): ', int(gpu_enc_util))
        print('Overall Average DEC Utilization (%): ', int(gpu_dec_util))
        print('Overall Active GPU Time (s): %.3lf' % (SM_time * gpu_sm_util/100.0))
    df = pd.DataFrame({'name':['gpu_sm_util_q2', 'gpu_sm_util_q3', 'gpu_sm_util', 'gpu_mem_util_q2', 'gpu_mem_util_q3', 'gpu_mem_util'],
                    'value':[df_nvsmi[sm & gpuid]['duration'].quantile(0.5),
                             df_nvsmi[sm & gpuid]['duration'].quantile(0.75),
                             int(gpu_sm_util),
                             df_nvsmi[mem & gpuid]['duration'].quantile(0.5),
                             df_nvsmi[mem & gpuid]['duration'].quantile(0.75),
                             int(gpu_mem_util),
                            ]},
                    columns=['name','value'])
    features = pd.concat([features, df])

    return features

def gpu_profile(logdir, cfg, df_gpu, features):
    if cfg.verbose:
        print_title('GPU Profiling')
        print('Per-GPU time (s):')
    groups = df_gpu.groupby("deviceId")["duration"]
    gpu_time = 0
    for key, item in groups:
        gpuid = int(float(key))
        per_gpu_time = groups.get_group(key).sum()
        if cfg.verbose:
            print("[%d]: %lf" % (gpuid, per_gpu_time))
        gpu_time = gpu_time + per_gpu_time
    num_gpus = len(groups)

    kernel_time = 0
    grouped_df = df_gpu.groupby("copyKind")["duration"]
    for key, item in grouped_df:
        if key == 0:
            kernel_time = grouped_df.get_group(key).sum()

    nccl_time = 0
    grouped_df = df_gpu.groupby("name")["duration"]
    for key, item in grouped_df:
        #print("[%s]: %lf" % (key, grouped_df.get_group(key).sum()))
        if key.find("nccl") != -1:
            nccl_time = nccl_time + grouped_df.get_group(key).sum()

    features = comm_profile(logdir, cfg, df_gpu, features)

    get_top_k_events(cfg, df_gpu, 10)
    df = pd.DataFrame({'name':['gpu_time', 'num_gpus', 'kernel_time', 'nccl_time'],
                        'value':[gpu_time, num_gpus, kernel_time, nccl_time] },
                        columns=['name','value'])
    features = pd.concat([features, df])
    return features

def strace_profile(logdir, cfg, df, features):
    print_title('STRACE Profiling:')

    return features

def net_profile(logdir, cfg, df, features):
    if not cfg.cluster_ip:
        print_title("Network Profiling:")
    grouped_df = df.groupby("name")["duration"]
    net_time = 0
    n_packets = 0
    for key, item in grouped_df:
        #print("[%s]: %lf" % (key, grouped_df.get_group(key).sum()))
        if key.find("network:tcp:") != -1:
            net_time = net_time + grouped_df.get_group(key).sum()
            n_packets = n_packets + 1
    #print(("total network time (s) = %.3lf" % net_time))
    #print(("total amount of network packets  = %d" % n_packets))
    # total network packet
    packet_num_matrix = df.groupby(['pkt_src','pkt_dst','payload']).size().unstack(level=1, fill_value=0)

    # total network traffic
    packet_sum_matrix = df.groupby(['pkt_src','pkt_dst'])["payload"].sum().unstack(level=1, fill_value=0)

    # ================ change pandas table columns and index name ====
    rename_index = packet_sum_matrix.index.tolist()
    rename_index2 = packet_num_matrix.index.tolist()
    rename_columns = packet_sum_matrix.columns.tolist()
    rename_columns2 = packet_num_matrix.columns.tolist()

    def zero(s):
        if s[0:2] == '00':
            s = s[2]
        elif (s[0] == '0') and (s[1] != '0'):
            s = s[1:3]
        return(s)

    def check_str(rename_list):
        rename_list_new = []
        for j in rename_list:
            j = str(int(j))
            a = j[-9:-6]
            b = j[-6:-3]
            c = j[-3:]
            j = j[:-9] + '.' + zero(a) + '.' + zero(b) + '.' + zero(c)
            rename_list_new.append(j)
        return(rename_list_new)

    def check_str2(rename_list):
        rename_columns_2 = []
        for i in rename_list:
            i = str(int(i[0]))
            a = i[-9:-6]
            b = i[-6:-3]
            c = i[-3:]
            i = i[:-9] + '.' + zero(a) + '.' + zero(b) + '.' + zero(c)
            rename_columns_2.append(i)
        return(rename_columns_2)

    rename_index_new = check_str(rename_index)
    rename_index_new = dict(zip(rename_index, rename_index_new))

    rename_index2_new = check_str2(rename_index2)
    rename_index2_final = list(set(rename_index2_new))
    rename_index2_final.sort(key=rename_index2_new.index)

    rename_columns_new = check_str(rename_columns)
    rename_columns_new = dict(zip(rename_columns, rename_columns_new))

    rename_columns2_new = check_str(rename_columns2)
    rename_columns2_new = dict(zip(rename_columns2, rename_columns2_new))

    # rename here
    packet_sum_matrix = packet_sum_matrix.rename(columns=rename_columns_new)
    packet_num_matrix = packet_num_matrix.rename(columns=rename_columns2_new)
    packet_sum_matrix = packet_sum_matrix.rename(index=rename_index_new)
    packet_num_matrix.index.set_levels(rename_index2_final , level = 0, inplace = True)

    if cfg.verbose:
        print("total amount of network traffic : ", convertbyte(df['payload'].sum()), '\n', packet_sum_matrix.to_string(), "\n")
        print("total amount of network packets = %d\n" % packet_num_matrix.sum().sum() ,packet_num_matrix.to_string(), "\n")

    network_value = []
    src = []
    dst = []
    final = []

    for index in packet_sum_matrix.index:
        for column in packet_sum_matrix.columns:
            src.append(index)
            dst.append(column)
            network_value.append(packet_sum_matrix[column][index])
    record = list(zip(src, dst, network_value))
    record.sort(key=lambda tup:tup[2], reverse=True)

    for src, dst, value in record:
        if value == 0:
            pass
        else:
            item = [src, dst, convertbyte(value), round(value / df['payload'].sum(), 2)]
            final.append(item)
    summary = pd.DataFrame(final, columns=['Source', 'Destination', 'Amount', 'Percentage of a Node'])
    summary.to_csv(logdir + 'netrank.csv',
                mode='w',
                header=True,
                index=False)

    df = pd.DataFrame({'name':['net_time'],
                       'value':[net_time] },
                       columns=['name','value'])
    features = pd.concat([features, df])
    return features

def convertbyte(B):
    B = int(B)
    KB = float(1024)
    MB = float(KB ** 2) # 1,048,576
    GB = float(KB ** 3) # 1,073,741,824
    TB = float(KB ** 4) # 1,099,511,627,776

    if B < KB:
        return '{} Bytes'.format(B)
    elif KB <= B < MB:
        return '{0:.2f} KB'.format(B/KB)
    elif MB <= B < GB:
        return '{0:.2f} MB'.format(B/MB)
    elif GB <= B < TB:
        return '{0:.2f} GB'.format(B/GB)
    elif TB <= B:
        return '{0:.2f} TB'.format(B/TB)

def convertbytes(B):
    B = float(B)
    KB = float(1024)
    MB = float(KB ** 2) # 1,048,576
    GB = float(KB ** 3) # 1,073,741,824
    TB = float(KB ** 4) # 1,099,511,627,776

    if B < KB:
        return '{0:.2f} B/s'.format(B)
    elif KB <= B < MB:
        return '{0:.2f} KB/s'.format(B/KB)
    elif MB <= B < GB:
        return '{0:.2f} MB/s'.format(B/MB)
    elif GB <= B < TB:
        return '{0:.2f} GB/s'.format(B/GB)
    elif TB <= B:
        return '{0:.2f} TB/s'.format(B/TB)

def netbandwidth_profile(logdir, cfg, df, features):
    if not cfg.cluster_ip and cfg.verbose:
        print_title('Network Bandwidth Profiling:')
    tx = df['event'] == float(0)
    rx = df['event'] == float(1)

    bw_tx_q1 = df[tx]['bandwidth'].quantile(0.25)
    bw_tx_q2 = df[tx]['bandwidth'].quantile(0.5)
    bw_tx_q3 = df[tx]['bandwidth'].quantile(0.75)
    bw_tx_mean = int(df[tx]['bandwidth'].mean())
    bw_rx_q1 = df[rx]['bandwidth'].quantile(0.25)
    bw_rx_q2 = df[rx]['bandwidth'].quantile(0.5)
    bw_rx_q3 = df[rx]['bandwidth'].quantile(0.75)
    bw_rx_mean = int(df[rx]['bandwidth'].mean())
    with open('%s/netstat.txt' % logdir) as f:
        lines = f.readlines()
        first_line = lines[0]
        last_line = lines[-1]
        tx_begin = first_line.split(',')[1]
        rx_begin = first_line.split(',')[2]
        tx_end = last_line.split(',')[1]
        rx_end = last_line.split(',')[2]
        tx_amount = int(last_line.split(',')[1]) - int(first_line.split(',')[1])
        rx_amount = int(last_line.split(',')[2]) - int(first_line.split(',')[2])
    if not cfg.cluster_ip:
        bw_tx_q1 = df[tx]['bandwidth'].quantile(0.25)
        bw_tx_q2 = df[tx]['bandwidth'].quantile(0.5)
        bw_tx_q3 = df[tx]['bandwidth'].quantile(0.75)
        bw_tx_mean = int(df[tx]['bandwidth'].mean())
        bw_rx_q1 = df[rx]['bandwidth'].quantile(0.25)
        bw_rx_q2 = df[rx]['bandwidth'].quantile(0.5)
        bw_rx_q3 = df[rx]['bandwidth'].quantile(0.75)
        bw_rx_mean = int(df[rx]['bandwidth'].mean())
        if cfg.verbose:
            print('Amount of Network Traffic : %s' % (convertbyte(tx_amount + rx_amount)))
            print('Amount of tx : %s' % convertbyte(tx_amount))
            print('Amount of rx : %s' % convertbyte(rx_amount))
            print('Bandwidth Quartile :')
            print('Q1  tx : %s, rx : %s' % ( convertbytes(bw_tx_q1), convertbytes(bw_rx_q1)))
            print('Q2  tx : %s, rx : %s' % ( convertbytes(bw_tx_q2), convertbytes(bw_rx_q2)))
            print('Q3  tx : %s, rx : %s' % ( convertbytes(bw_tx_q3), convertbytes(bw_rx_q3)))
            print('Avg tx : %s, rx : %s'% ( convertbytes(bw_tx_mean), convertbytes(bw_rx_mean)))

    #network chart part
    all_time = df[tx]['timestamp'].tolist()
    all_tx = df[tx]['bandwidth'].tolist()
    all_rx = df[rx]['bandwidth'].tolist()
    fig = plt.figure(dpi=128, figsize=(16, 14))
    plt.plot(all_time, all_tx, c='red', alpha=0.5, label='tx')
    plt.plot(all_time, all_rx, c='blue', alpha=0.5, label='rx')
    plt.legend(loc='upper right')
    plt.title("Network Report", fontsize=18)
    plt.xlabel('Timestamp (s)', fontsize=16)
    plt.ylabel("Bandwidth (bytes)", fontsize=16)
    fig.savefig("%s/network_report.pdf" % logdir, bbox_inches='tight')
    if not cfg.cluster_ip and cfg.verbose:
        print('Network Bandwidth Chart is saved at %s/network_report.pdf' %logdir)

        df_feature = pd.DataFrame({ 'name':['bw_tx_q2', 'bw_tx_q3', 'bw_rx_q2', 'bw_rx_q3'],
                        'value':[bw_tx_q2, bw_tx_q3, bw_rx_q2, bw_rx_q3] },
                        columns=['name','value'])
        features = pd.concat([features, df_feature])

    return features

def blktrace_latency_profile(logdir, cfg, df, features):
    with open('%s/btt.txt' % logdir) as f:
        lines = f.readlines()
        for i, line in enumerate(lines):
            if '==================== All Devices ====================' in line:
                start = i
            if '==================== Device Merge Information ====================' in line:
                end = i
                break
        bttoutput_result = lines[start:end]

    df_offset = pd.read_table('%s/offset_all.txt' % logdir, delim_whitespace=True, names=('time', 'start', 'end'))
    time = df_offset['time'].tolist()
    start_b = df_offset['start'].tolist()
    end_b = df_offset['end'].tolist()
    fig = plt.figure(dpi=128, figsize=(16, 14))
    plt.plot(time, start_b, c='red', marker='o', alpha=0.3, label='Start block')
    plt.legend(loc='upper right')
    plt.title("Block Offset Report", fontsize=18)
    plt.xlabel('Timestamp (s)', fontsize=16)
    plt.ylabel("Block Number", fontsize=16)
    fig.savefig("%s/offset_of_device_report.pdf" % logdir, bbox_inches='tight')
    print('Offset of Device Report is saved at %s/offset_of_device_report.pdf' %logdir)

    if cfg.verbose:
        print_title('Storage Profiling:')
        print('Blktracae Latency (s):')
        for btt in bttoutput_result:
            print(btt[:-1])

    blktrace_latency = df['event'] == 'C'
    blktrace_latency_q1 = df[blktrace_latency]['duration'].quantile(0.25)
    blktrace_latency_q2 = df[blktrace_latency]['duration'].quantile(0.5)
    blktrace_latency_q3 = df[blktrace_latency]['duration'].quantile(0.75)
    blktrace_latency_mean = df[blktrace_latency]['duration'].mean()

    df_feature = pd.DataFrame({ 'name':['blktrace_latency_q1','blktrace_latency_q2','blktrace_latency_q3'],
                        'value': [blktrace_latency_q1, blktrace_latency_q2, blktrace_latency_q3] },
                        columns=['name','value'])
    features = pd.concat([features, df_feature])

    return features

def diskstat_profile(logdir, cfg, df, features):
    #diskstat_dev = list(set(df['dev']))
    diskstat_r_q1 = df.groupby('dev')['d_read'].quantile(0.25)
    diskstat_w_q1 = df.groupby('dev')['d_write'].quantile(0.25)
    diskstat_q1 = df.groupby('dev')['d_disk_total'].quantile(0.25)
    diskstat_r_q2 = df.groupby('dev')['d_read'].quantile(0.5)
    diskstat_w_q2 = df.groupby('dev')['d_write'].quantile(0.5)
    diskstat_q2 = df.groupby('dev')['d_disk_total'].quantile(0.5)
    diskstat_r_q3 = df.groupby('dev')['d_read'].quantile(0.75)
    diskstat_w_q3 = df.groupby('dev')['d_write'].quantile(0.75)
    diskstat_q3 = df.groupby('dev')['d_disk_total'].quantile(0.75)
    diskstat_r_avg = df.groupby('dev')['d_read'].mean()
    diskstat_w_avg = df.groupby('dev')['d_write'].mean()
    diskstat_avg = df.groupby('dev')['d_disk_total'].mean()
    diskstat_r_iops = df.groupby('dev')['r_iops'].mean()
    diskstat_w_iops = df.groupby('dev')['w_iops'].mean()
    diskstat_iops = df.groupby('dev')['iops'].mean()
    diskstat_wait = df.groupby('dev')['await_time'].mean()
    diskstat_table = pd.concat([diskstat_r_q1, diskstat_r_q2, diskstat_r_q3, diskstat_r_avg,
                                diskstat_w_q1, diskstat_w_q2, diskstat_w_q3, diskstat_w_avg,
                                diskstat_q1, diskstat_q2, diskstat_q3, diskstat_avg,
                                diskstat_r_iops, diskstat_w_iops, diskstat_iops,
                                diskstat_wait], axis=1, sort=False)
    diskstat_columns = ['Q1 throughput(Read)', 'Q2 throughput(Read)', 'Q3 throughput(Read)', 'Avg throughput(Read)',
                        'Q1 throughput(Write)', 'Q2 throughput(Write)', 'Q3 throughput(Write)', 'Avg throughput(Write)',
                        'Q1 throughput(R+W)', 'Q2 throughput(R+W)', 'Q3 throughput(R+W)', 'Avg throughput(R+W)',
                        'Avg IOPS(Read)', 'Avg IOPS(Write)', 'Avg IOPS(R+W)', 'Avg Await time(ms)']
    diskstat_table.columns = diskstat_columns
    diskstat_dev = diskstat_table.index.format()

    final_table = pd.DataFrame(columns=diskstat_columns)
    for j, dev in enumerate(diskstat_dev):
        tmp_list = []
        for i in diskstat_columns[:-4]:
            tmp_list.append(convertbytes(diskstat_table.iloc[j][i]))
        for i in diskstat_columns[-4:-1]:
            tmp_list.append('%d' % int(diskstat_table.iloc[j][i]))
        tmp_list.append('%.3lf ms' % diskstat_table.iloc[j][-1])
        tmp_table = pd.DataFrame([tuple(tmp_list)],
                                 columns=diskstat_columns,
                                 index=[dev])
        final_table = pd.concat([final_table, tmp_table])
    if cfg.verbose:
        print_title('DISKSTAT Profiling:')
        print('Disk Throughput Quartile :')
        print(final_table.T)

    df_feature = pd.DataFrame({ 'name':['diskstat_q1','diskstat_q2','diskstat_q3'],
                        'value': [diskstat_q1.mean(), diskstat_q2.mean(),  diskstat_q3.mean()] },
                        columns=['name','value'])
    features = pd.concat([features, df_feature])

    return features

def cpu_profile(logdir, cfg, df):
    if cfg.verbose:
        print_title('CPU Profiling:')
        print('elapsed_time (s) = %.6lf' % cfg.elapsed_time)
    grouped_df = df.groupby("deviceId")["duration"]
    total_exec_time = 0
    for key, item in grouped_df:
        print(("[%d]: %lf" % (key, grouped_df.get_group(key).sum())))
        total_exec_time = total_exec_time + grouped_df.get_group(key).sum()

    print("total execution time (s) = %.3lf" % total_exec_time)

    cpu_detail_profile_df = df[['timestamp','duration','name']]
    cpu_detail_profile_df = cpu_detail_profile_df.sort_values(by=['duration'], ascending=False)
    cpu_detail_profile_df['ratio(%)'] = cpu_detail_profile_df['duration']/total_exec_time * 100
    cpu_detail_profile_df = cpu_detail_profile_df[['timestamp','ratio(%)','duration','name']]
    print(cpu_detail_profile_df[:20].to_string(index=False))

def vmstat_profile(logdir, cfg, df, features):
    _,_,_,_,_,_,df['si'],df['so'],df['bi'],df['bo'],df['in'],df['cs'],_,_,_,_,_=df['name'].str.split('|').str
    for col_name in ('si','so','bi','bo','in','cs'):
        df[col_name] = df[col_name].str[3:]
    vmstat_traces = df[['si','so','bi','bo','in','cs']].astype(float)
    vm_bi = vmstat_traces['bi'].mean()
    vm_bo = vmstat_traces['bo'].mean()
    vm_cs = vmstat_traces['cs'].mean()
    vm_in = vmstat_traces['in'].mean()
    if cfg.verbose:
        print_title('VMSTAT Profiling:')
        print('average bi/s: %d' % int(vm_cs))
        print('average bo/s: %d' % int(vm_in))
        print('average cs/s: %d' % int(vm_bi))
        print('average in/s: %d' % int(vm_bo))

    df_feature = pd.DataFrame({ 'name':['vm_bi', 'vm_bo', 'vm_cs', 'vm_in' ],
                        'value':[vm_bi, vm_bo, vm_cs, vm_in] },
                        columns=['name','value'])
    features = pd.concat([features, df_feature])

    return features

def mpstat_profile(logdir, cfg, df, features):
    if not cfg.cluster_ip and cfg.verbose:
        print_title('MPSTAT Profiling:')
    num_cores = int(df['deviceId'].max() + 1)
    df_summary = pd.DataFrame( np.zeros((num_cores,5)), columns=['USR','SYS','IDL','IOW','IRQ'])
    _,_,_,_,_,df['USR'],df['SYS'],df['IDL'],df['IOW'],df['IRQ'],_ = df["name"].str.split('|').str
    df[['USR','SYS','IDL','IOW','IRQ']] = df[['USR','SYS','IDL','IOW','IRQ']].astype(float)
    df["dt_all"] = np.where(df["IDL"]==100, 0.1, df["duration"]/((100-df["IDL"])/100.0))
    df["t_USR"] = df['dt_all'] * df['USR']/100.0
    df["t_SYS"] = df['dt_all'] * df['SYS']/100.0
    df["t_IDL"] = df['dt_all'] * df['IDL']/100.0
    df["t_IOW"] = df['dt_all'] * df['IOW']/100.0
    df["t_IRQ"] = df['dt_all'] * df['IRQ']/100.0
    dfs=[]
    for i in range(num_cores):
        dfs.append(df.loc[df['deviceId'] == float(i)])
    for index,dff in enumerate(dfs):
        df_summary.iloc[index]['USR'] = dff['t_USR'].sum()
        df_summary.iloc[index]['SYS'] = dff['t_SYS'].sum()
        df_summary.iloc[index]['IDL'] = dff['t_IDL'].sum()
        df_summary.iloc[index]['IRQ'] = dff['t_IRQ'].sum()
        df_summary.iloc[index]['IOW'] = dff['t_IOW'].sum()
    if not cfg.cluster_ip and cfg.verbose:
        print('CPU Utilization (%):')
        print('core\tUSR\tSYS\tIDL\tIOW\tIRQ')
    for i in range(len(df_summary)):
        t_sum = df_summary.iloc[i].sum()
        if not cfg.cluster_ip and cfg.verbose:
            print('%3d\t%3d\t%3d\t%3d\t%3d\t%3d'%(i,int(100.0*df_summary.iloc[i]['USR']/t_sum),
                                                    int(100.0*df_summary.iloc[i]['SYS']/t_sum),
                                                    int(100.0*df_summary.iloc[i]['IDL']/t_sum),
                                                    int(100.0*df_summary.iloc[i]['IOW']/t_sum),
                                                    int(100.0*df_summary.iloc[i]['IRQ']/t_sum) ))
    if not cfg.cluster_ip and cfg.verbose:
        print('CPU Time (s):')
        print('core\tUSR\tSYS\tIDL\tIOW\tIRQ')
    for i in range(len(df_summary)):
        t_sum = df_summary.iloc[i].sum()
        if not cfg.cluster_ip and cfg.verbose:
            print('%3d\t%.2lf\t%.2lf\t%.2lf\t%.2lf\t%.2lf'%(i,
                                                    df_summary.iloc[i]['USR'],
                                                    df_summary.iloc[i]['SYS'],
                                                    df_summary.iloc[i]['IDL'],
                                                    df_summary.iloc[i]['IOW'],
                                                    df_summary.iloc[i]['IRQ'] ))

    total_cpu_time = df_summary[['USR','SYS','IRQ']].sum().sum()
    cpu_util = int(100*total_cpu_time / (num_cores*cfg.elapsed_time))
    if not cfg.cluster_ip and cfg.verbose:
        print('Active CPU Time (s): %.3lf' % total_cpu_time)
        print('Active CPU ratio (%%): %3d' % cpu_util)
    df_feature = pd.DataFrame({ 'name':['num_cores', 'cpu_util'],
                        'value':[num_cores, cpu_util] },
                        columns=['name','value'])
    features = pd.concat([features, df_feature])
    return features

def sofa_analyze(cfg):
    print_main_progress('SOFA analyzing...')
    filein = []
    df_cpu = pd.DataFrame([], columns=cfg.columns)
    df_gpu = pd.DataFrame([], columns=cfg.columns)
    df_net = pd.DataFrame([], columns=cfg.columns)
    df_mpstat = pd.DataFrame([], columns=cfg.columns)
    df_vmstat = pd.DataFrame([], columns=cfg.columns)
    df_bandwidth = pd.DataFrame([], columns=cfg.columns)
    df_blktrace = pd.DataFrame([], columns=cfg.columns)
    df_diskstat = pd.DataFrame([], columns=cfg.columns)
    df_nvsmi = pd.DataFrame([], columns=cfg.columns)
    iter_summary = None
    logdir = cfg.logdir

    with open(logdir+'/misc.txt') as f:
        lines = f.readlines()
        elapsed_time = float(lines[0].split()[1])
        vcores = int(lines[2].split()[1])
        cfg.elapsed_time = float(lines[0].split()[1])

    filein_gpu = logdir + "gputrace.csv"
    filein_cpu = logdir + "cputrace.csv"
    filein_net = logdir + "nettrace.csv"
    filein_vmstat = logdir + "vmstat.csv"
    filein_mpstat = logdir + "mpstat.csv"
    filein_strace = logdir + "strace.csv"
    filein_nvsmi = logdir + "nvsmi_trace.csv"
    filein_bandwidth = logdir + "netstat.csv"
    filein_blktrace = logdir + "blktrace.csv"
    filein_diskstat = logdir + "diskstat_vector.csv"

    if os.path.isfile('%s/nvlink_topo.txt' % logdir):

        with open(logdir + 'nvlink_topo.txt') as f:
            lines = f.readlines()
            if len(lines) > 0:
                title = lines[0]
                num_gpus = 1
                for word in title.split():
                    if re.match(r'GPU', word) != None :
                       num_gpus = num_gpus + 1
                print_info(cfg,'# of GPUs: ' + str(num_gpus) )
                edges = []
                if len(lines) >= num_gpus+1:
                    for i in range(num_gpus):
                        connections = lines[1+i].split()
                        for j in range(len(connections)):
                            if connections[j] == 'NV1' or connections[j] == 'NV2':
                                edges.append((i,j-1))
                                #print('%d connects to %d' % (i, j-1))

                    ring_found = False
                    G = nx.DiGraph(edges)
                    # Try to find ring with its length of num_gpus
                    for cycle in nx.simple_cycles(G):
                        if len(cycle) == num_gpus:
                            if cfg.verbose:
                                print('One of the recommended ring having length of %d' % len(cycle))
                            ring_found = True
                            os.system("mkdir -p sofalog/sofa_hints/")
                            xring_order = ','.join(map(str, cycle))
                            with open("sofalog/sofa_hints/xring_order.txt", "w") as f:
                                f.write('export CUDA_VISIBLE_DEVICES=' + xring_order)
                            break

                    # Try to find ring with its length of num_gpus/2
                    if not ring_found:
                        for cycle in nx.simple_cycles(G):
                            if len(cycle) == num_gpus/2:
                                print(("One of the recommended ring having length of %d" % len(cycle) ))
                                ring_found = True
                                os.system("mkdir -p sofalog/sofa_hints/")
                                xring_order = ','.join(map(str, cycle))
                                with open("sofalog/sofa_hints/xring_order.txt", "w") as f:
                                    f.write('export CUDA_VISIBLE_DEVICES=' + xring_order)
                                break
    # Construct Performance Features
    features = pd.DataFrame({'name':['elapsed_time'], 'value':[cfg.elapsed_time]}, columns=['name','value'])

    try:
        df_nvsmi = pd.read_csv(filein_nvsmi)
        if not df_nvsmi.empty and cfg.spotlight_gpu:
            state = 0
            sm_high = 0
            trigger = 10
            for i in range(len(df_nvsmi)):
                if df_nvsmi.iloc[i].event == 0 and df_nvsmi.iloc[i].deviceId == 0 :
                    if df_nvsmi.iloc[i].duration >= 50:
                        sm_high = min(trigger, sm_high + 1)
                    if df_nvsmi.iloc[i].duration < 10:
                        sm_high = max(0, sm_high - 1)
                    if state == 0 and sm_high == trigger:
                        state = 1
                        cfg.roi_begin = df_nvsmi.iloc[i].timestamp
                    elif state == 1 and sm_high == 0:
                        state = 0
                        cfg.roi_end = df_nvsmi.iloc[i].timestamp
                    #print('sm_high=%d state=%d' % (sm_high, state))
            if cfg.roi_end - cfg.roi_begin < 0:
                cfg.roi_end = 0
                cfg.roi_begin = 0
    except IOError:
        print_warning(cfg, "nvsmi_trace.csv is not found")

    try:
        df_cpu = pd.read_csv(filein_cpu)
        if not df_cpu.empty:
            if cfg.verbose:
                cpu_profile(logdir, cfg, df_cpu)
            if cfg.enable_swarms and len(df_cpu) > cfg.num_swarms:
                df_cpu, swarms = hsg_v2(cfg, df_cpu)
    except IOError as e:
        df_cpu = pd.DataFrame([], columns=cfg.columns)
        print_warning(cfg, "%s is not found" % filein_cpu)

    try:
        df_strace = pd.read_csv(filein_strace)
        if not df_strace.empty:
            features = strace_profile(logdir, cfg, df_strace, features)
    except IOError as e:
        df_strace = pd.DataFrame([], columns=cfg.columns)
        print_warning(cfg, "%s is not found" % filein_strace)

    try:
        df_net = pd.read_csv(filein_net)
        if not df_net.empty:
            features = net_profile(logdir, cfg, df_net, features)
    except IOError as e:
        df_net = pd.DataFrame([], columns=cfg.columns)
        print_warning(cfg, "%s is not found" % filein_net)

    try:
        df_bandwidth = pd.read_csv(filein_bandwidth)
        if not df_bandwidth.empty:
            features = netbandwidth_profile(logdir, cfg, df_bandwidth, features)
    except IOError as e:
        df_bandwidth = pd.DataFrame([], columns=cfg.columns)
        print_warning(cfg, "%s is not found" % filein_bandwidth)

    try:
        df_blktrace = pd.read_csv(filein_blktrace)
        if not df_blktrace.empty:
            features = blktrace_latency_profile(logdir, cfg, df_blktrace, features)
    except IOError as e:
        df_blktrace = pd.DataFrame([], columns=cfg.columns)
        print_warning(cfg, "%s is not found" % filein_blktrace)

    try:
        df_diskstat = pd.read_csv(filein_diskstat)
        if not df_diskstat.empty:
            features = diskstat_profile(logdir, cfg, df_diskstat, features)
    except IOError as e:
        df_diskstat = pd.DataFrame([], columns=cfg.columns)
        print_warning(cfg, "%s is not found" % filein_diskstat)

    try:
        df_vmstat = pd.read_csv(filein_vmstat)
        if not df_vmstat.empty:
            features = vmstat_profile(logdir, cfg, df_vmstat, features)
    except IOError as e:
        df_vmstat = pd.DataFrame([], columns=cfg.columns)
        print_warning(cfg, "%s is not found" % filein_vmstat)

    try:
        df_mpstat = pd.read_csv(filein_mpstat)
        if not df_mpstat.empty:
            features = mpstat_profile(logdir, cfg, df_mpstat, features)
    except IOError as e:
        df_mpstat = pd.DataFrame([], columns=cfg.columns)
        print_warning(cfg, "%s is not found" % filein_mpstat)

    try:
        df_nvsmi = pd.read_csv(filein_nvsmi)
        features = nvsmi_profile(logdir, cfg, df_nvsmi, features)
    except IOError:
        print_warning(cfg, "nvsmi_trace.csv is not found")

    try:
        df_gpu = pd.read_csv(filein_gpu)
        if not df_gpu.empty:
            features = gpu_profile(logdir, cfg, df_gpu, features)
    except IOError:
        df_gpu = pd.DataFrame([], columns=cfg.columns)
        print_warning(cfg, "%s is not found. If there is no need to profile GPU, just ignore it." % filein_gpu)

    try:
        if len(df_mpstat)>0:
            df_nvsmi.append(df_mpstat.iloc[0])
            features = concurrency_breakdown(logdir, cfg, df_mpstat, df_cpu, df_gpu, df_nvsmi, df_bandwidth, features)
    except IOError as e:
        print_warning(cfg, "Some files are not found, which are needed for concurrency_breakdown analysis")

    if cfg.enable_aisi:
        selected_pattern, iter_summary, features = sofa_aisi(logdir, cfg, df_cpu, df_gpu, df_strace, df_mpstat, features)

    if 'IS_SOFA_ON_HAIHUB' not in os.environ or os.environ['IS_SOFA_ON_HAIHUB'] == 'no':
            print_title('Final Performance Features')
            print('%s%s%s%s' % ('ID'.ljust(10),'Feature'.ljust(30),'Value'.ljust(20),'Unit'.ljust(20)) )
            for i in range(len(features)):
                name = features.iloc[i]['name']
                value = features.iloc[i]['value']
                print('%s%s%s' % (str(i).ljust(10), name.ljust(30), ('%.3lf'%value).ljust(20)))

    if cfg.spotlight_gpu:
        try:
            print('Elapsed hotspot time: %.3lf' % features[features.name=='elapsed_hotspot_time'].value)
        except:
            print_warning(cfg, 'elpased_hostspot_time is not defined.')

    if cfg.potato_server:
        if cfg.potato_server.find(':') == -1:
            cfg.potato_server = cfg.potato_server + ':50051'
        hint, docker_image = get_hint(cfg.potato_server, features)
        df_report = pd.read_json(hint, orient='table')
        file_potato_report = cfg.logdir + 'potato_report.html'

        # Export report to HTML file.
        df_report.to_html(file_potato_report )
        with open(file_potato_report, 'a') as f:
            f.write('<head><link rel=stylesheet type="text/css" href="potato_report.css"></head>')

        print_title('POTATO Feedback')
        print('%s%s%s%s' % ('ID'.ljust(5), 'Metric'.ljust(20), 'Value'.ljust(10), 'Reference-Value'.ljust(30) ) )
        for i in range(len(df_report)):
            metric = df_report.iloc[i]['Metric']
            if metric != 'hybrid_suggestion':
                value = df_report.iloc[i]['Value']
                ref_value = df_report.iloc[i]['ReferenceValue']
                print('%s%s%s%s' % (str(i).ljust(5), metric.ljust(20), ('%.3lf'%value).ljust(20), str(ref_value).ljust(30)))

        print('\n')
        print_hint('General Suggestions:')
        for i in range(len(df_report)):
            metric = df_report.iloc[i]['Metric']
            if metric != 'hybrid_suggestion':
                suggestion = df_report.iloc[i]['Suggestion']
                print('%d. %s' % (i, suggestion))

        print('\n')
        print_hint('Framework-specific Optimization Suggestions:')
        for i in range(len(df_report)):
            metric = df_report.iloc[i]['Metric']
            if metric == 'hybrid_suggestion':
                suggestion = df_report.iloc[i]['Suggestion']
                print('%d. %s' % (i, suggestion))

        #print(df_report[['Metric', 'Value', 'Reference Value']])
        #print(df_report[['Suggestion']])
        #print('Tag of optimal image recommended from POTATO: ' + highlight(docker_image))
        print('\n')
        print_hint('Please re-launch KubeFlow Jupyter-notebook to have suggested images or resources if necessary.')

    sofa_home = os.path.dirname(os.path.realpath(__file__))
    subprocess.Popen(
        ['bash', '-c', 'cp %s/../sofaboard/* %s;' % (sofa_home, cfg.logdir)])
    subprocess.Popen(['sleep', '2'])
    print('\n\n')
    print('Complete!!')

def cluster_analyze(cfg):
    if cfg.verbose:
        print_title('Cluster Network Profiling :')
    cluster = cfg.cluster_ip.split(',')
    summary_net = pd.DataFrame([], columns=['Source', 'Destination', 'Amount', 'Percentage of a Node'])
    summary_compute = pd.DataFrame([], columns=['gpu_sm_util','gpu_mem_util','cpu_util'])
    summary_band = pd.DataFrame([], columns=['Q1', 'Q2', 'Q3', 'Avg'])
    all = []
    for i, ip in enumerate(cluster):
        features = pd.DataFrame({'name':['elapsed_time'],
                                 'value':[cfg.elapsed_time]},
                                 columns=['name','value'])

        node = 'node ' + str(i)
        if cfg.verbose:
            print('node ' + str(i) + ' is ' + ip)
        logdir = tmp_dir[0:-1] + '-' + ip + '/'
        filein_net = logdir + "nettrace.csv"
        filein_mpstat = logdir + "mpstat.csv"
        filein_nvsmi = logdir + "nvsmi_trace.csv"
        filein_bandwidth = logdir + "netstat.csv"
        with open(logdir+'/misc.txt') as f:
            lines = f.readlines()
            elapsed_time = float(lines[0].split()[1])
            vcores = int(lines[2].split()[1])
            cfg.elapsed_time = float(lines[0].split()[1])
        try:
            df_net = pd.read_csv(filein_net)
            features = net_profile(logdir, cfg, df_net, features)
        except IOError as e:
            df_net = pd.DataFrame([], columns=cfg.columns)
            print_warning(cfg, "%s is not found" % filein_net)
        try:
            df_mpstat = pd.read_csv(filein_mpstat)
            features = mpstat_profile(logdir, cfg, df_mpstat, features)
        except IOError as e:
            df_mpstat = pd.DataFrame([], columns=cfg.columns)
            print_warning(cfg, "%s is not found" % filein_mpstat)
        try:
            df_nvsmi = pd.read_csv(filein_nvsmi)
            features = nvsmi_profile(logdir, cfg, df_nvsmi, features)
        except IOError:
            print_warning(cfg, "nvsmi_trace.csv is not found")
        try:
            df_bandwidth = pd.read_csv(filein_bandwidth)
            features = netbandwidth_profile(logdir, cfg, df_bandwidth, features)
        except IOError as e:
            df_bandwidth = pd.DataFrame([], columns=cfg.columns)
            print_warning(cfg, "%s is not found" % filein_bandwidth)

        sm = int(features[features['name'] == 'gpu_sm_util']['value'])
        mem = int(features[features['name'] == 'gpu_mem_util']['value'])
        cpu = int(features[features['name'] == 'cpu_util']['value'])
        sm_mem_cpu = [sm, mem, cpu]

        compute_tmp = pd.DataFrame([sm_mem_cpu], columns = ['gpu_sm_util', 'gpu_mem_util', 'cpu_util'])
        summary_compute = pd.concat([summary_compute, pd.concat([compute_tmp], keys=[node])])
        net_tmp = pd.read_csv(logdir + "netrank.csv")
        summary_net = pd.concat([summary_net, pd.concat([net_tmp], keys=[node])])

        # for bandwidth report
        tx = df_bandwidth['event'] == float(0)
        rx = df_bandwidth['event'] == float(1)
        tx_tmp = [convertbytes(df_bandwidth[tx]['bandwidth'].quantile(0.25)),
                    convertbytes(df_bandwidth[tx]['bandwidth'].quantile(0.5)),
                    convertbytes(df_bandwidth[tx]['bandwidth'].quantile(0.75)),
                    convertbytes(df_bandwidth[tx]['bandwidth'].mean())]
        rx_tmp = [convertbytes(df_bandwidth[rx]['bandwidth'].quantile(0.25)),
                    convertbytes(df_bandwidth[rx]['bandwidth'].quantile(0.5)),
                    convertbytes(df_bandwidth[rx]['bandwidth'].quantile(0.75)),
                    convertbytes(df_bandwidth[rx]['bandwidth'].mean())]
        band_tmp = pd.DataFrame([tx_tmp], columns = ['Q1', 'Q2', 'Q3', 'Avg'], index = ['tx'])
        rx_pd = pd.DataFrame([rx_tmp], columns = ['Q1', 'Q2', 'Q3', 'Avg'], index = ['rx'])
        band_tmp = pd.concat([band_tmp, rx_pd])
        summary_band = pd.concat([summary_band, pd.concat([band_tmp], keys=[node])])
    if cfg.verbose:
        with pd.option_context('display.max_rows', None, 'display.max_columns', None):  # more options can be specified also
            print('Ranked Network Traffic : \n', summary_net, '\n')
            print('Cluster Bandwidth Quartile: \n', summary_band)
        print_title('Cluster Computation Profiling:')
        print(summary_compute)


from blaze.interactive import Data, compute, concrete_head, expr_repr, to_html

import datetime
from odo import into, append
from odo.backends.csv import CSV
from blaze import discover
from blaze.compute.core import compute
from blaze.compute.python import compute
from blaze.expr import symbol
from datashape import dshape
from blaze.utils import tmpfile, example
from blaze.compatibility import xfail
import pytest
import sys
from types import MethodType

import pandas as pd
import pandas.util.testing as tm
import numpy as np

data = (('Alice', 100),
        ('Bob', 200))

L = [[1, 'Alice',   100],
     [2, 'Bob',    -200],
     [3, 'Charlie', 300],
     [4, 'Denis',   400],
     [5, 'Edith',  -500]]

t = Data(data, fields=['name', 'amount'])

x = np.ones((2, 2))

def test_table_raises_on_inconsistent_inputs():
    with pytest.raises(ValueError):
        t = Data(data, schema='{name: string, amount: float32}',
            dshape=dshape("{name: string, amount: float32}"))

def test_resources():
    assert t._resources() == {t: t.data}

def test_resources_fail():
    t = symbol('t', 'var * {x: int, y: int}')
    d = t[t['x'] > 100]
    with pytest.raises(ValueError):
        compute(d)

def test_compute_on_Data_gives_back_data():
    assert compute(Data([1, 2, 3])) == [1, 2, 3]

def test_len():
    assert len(t) == 2
    assert len(t.name) == 2

def test_compute():
    assert list(compute(t['amount'] + 1)) == [101, 201]

def test_create_with_schema():
    t = Data(data, schema='{name: string, amount: float32}')
    assert t.schema == dshape('{name: string, amount: float32}')

def test_create_with_raw_data():
    t = Data(data, fields=['name', 'amount'])
    assert t.schema == dshape('{name: string, amount: int64}')
    assert t.name
    assert t.data == data

def test_repr():
    result = expr_repr(t['name'])
    print(result)
    assert isinstance(result, str)
    assert 'Alice' in result
    assert 'Bob' in result
    assert '...' not in result

    result = expr_repr(t['amount'] + 1)
    print(result)
    assert '101' in result

    t2 = Data(tuple((i, i**2) for i in range(100)), fields=['x', 'y'])
    assert t2.dshape == dshape('100 * {x: int64, y: int64}')

    result = expr_repr(t2)
    print(result)
    assert len(result.split('\n')) < 20
    assert '...' in result

def test_repr_of_scalar():
    assert repr(t.amount.sum()) == '300'

def test_mutable_backed_repr():
    mutable_backed_table = Data([[0]], fields=['col1'])
    repr(mutable_backed_table)

def test_dataframe_backed_repr():
    df = pd.DataFrame(data=[0], columns=['col1'])
    dataframe_backed_table = Data(df)
    repr(dataframe_backed_table)

def test_dataframe_backed_repr_complex():
    df = pd.DataFrame([(1, 'Alice', 100),
                       (2, 'Bob', -200),
                       (3, 'Charlie', 300),
                       (4, 'Denis', 400),
                       (5, 'Edith', -500)],
                      columns=['id', 'name', 'balance'])
    t = Data(df)
    repr(t[t['balance'] < 0])

def test_repr_html_on_no_resources_symbol():
    t = symbol('t', '5 * {id: int, name: string, balance: int}')
    assert to_html(t) == 't'

def test_expr_repr_empty():
    s = repr(t[t.amount > 1e9])
    assert isinstance(s, str)
    assert 'amount' in s

def test_to_html():
    s = to_html(t)
    assert s
    assert 'Alice' in s
    assert '<table' in s

    assert to_html(1) == '1'

    assert to_html(t.count()) == '2'

def test_to_html_on_arrays():
    s = to_html(Data(np.ones((2, 2))))
    assert '1' in s
    assert 'br>' in s

def test_repr_html():
    assert '<table' in t._repr_html_()
    assert '<table' in t.name._repr_html_()

def test_into():
    assert into(list, t) == into(list, data)

def test_serialization():
    import pickle
    t2 = pickle.loads(pickle.dumps(t))

    assert t.schema == t2.schema
    assert t._name == t2._name

def test_table_resource():
    with tmpfile('csv') as filename:
        ds = dshape('var * {a: int, b: int}')
        csv = CSV(filename)
        append(csv, [[1, 2], [10, 20]], dshape=ds)

        t = Data(filename)
        assert isinstance(t.data, CSV)
        assert into(list, compute(t)) == into(list, csv)

def test_concretehead_failure():
    t = symbol('t', 'var * {x:int, y:int}')
    d = t[t['x'] > 100]
    with pytest.raises(ValueError):
        concrete_head(d)

def test_into_np_ndarray_column():
    t = Data(L, fields=['id', 'name', 'balance'])
    expr = t[t.balance < 0].name
    colarray = into(np.ndarray, expr)
    assert len(list(compute(expr))) == len(colarray)

def test_into_nd_array_selection():
    t = Data(L, fields=['id', 'name', 'balance'])
    expr = t[t['balance'] < 0]
    selarray = into(np.ndarray, expr)
    assert len(list(compute(expr))) == len(selarray)

def test_into_nd_array_column_failure():
    tble = Data(L, fields=['id', 'name', 'balance'])
    expr = tble[tble['balance'] < 0]
    colarray = into(np.ndarray, expr)
    assert len(list(compute(expr))) == len(colarray)

def test_Data_attribute_repr():
    t = Data(CSV(example('accounts-datetimes.csv')))
    result = t.when.day
    expected = pd.DataFrame({'when_day': [1,2,3,4,5]})
    assert repr(result) == repr(expected)

def test_can_trivially_create_csv_Data():
    Data(example('iris.csv'))

    # in context
    with Data(example('iris.csv')) as d:
        assert d is not None

def test_can_trivially_create_csv_Data_with_unicode():
    if sys.version[0] == '2':
        assert isinstance(Data(example(u'iris.csv')).data, CSV)

def test_can_trivially_create_sqlite_table():
    pytest.importorskip('sqlalchemy')
    Data('sqlite:///'+example('iris.db')+'::iris')

    # in context
    with Data('sqlite:///'+example('iris.db')+'::iris') as d:
        assert d is not None

@xfail(reason="h5py/pytables mismatch")
def test_can_trivially_create_pytables():
    pytest.importorskip('tables')
    with Data(example('accounts.h5')+'::/accounts') as d:
        assert d is not None

def test_data_passes_kwargs_to_resource():
    assert Data(example('iris.csv'), encoding='ascii').data.encoding == 'ascii'

def test_data_on_iterator_refies_data():
    data = [1, 2, 3]
    d = Data(iter(data))

    assert into(list, d) == data
    assert into(list, d) == data

    # in context
    with Data(iter(data)) as d:
        assert d is not None

def test_Data_on_json_is_concrete():
    d = Data(example('accounts-streaming.json'))

    assert compute(d.amount.sum()) == 100 - 200 + 300 + 400 - 500
    assert compute(d.amount.sum()) == 100 - 200 + 300 + 400 - 500

def test_repr_on_nd_array_doesnt_err():
    d = Data(np.ones((2, 2, 2)))
    repr(d + 1)

def test_generator_reprs_concretely():
    x = [1, 2, 3, 4, 5, 6]
    d = Data(x)
    expr = d[d > 2] + 1
    assert '4' in repr(expr)

def test_incompatible_types():
    d = Data(pd.DataFrame(L, columns=['id', 'name', 'amount']))

    with pytest.raises(ValueError):
        d.id == 'foo'

    result = compute(d.id == 3)
    expected = pd.Series([False, False, True, False, False], name='id')
    tm.assert_series_equal(result, expected)

def test___array__():
    x = np.ones(4)
    d = Data(x)
    assert (np.array(d + 1) == x + 1).all()

    d = Data(x[:2])
    x[2:] = d + 1
    assert x.tolist() == [1, 1, 2, 2]

def test_python_scalar_protocols():
    d = Data(1)
    assert int(d + 1) == 2
    assert float(d + 1.0) == 2.0
    assert bool(d > 0) is True
    assert complex(d + 1.0j) == 1 + 1.0j

def test_iter():
    x = np.ones(4)
    d = Data(x)
    assert list(d + 1) == [2, 2, 2, 2]

@xfail(reason="DataFrame constructor doesn't yet support __array__")
def test_DataFrame():
    x = np.array([(1, 2), (1., 2.)], dtype=[('a', 'i4'), ('b', 'f4')])
    d = Data(x)
    assert isinstance(pd.DataFrame(d), pd.DataFrame)

def test_head_compute():
    data = tm.makeMixedDataFrame()
    t = symbol('t', discover(data))
    db = into('sqlite:///:memory:::t', data, dshape=t.dshape)
    n = 2
    d = Data(db)

    # skip the header and the ... at the end of the repr
    expr = d.head(n)
    s = repr(expr)
    assert '...' not in s
    result = s.split('\n')[1:]
    assert len(result) == n

def test_scalar_sql_compute():
    t = into('sqlite:///:memory:::t', data,
            dshape=dshape('var * {name: string, amount: int}'))
    d = Data(t)
    assert repr(d.amount.sum()) == '300'

def test_no_name_for_simple_data():
    d = Data([1, 2, 3])
    assert repr(d) == '    \n0  1\n1  2\n2  3'
    assert not d._name

    d = Data(1)
    assert not d._name
    assert repr(d) == '1'

def test_coerce_date_and_datetime():
    x = datetime.datetime.now().date()
    d = Data(x)
    assert repr(d) == repr(x)

    x = datetime.datetime.now()
    d = Data(x)
    assert repr(d) == repr(x)

def test_highly_nested_repr():
    data = [[0, [[1, 2], [3]], 'abc']]
    d = Data(data)
    assert 'abc' in repr(d.head())

def test_asarray_fails_on_different_column_names():
    vs = {'first': [2., 5., 3.],
          'second': [4., 1., 4.],
          'third': [6., 4., 3.]}
    df = pd.DataFrame(vs)
    with pytest.raises(ValueError):
        Data(df, fields=list('abc'))

def test_data_does_not_accept_columns_kwarg():
    with pytest.raises(ValueError):
        Data([(1, 2), (3, 4)], columns=list('ab'))

def test_functions_as_bound_methods():
    """
    Test that all functions on an InteractiveSymbol are instance methods
    of that object.
    """
    # Filter out __class__ and friends that are special, these can be
    # callables without being instance methods.
    callable_attrs = filter(
        callable,
        (getattr(t, a, None) for a in dir(t) if not a.startswith('__')),
    )
    for attr in callable_attrs:
        assert isinstance(attr, MethodType)
        # Make sure this is bound to the correct object.
        assert attr.__self__ is t


"""Caching loader for the 20 newsgroups text classification dataset

The description of the dataset is available on the official website at:

    http://people.csail.mit.edu/jrennie/20Newsgroups/

Quoting the introduction:

    The 20 Newsgroups data set is a collection of approximately 20,000
    newsgroup documents, partitioned (nearly) evenly across 20 different
    newsgroups. To the best of my knowledge, it was originally collected
    by Ken Lang, probably for his Newsweeder: Learning to filter netnews
    paper, though he does not explicitly mention this collection. The 20
    newsgroups collection has become a popular data set for experiments
    in text applications of machine learning techniques, such as text
    classification and text clustering.

This dataset loader will download the recommended "by date" variant of the
dataset and which features a point in time split between the train and
test sets. The compressed dataset size is around 14 Mb compressed. Once
uncompressed the train set is 52 MB and the test set is 34 MB.

The data is downloaded, extracted and cached in the '~/scikit_learn_data'
folder.

The `fetch_20newsgroups` function will not vectorize the data into numpy
arrays but the dataset lists the filenames of the posts and their categories
as target labels.

The `fetch_20newsgroups_tfidf` function will in addition do a simple tf-idf
vectorization step.

"""
# Copyright (c) 2011 Olivier Grisel <olivier.grisel@ensta.org>
# License: BSD 3 clause

import os
import logging
import tarfile
import pickle
import shutil
import re
import codecs

import numpy as np
import scipy.sparse as sp

from .base import get_data_home
from .base import Bunch
from .base import load_files
from ..utils import check_random_state
from ..feature_extraction.text import CountVectorizer
from ..preprocessing import normalize
from ..externals import joblib, six

if six.PY3:
    from urllib.request import urlopen
else:
    from urllib2 import urlopen

logger = logging.getLogger(__name__)

URL = ("http://people.csail.mit.edu/jrennie/"
       "20Newsgroups/20news-bydate.tar.gz")
ARCHIVE_NAME = "20news-bydate.tar.gz"
CACHE_NAME = "20news-bydate.pkz"
TRAIN_FOLDER = "20news-bydate-train"
TEST_FOLDER = "20news-bydate-test"

def download_20newsgroups(target_dir, cache_path):
    """Download the 20 newsgroups data and stored it as a zipped pickle."""
    archive_path = os.path.join(target_dir, ARCHIVE_NAME)
    train_path = os.path.join(target_dir, TRAIN_FOLDER)
    test_path = os.path.join(target_dir, TEST_FOLDER)

    if not os.path.exists(target_dir):
        os.makedirs(target_dir)

    if os.path.exists(archive_path):
        # Download is not complete as the .tar.gz file is removed after
        # download.
        logger.warning("Download was incomplete, downloading again.")
        os.remove(archive_path)

    logger.warning("Downloading dataset from %s (14 MB)", URL)
    opener = urlopen(URL)
    with open(archive_path, 'wb') as f:
        f.write(opener.read())

    logger.info("Decompressing %s", archive_path)
    tarfile.open(archive_path, "r:gz").extractall(path=target_dir)
    os.remove(archive_path)

    # Store a zipped pickle
    cache = dict(train=load_files(train_path, encoding='latin1'),
                 test=load_files(test_path, encoding='latin1'))
    compressed_content = codecs.encode(pickle.dumps(cache), 'zlib_codec')
    with open(cache_path, 'wb') as f:
        f.write(compressed_content)

    shutil.rmtree(target_dir)
    return cache

def strip_newsgroup_header(text):
    """
    Given text in "news" format, strip the headers, by removing everything
    before the first blank line.
    """
    _before, _blankline, after = text.partition('\n\n')
    return after

_QUOTE_RE = re.compile(r'(writes in|writes:|wrote:|says:|said:'
                       r'|^In article|^Quoted from|^\||^>)')

def strip_newsgroup_quoting(text):
    """
    Given text in "news" format, strip lines beginning with the quote
    characters > or |, plus lines that often introduce a quoted section
    (for example, because they contain the string 'writes:'.)
    """
    good_lines = [line for line in text.split('\n')
                  if not _QUOTE_RE.search(line)]
    return '\n'.join(good_lines)

def strip_newsgroup_footer(text):
    """
    Given text in "news" format, attempt to remove a signature block.

    As a rough heuristic, we assume that signatures are set apart by either
    a blank line or a line made of hyphens, and that it is the last such line
    in the file (disregarding blank lines at the end).
    """
    lines = text.strip().split('\n')
    for line_num in range(len(lines) - 1, -1, -1):
        line = lines[line_num]
        if line.strip().strip('-') == '':
            break

    if line_num > 0:
        return '\n'.join(lines[:line_num])
    else:
        return text

def fetch_20newsgroups(data_home=None, subset='train', categories=None,
                       shuffle=True, random_state=42,
                       remove=(),
                       download_if_missing=True):
    """Load the filenames and data from the 20 newsgroups dataset.

    Read more in the :ref:`User Guide <20newsgroups>`.

    Parameters
    ----------
    subset: 'train' or 'test', 'all', optional
        Select the dataset to load: 'train' for the training set, 'test'
        for the test set, 'all' for both, with shuffled ordering.

    data_home: optional, default: None
        Specify a download and cache folder for the datasets. If None,
        all scikit-learn data is stored in '~/scikit_learn_data' subfolders.

    categories: None or collection of string or unicode
        If None (default), load all the categories.
        If not None, list of category names to load (other categories
        ignored).

    shuffle: bool, optional
        Whether or not to shuffle the data: might be important for models that
        make the assumption that the samples are independent and identically
        distributed (i.i.d.), such as stochastic gradient descent.

    random_state: numpy random number generator or seed integer
        Used to shuffle the dataset.

    download_if_missing: optional, True by default
        If False, raise an IOError if the data is not locally available
        instead of trying to download the data from the source site.

    remove: tuple
        May contain any subset of ('headers', 'footers', 'quotes'). Each of
        these are kinds of text that will be detected and removed from the
        newsgroup posts, preventing classifiers from overfitting on
        metadata.

        'headers' removes newsgroup headers, 'footers' removes blocks at the
        ends of posts that look like signatures, and 'quotes' removes lines
        that appear to be quoting another post.

        'headers' follows an exact standard; the other filters are not always
        correct.
    """

    data_home = get_data_home(data_home=data_home)
    cache_path = os.path.join(data_home, CACHE_NAME)
    twenty_home = os.path.join(data_home, "20news_home")
    cache = None
    if os.path.exists(cache_path):
        try:
            with open(cache_path, 'rb') as f:
                compressed_content = f.read()
            uncompressed_content = codecs.decode(
                compressed_content, 'zlib_codec')
            cache = pickle.loads(uncompressed_content)
        except Exception as e:
            print(80 * '_')
            print('Cache loading failed')
            print(80 * '_')
            print(e)

    if cache is None:
        if download_if_missing:
            cache = download_20newsgroups(target_dir=twenty_home,
                                          cache_path=cache_path)
        else:
            raise IOError('20Newsgroups dataset not found')

    if subset in ('train', 'test'):
        data = cache[subset]
    elif subset == 'all':
        data_lst = list()
        target = list()
        filenames = list()
        for subset in ('train', 'test'):
            data = cache[subset]
            data_lst.extend(data.data)
            target.extend(data.target)
            filenames.extend(data.filenames)

        data.data = data_lst
        data.target = np.array(target)
        data.filenames = np.array(filenames)
    else:
        raise ValueError(
            "subset can only be 'train', 'test' or 'all', got '%s'" % subset)

    data.description = 'the 20 newsgroups by date dataset'

    if 'headers' in remove:
        data.data = [strip_newsgroup_header(text) for text in data.data]
    if 'footers' in remove:
        data.data = [strip_newsgroup_footer(text) for text in data.data]
    if 'quotes' in remove:
        data.data = [strip_newsgroup_quoting(text) for text in data.data]

    if categories is not None:
        labels = [(data.target_names.index(cat), cat) for cat in categories]
        # Sort the categories to have the ordering of the labels
        labels.sort()
        labels, categories = zip(*labels)
        mask = np.in1d(data.target, labels)
        data.filenames = data.filenames[mask]
        data.target = data.target[mask]
        # searchsorted to have continuous labels
        data.target = np.searchsorted(labels, data.target)
        data.target_names = list(categories)
        # Use an object array to shuffle: avoids memory copy
        data_lst = np.array(data.data, dtype=object)
        data_lst = data_lst[mask]
        data.data = data_lst.tolist()

    if shuffle:
        random_state = check_random_state(random_state)
        indices = np.arange(data.target.shape[0])
        random_state.shuffle(indices)
        data.filenames = data.filenames[indices]
        data.target = data.target[indices]
        # Use an object array to shuffle: avoids memory copy
        data_lst = np.array(data.data, dtype=object)
        data_lst = data_lst[indices]
        data.data = data_lst.tolist()

    return data

def fetch_20newsgroups_vectorized(subset="train", remove=(), data_home=None):
    """Load the 20 newsgroups dataset and transform it into tf-idf vectors.

    This is a convenience function; the tf-idf transformation is done using the
    default settings for `sklearn.feature_extraction.text.Vectorizer`. For more
    advanced usage (stopword filtering, n-gram extraction, etc.), combine
    fetch_20newsgroups with a custom `Vectorizer` or `CountVectorizer`.

    Read more in the :ref:`User Guide <20newsgroups>`.

    Parameters
    ----------

    subset: 'train' or 'test', 'all', optional
        Select the dataset to load: 'train' for the training set, 'test'
        for the test set, 'all' for both, with shuffled ordering.

    data_home: optional, default: None
        Specify an download and cache folder for the datasets. If None,
        all scikit-learn data is stored in '~/scikit_learn_data' subfolders.

    remove: tuple
        May contain any subset of ('headers', 'footers', 'quotes'). Each of
        these are kinds of text that will be detected and removed from the
        newsgroup posts, preventing classifiers from overfitting on
        metadata.

        'headers' removes newsgroup headers, 'footers' removes blocks at the
        ends of posts that look like signatures, and 'quotes' removes lines
        that appear to be quoting another post.

    Returns
    -------

    bunch : Bunch object
        bunch.data: sparse matrix, shape [n_samples, n_features]
        bunch.target: array, shape [n_samples]
        bunch.target_names: list, length [n_classes]
    """
    data_home = get_data_home(data_home=data_home)
    filebase = '20newsgroup_vectorized'
    if remove:
        filebase += 'remove-' + ('-'.join(remove))
    target_file = os.path.join(data_home, filebase + ".pk")

    # we shuffle but use a fixed seed for the memoization
    data_train = fetch_20newsgroups(data_home=data_home,
                                    subset='train',
                                    categories=None,
                                    shuffle=True,
                                    random_state=12,
                                    remove=remove)

    data_test = fetch_20newsgroups(data_home=data_home,
                                   subset='test',
                                   categories=None,
                                   shuffle=True,
                                   random_state=12,
                                   remove=remove)

    if os.path.exists(target_file):
        X_train, X_test = joblib.load(target_file)
    else:
        vectorizer = CountVectorizer(dtype=np.int16)
        X_train = vectorizer.fit_transform(data_train.data).tocsr()
        X_test = vectorizer.transform(data_test.data).tocsr()
        joblib.dump((X_train, X_test), target_file, compress=9)

    # the data is stored as int16 for compactness
    # but normalize needs floats
    X_train = X_train.astype(np.float64)
    X_test = X_test.astype(np.float64)
    normalize(X_train, copy=False)
    normalize(X_test, copy=False)

    target_names = data_train.target_names

    if subset == "train":
        data = X_train
        target = data_train.target
    elif subset == "test":
        data = X_test
        target = data_test.target
    elif subset == "all":
        data = sp.vstack((X_train, X_test)).tocsr()
        target = np.concatenate((data_train.target, data_test.target))
    else:
        raise ValueError("%r is not a valid subset: should be one of "
                         "['train', 'test', 'all']" % subset)

    return Bunch(data=data, target=target, target_names=target_names)


"""
==============================================
Plot randomly generated classification dataset
==============================================

Plot several randomly generated 2D classification datasets.
This example illustrates the :func:`datasets.make_classification`
:func:`datasets.make_blobs` and :func:`datasets.make_gaussian_quantiles`
functions.

For ``make_classification``, three binary and two multi-class classification
datasets are generated, with different numbers of informative features and
clusters per class.  """

print(__doc__)

import matplotlib.pyplot as plt

from sklearn.datasets import make_classification
from sklearn.datasets import make_blobs
from sklearn.datasets import make_gaussian_quantiles

plt.figure(figsize=(8, 8))
plt.subplots_adjust(bottom=.05, top=.9, left=.05, right=.95)

plt.subplot(321)
plt.title("One informative feature, one cluster per class", fontsize='small')
X1, Y1 = make_classification(n_features=2, n_redundant=0, n_informative=1,
                             n_clusters_per_class=1)
plt.scatter(X1[:, 0], X1[:, 1], marker='o', c=Y1)

plt.subplot(322)
plt.title("Two informative features, one cluster per class", fontsize='small')
X1, Y1 = make_classification(n_features=2, n_redundant=0, n_informative=2,
                             n_clusters_per_class=1)
plt.scatter(X1[:, 0], X1[:, 1], marker='o', c=Y1)

plt.subplot(323)
plt.title("Two informative features, two clusters per class", fontsize='small')
X2, Y2 = make_classification(n_features=2, n_redundant=0, n_informative=2)
plt.scatter(X2[:, 0], X2[:, 1], marker='o', c=Y2)

plt.subplot(324)
plt.title("Multi-class, two informative features, one cluster",
          fontsize='small')
X1, Y1 = make_classification(n_features=2, n_redundant=0, n_informative=2,
                             n_clusters_per_class=1, n_classes=3)
plt.scatter(X1[:, 0], X1[:, 1], marker='o', c=Y1)

plt.subplot(325)
plt.title("Three blobs", fontsize='small')
X1, Y1 = make_blobs(n_features=2, centers=3)
plt.scatter(X1[:, 0], X1[:, 1], marker='o', c=Y1)

plt.subplot(326)
plt.title("Gaussian divided into three quantiles", fontsize='small')
X1, Y1 = make_gaussian_quantiles(n_features=2, n_classes=3)
plt.scatter(X1[:, 0], X1[:, 1], marker='o', c=Y1)

plt.show()


import numpy as np
import scipy.sparse
from sklearn import datasets, svm, linear_model
from numpy.testing import assert_array_almost_equal, \
     assert_array_equal, assert_equal

from nose.tools import assert_raises
from sklearn.datasets.samples_generator import make_classification
from . import test_svm

# test sample 1
X = np.array([[-2, -1], [-1, -1], [-1, -2], [1, 1], [1, 2], [2, 1]])
Y = [1, 1, 1, 2, 2, 2]
T = np.array([[-1, -1], [2, 2], [3, 2]])
true_result = [1, 2, 2]

# test sample 2
X2 = np.array([[0, 0, 0], [1, 1, 1], [2, 0, 0, ],
               [0, 0, 2], [3, 3, 3]])
Y2 = [1, 2, 2, 2, 3]
T2 = np.array([[-1, -1, -1], [1, 1, 1], [2, 2, 2]])
true_result2 = [1, 2, 3]

iris = datasets.load_iris()
# permute
perm = np.random.permutation(iris.target.size)
iris.data = iris.data[perm]
iris.target = iris.target[perm]
# sparsify
iris.data = scipy.sparse.csr_matrix(iris.data)

def test_SVC():
    """Check that sparse SVC gives the same result as SVC"""

    clf = svm.SVC(kernel='linear').fit(X, Y)
    sp_clf = svm.sparse.SVC(kernel='linear').fit(X, Y)

    assert_array_equal(sp_clf.predict(T), true_result)

    assert scipy.sparse.issparse(sp_clf.support_vectors_)
    assert_array_almost_equal(clf.support_vectors_, sp_clf.support_vectors_.todense())

    assert scipy.sparse.issparse(sp_clf.dual_coef_)
    assert_array_almost_equal(clf.dual_coef_, sp_clf.dual_coef_.todense())

    assert scipy.sparse.issparse(sp_clf.coef_)
    assert_array_almost_equal(clf.coef_, sp_clf.coef_.todense())
    assert_array_almost_equal(clf.predict(T), sp_clf.predict(T))

    # refit with a different dataset
    clf.fit(X2, Y2)
    sp_clf.fit(X2, Y2)
    assert_array_almost_equal(clf.support_vectors_, sp_clf.support_vectors_.todense())
    assert_array_almost_equal(clf.dual_coef_, sp_clf.dual_coef_.todense())
    assert_array_almost_equal(clf.coef_, sp_clf.coef_.todense())
    assert_array_almost_equal(clf.predict(T2), sp_clf.predict(T2))

def test_SVC_iris():
    """Test the sparse SVC with the iris dataset"""
    for k in ('linear', 'poly', 'rbf'):
        sp_clf = svm.sparse.SVC(kernel=k).fit(iris.data, iris.target)
        clf = svm.SVC(kernel=k).fit(iris.data.todense(), iris.target)

        assert_array_almost_equal(clf.support_vectors_, sp_clf.support_vectors_.todense())
        assert_array_almost_equal(clf.dual_coef_, sp_clf.dual_coef_.todense())
        assert_array_almost_equal(
            clf.predict(iris.data.todense()), sp_clf.predict(iris.data))
        if k == 'linear':
            assert_array_almost_equal(clf.coef_, sp_clf.coef_.todense())

def test_error():
    """
    Test that it gives proper exception on deficient input
    """
    # impossible value of C
    assert_raises(ValueError, svm.SVC(C=-1).fit, X, Y)

    # impossible value of nu
    clf = svm.sparse.NuSVC(nu=0.0)
    assert_raises(ValueError, clf.fit, X, Y)

    Y2 = Y[:-1]  # wrong dimensions for labels
    assert_raises(ValueError, clf.fit, X, Y2)

    clf = svm.sparse.SVC()
    clf.fit(X, Y)
    assert_array_equal(clf.predict(T), true_result)

def test_LinearSVC():
    """
    Similar to test_SVC
    """
    clf = svm.LinearSVC().fit(X, Y)
    sp_clf = svm.sparse.LinearSVC().fit(X, Y)

    assert sp_clf.fit_intercept

    assert_array_almost_equal(clf.raw_coef_, sp_clf.raw_coef_, decimal=4)

    assert_array_almost_equal(clf.predict(X), sp_clf.predict(X))

    clf.fit(X2, Y2)
    sp_clf.fit(X2, Y2)

    assert_array_almost_equal(clf.raw_coef_, sp_clf.raw_coef_, decimal=4)

def test_LinearSVC_iris():
    """Test the sparse LinearSVC with the iris dataset"""

    sp_clf = svm.sparse.LinearSVC().fit(iris.data, iris.target)
    clf = svm.LinearSVC().fit(iris.data.todense(), iris.target)

    assert_array_almost_equal(clf.label_, sp_clf.label_)
    assert_equal(clf.fit_intercept, sp_clf.fit_intercept)

    assert_array_almost_equal(clf.raw_coef_, sp_clf.raw_coef_, decimal=1)
    assert_array_almost_equal(
        clf.predict(iris.data.todense()), sp_clf.predict(iris.data))

    # check decision_function
    pred = np.argmax(sp_clf.decision_function(iris.data), 1)
    assert_array_almost_equal(pred, clf.predict(iris.data.todense()))

def test_weight():
    """
    Test class weights
    """

    X_, y_ = make_classification(n_samples=200, n_features=100,
                                 weights=[0.833, 0.167], random_state=0)

    X_ = scipy.sparse.csr_matrix(X_)
    for clf in (linear_model.sparse.LogisticRegression(),
                svm.sparse.LinearSVC(),
                svm.sparse.SVC()):
        clf.fit(X_[:180], y_[:180], class_weight={0: 5})
        y_pred = clf.predict(X_[180:])
        assert np.sum(y_pred == y_[180:]) >= 11

def test_sample_weights():
    """
    Test weights on individual samples
    """
    clf = svm.sparse.SVC()
    clf.fit(X, Y)
    assert_array_equal(clf.predict(X[2]), [1.])

    sample_weight = [.1] * 3 + [10] * 3
    clf.fit(X, Y, sample_weight=sample_weight)
    assert_array_equal(clf.predict(X[2]), [2.])

def test_sparse_liblinear_intercept_handling():
    """
    Test that sparse liblinear honours intercept_scaling param
    """
    test_svm.test_dense_liblinear_intercept_handling(svm.sparse.LinearSVC)

def test_sparse_realdata():
    """
    Test on a subset from the 20newsgroups dataset.

    This catchs some bugs if input is not correctly converted into
    sparse format or weights are not correctly initialized.
    """

    data = np.array([ 0.03771744,  0.1003567,  0.01174647,  0.027069  ])
    indices = np.array([6, 5, 35, 31])
    indptr = np.array(
        [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2,
         2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
         2, 2, 2, 2, 2, 2, 2, 2, 2, 4, 4, 4])
    X = scipy.sparse.csr_matrix((data, indices, indptr))
    y = np.array(
        [ 1.,  0.,  2.,  2.,  1.,  1.,  1.,  2.,  2.,  0.,  1.,  2.,  2.,
        0.,  2.,  0.,  3.,  0.,  3.,  0.,  1.,  1.,  3.,  2.,  3.,  2.,
        0.,  3.,  1.,  0.,  2.,  1.,  2.,  0.,  1.,  0.,  2.,  3.,  1.,
        3.,  0.,  1.,  0.,  0.,  2.,  0.,  1.,  2.,  2.,  2.,  3.,  2.,
        0.,  3.,  2.,  1.,  2.,  3.,  2.,  2.,  0.,  1.,  0.,  1.,  2.,
        3.,  0.,  0.,  2.,  2.,  1.,  3.,  1.,  1.,  0.,  1.,  2.,  1.,
        1.,  3.])

    clf = svm.SVC(kernel='linear').fit(X.todense(), y)
    sp_clf = svm.sparse.SVC(kernel='linear').fit(X, y)

    assert_array_equal(clf.support_vectors_, sp_clf.support_vectors_.todense())
    assert_array_equal(clf.dual_coef_, sp_clf.dual_coef_.todense())

if __name__ == '__main__':
    import nose
    nose.runmodule()