#    Copyright (C) 2004-2016 by
#    Aric Hagberg <hagberg@lanl.gov>
#    Dan Schult <dschult@colgate.edu>
#    Pieter Swart <swart@lanl.gov>
#    All rights reserved.
#    BSD license.
#
# Author: Aric Hagberg (hagberg@lanl.gov)
"""
**********
Matplotlib
**********

Draw networks with matplotlib.

See Also
--------

matplotlib:     http://matplotlib.org/

pygraphviz:     http://pygraphviz.github.io/

"""
import networkx as nx
from networkx.drawing.layout import shell_layout,\
    circular_layout,spectral_layout,spring_layout,random_layout

__all__ = ['draw',
           'draw_networkx',
           'draw_networkx_nodes',
           'draw_networkx_edges',
           'draw_networkx_labels',
           'draw_networkx_edge_labels',
           'draw_circular',
           'draw_random',
           'draw_spectral',
           'draw_spring',
           'draw_shell']

def draw(G, pos=None, ax=None, **kwds):
    """Draw the graph G with Matplotlib.

    Draw the graph as a simple representation with no node
    labels or edge labels and using the full Matplotlib figure area
    and no axis labels by default.  See draw_networkx() for more
    full-featured drawing that allows title, axis labels etc.

    Parameters
    ----------
    G : graph
       A networkx graph

    pos : dictionary, optional
       A dictionary with nodes as keys and positions as values.
       If not specified a spring layout positioning will be computed.
       See :py:mod:`networkx.drawing.layout` for functions that
       compute node positions.

    ax : Matplotlib Axes object, optional
       Draw the graph in specified Matplotlib axes.

    kwds : optional keywords
       See networkx.draw_networkx() for a description of optional keywords.

    Examples
    --------
    >>> G=nx.dodecahedral_graph()
    >>> nx.draw(G)
    >>> nx.draw(G,pos=nx.spring_layout(G)) # use spring layout

    See Also
    --------
    draw_networkx()
    draw_networkx_nodes()
    draw_networkx_edges()
    draw_networkx_labels()
    draw_networkx_edge_labels()

    Notes
    -----
    This function has the same name as pylab.draw and pyplot.draw
    so beware when using

    >>> from networkx import *

    since you might overwrite the pylab.draw function.

    With pyplot use

    >>> import matplotlib.pyplot as plt
    >>> import networkx as nx
    >>> G=nx.dodecahedral_graph()
    >>> nx.draw(G)  # networkx draw()
    >>> plt.draw()  # pyplot draw()

    Also see the NetworkX drawing examples at
    http://networkx.readthedocs.io/en/latest/gallery.html
    """
    try:
        import matplotlib.pyplot as plt
    except ImportError:
        raise ImportError("Matplotlib required for draw()")
    except RuntimeError:
        print("Matplotlib unable to open display")
        raise

    if ax is None:
        cf = plt.gcf()
    else:
        cf = ax.get_figure()
    cf.set_facecolor('w')
    if ax is None:
        if cf._axstack() is None:
            ax = cf.add_axes((0, 0, 1, 1))
        else:
            ax = cf.gca()

    if 'with_labels' not in kwds:
        kwds['with_labels'] = 'labels' in kwds

    try:
        draw_networkx(G, pos=pos, ax=ax, **kwds)
        ax.set_axis_off()
        plt.draw_if_interactive()
    except:
        raise
    return

def draw_networkx(G, pos=None, arrows=True, with_labels=True, **kwds):
    """Draw the graph G using Matplotlib.

    Draw the graph with Matplotlib with options for node positions,
    labeling, titles, and many other drawing features.
    See draw() for simple drawing without labels or axes.

    Parameters
    ----------
    G : graph
       A networkx graph

    pos : dictionary, optional
       A dictionary with nodes as keys and positions as values.
       If not specified a spring layout positioning will be computed.
       See :py:mod:`networkx.drawing.layout` for functions that
       compute node positions.

    arrows : bool, optional (default=True)
       For directed graphs, if True draw arrowheads.

    with_labels :  bool, optional (default=True)
       Set to True to draw labels on the nodes.

    ax : Matplotlib Axes object, optional
       Draw the graph in the specified Matplotlib axes.

    nodelist : list, optional (default G.nodes())
       Draw only specified nodes

    edgelist : list, optional (default=G.edges())
       Draw only specified edges

    node_size : scalar or array, optional (default=300)
       Size of nodes.  If an array is specified it must be the
       same length as nodelist.

    node_color : color string, or array of floats, (default='r')
       Node color. Can be a single color format string,
       or a  sequence of colors with the same length as nodelist.
       If numeric values are specified they will be mapped to
       colors using the cmap and vmin,vmax parameters.  See
       matplotlib.scatter for more details.

    node_shape :  string, optional (default='o')
       The shape of the node.  Specification is as matplotlib.scatter
       marker, one of 'so^>v<dph8'.

    alpha : float, optional (default=1.0)
       The node and edge transparency

    cmap : Matplotlib colormap, optional (default=None)
       Colormap for mapping intensities of nodes

    vmin,vmax : float, optional (default=None)
       Minimum and maximum for node colormap scaling

    linewidths : [None | scalar | sequence]
       Line width of symbol border (default =1.0)

    width : float, optional (default=1.0)
       Line width of edges

    edge_color : color string, or array of floats (default='r')
       Edge color. Can be a single color format string,
       or a sequence of colors with the same length as edgelist.
       If numeric values are specified they will be mapped to
       colors using the edge_cmap and edge_vmin,edge_vmax parameters.

    edge_cmap : Matplotlib colormap, optional (default=None)
       Colormap for mapping intensities of edges

    edge_vmin,edge_vmax : floats, optional (default=None)
       Minimum and maximum for edge colormap scaling

    style : string, optional (default='solid')
       Edge line style (solid|dashed|dotted,dashdot)

    labels : dictionary, optional (default=None)
       Node labels in a dictionary keyed by node of text labels

    font_size : int, optional (default=12)
       Font size for text labels

    font_color : string, optional (default='k' black)
       Font color string

    font_weight : string, optional (default='normal')
       Font weight

    font_family : string, optional (default='sans-serif')
       Font family

    label : string, optional
        Label for graph legend

    Notes
    -----
    For directed graphs, "arrows" (actually just thicker stubs) are drawn
    at the head end.  Arrows can be turned off with keyword arrows=False.
    Yes, it is ugly but drawing proper arrows with Matplotlib this
    way is tricky.

    Examples
    --------
    >>> G=nx.dodecahedral_graph()
    >>> nx.draw(G)
    >>> nx.draw(G,pos=nx.spring_layout(G)) # use spring layout

    >>> import matplotlib.pyplot as plt
    >>> limits=plt.axis('off') # turn of axis

    Also see the NetworkX drawing examples at
    http://networkx.readthedocs.io/en/latest/gallery.html

    See Also
    --------
    draw()
    draw_networkx_nodes()
    draw_networkx_edges()
    draw_networkx_labels()
    draw_networkx_edge_labels()
    """
    try:
        import matplotlib.pyplot as plt
    except ImportError:
        raise ImportError("Matplotlib required for draw()")
    except RuntimeError:
        print("Matplotlib unable to open display")
        raise

    if pos is None:
        pos = nx.drawing.spring_layout(G)  # default to spring layout

    node_collection = draw_networkx_nodes(G, pos, **kwds)
    edge_collection = draw_networkx_edges(G, pos, arrows=arrows, **kwds)
    if with_labels:
        draw_networkx_labels(G, pos, **kwds)
    plt.draw_if_interactive()

def draw_networkx_nodes(G, pos,
                        nodelist=None,
                        node_size=300,
                        node_color='r',
                        node_shape='o',
                        alpha=1.0,
                        cmap=None,
                        vmin=None,
                        vmax=None,
                        ax=None,
                        linewidths=None,
                        label=None,
                        **kwds):
    """Draw the nodes of the graph G.

    This draws only the nodes of the graph G.

    Parameters
    ----------
    G : graph
       A networkx graph

    pos : dictionary
       A dictionary with nodes as keys and positions as values.
       Positions should be sequences of length 2.

    ax : Matplotlib Axes object, optional
       Draw the graph in the specified Matplotlib axes.

    nodelist : list, optional
       Draw only specified nodes (default G.nodes())

    node_size : scalar or array
       Size of nodes (default=300).  If an array is specified it must be the
       same length as nodelist.

    node_color : color string, or array of floats
       Node color. Can be a single color format string (default='r'),
       or a  sequence of colors with the same length as nodelist.
       If numeric values are specified they will be mapped to
       colors using the cmap and vmin,vmax parameters.  See
       matplotlib.scatter for more details.

    node_shape :  string
       The shape of the node.  Specification is as matplotlib.scatter
       marker, one of 'so^>v<dph8' (default='o').

    alpha : float
       The node transparency (default=1.0)

    cmap : Matplotlib colormap
       Colormap for mapping intensities of nodes (default=None)

    vmin,vmax : floats
       Minimum and maximum for node colormap scaling (default=None)

    linewidths : [None | scalar | sequence]
       Line width of symbol border (default =1.0)

    label : [None| string]
       Label for legend

    Returns
    -------
    matplotlib.collections.PathCollection
        `PathCollection` of the nodes.

    Examples
    --------
    >>> G=nx.dodecahedral_graph()
    >>> nodes=nx.draw_networkx_nodes(G,pos=nx.spring_layout(G))

    Also see the NetworkX drawing examples at
    http://networkx.readthedocs.io/en/latest/gallery.html

    See Also
    --------
    draw()
    draw_networkx()
    draw_networkx_edges()
    draw_networkx_labels()
    draw_networkx_edge_labels()
    """
    import collections
    try:
        import matplotlib.pyplot as plt
        import numpy
    except ImportError:
        raise ImportError("Matplotlib required for draw()")
    except RuntimeError:
        print("Matplotlib unable to open display")
        raise

    if ax is None:
        ax = plt.gca()

    if nodelist is None:
        nodelist = list(G)

    if not nodelist or len(nodelist) == 0:  # empty nodelist, no drawing
        return None

    try:
        xy = numpy.asarray([pos[v] for v in nodelist])
    except KeyError as e:
        raise nx.NetworkXError('Node %s has no position.'%e)
    except ValueError:
        raise nx.NetworkXError('Bad value in node positions.')

    if isinstance(alpha, collections.Iterable):
        node_color = apply_alpha(node_color, alpha, nodelist, cmap, vmin, vmax)
        alpha = None

    node_collection = ax.scatter(xy[:, 0], xy[:, 1],
                                 s=node_size,
                                 c=node_color,
                                 marker=node_shape,
                                 cmap=cmap,
                                 vmin=vmin,
                                 vmax=vmax,
                                 alpha=alpha,
                                 linewidths=linewidths,
                                 label=label)

    node_collection.set_zorder(2)
    return node_collection

def draw_networkx_edges(G, pos,
                        edgelist=None,
                        width=1.0,
                        edge_color='k',
                        style='solid',
                        alpha=1.0,
                        edge_cmap=None,
                        edge_vmin=None,
                        edge_vmax=None,
                        ax=None,
                        arrows=True,
                        label=None,
                        **kwds):
    """Draw the edges of the graph G.

    This draws only the edges of the graph G.

    Parameters
    ----------
    G : graph
       A networkx graph

    pos : dictionary
       A dictionary with nodes as keys and positions as values.
       Positions should be sequences of length 2.

    edgelist : collection of edge tuples
       Draw only specified edges(default=G.edges())

    width : float, or array of floats
       Line width of edges (default=1.0)

    edge_color : color string, or array of floats
       Edge color. Can be a single color format string (default='r'),
       or a sequence of colors with the same length as edgelist.
       If numeric values are specified they will be mapped to
       colors using the edge_cmap and edge_vmin,edge_vmax parameters.

    style : string
       Edge line style (default='solid') (solid|dashed|dotted,dashdot)

    alpha : float
       The edge transparency (default=1.0)

    edge_ cmap : Matplotlib colormap
       Colormap for mapping intensities of edges (default=None)

    edge_vmin,edge_vmax : floats
       Minimum and maximum for edge colormap scaling (default=None)

    ax : Matplotlib Axes object, optional
       Draw the graph in the specified Matplotlib axes.

    arrows : bool, optional (default=True)
       For directed graphs, if True draw arrowheads.

    label : [None| string]
       Label for legend

    Returns
    -------
    matplotlib.collection.LineCollection
        `LineCollection` of the edges

    Notes
    -----
    For directed graphs, "arrows" (actually just thicker stubs) are drawn
    at the head end.  Arrows can be turned off with keyword arrows=False.
    Yes, it is ugly but drawing proper arrows with Matplotlib this
    way is tricky.

    Examples
    --------
    >>> G=nx.dodecahedral_graph()
    >>> edges=nx.draw_networkx_edges(G,pos=nx.spring_layout(G))

    Also see the NetworkX drawing examples at
    http://networkx.readthedocs.io/en/latest/gallery.html

    See Also
    --------
    draw()
    draw_networkx()
    draw_networkx_nodes()
    draw_networkx_labels()
    draw_networkx_edge_labels()
    """
    try:
        import matplotlib
        import matplotlib.pyplot as plt
        import matplotlib.cbook as cb
        from matplotlib.colors import colorConverter, Colormap
        from matplotlib.collections import LineCollection
        import numpy
    except ImportError:
        raise ImportError("Matplotlib required for draw()")
    except RuntimeError:
        print("Matplotlib unable to open display")
        raise

    if ax is None:
        ax = plt.gca()

    if edgelist is None:
        edgelist = list(G.edges())

    if not edgelist or len(edgelist) == 0:  # no edges!
        return None

    # set edge positions
    edge_pos = numpy.asarray([(pos[e[0]], pos[e[1]]) for e in edgelist])

    if not cb.iterable(width):
        lw = (width,)
    else:
        lw = width

    if not cb.is_string_like(edge_color) \
           and cb.iterable(edge_color) \
           and len(edge_color) == len(edge_pos):
        if numpy.alltrue([cb.is_string_like(c)
                         for c in edge_color]):
            # (should check ALL elements)
            # list of color letters such as ['k','r','k',...]
            edge_colors = tuple([colorConverter.to_rgba(c, alpha)
                                 for c in edge_color])
        elif numpy.alltrue([not cb.is_string_like(c)
                           for c in edge_color]):
            # If color specs are given as (rgb) or (rgba) tuples, we're OK
            if numpy.alltrue([cb.iterable(c) and len(c) in (3, 4)
                             for c in edge_color]):
                edge_colors = tuple(edge_color)
            else:
                # numbers (which are going to be mapped with a colormap)
                edge_colors = None
        else:
            raise ValueError('edge_color must consist of either color names or numbers')
    else:
        if cb.is_string_like(edge_color) or len(edge_color) == 1:
            edge_colors = (colorConverter.to_rgba(edge_color, alpha), )
        else:
            raise ValueError('edge_color must be a single color or list of exactly m colors where m is the number or edges')

    edge_collection = LineCollection(edge_pos,
                                     colors=edge_colors,
                                     linewidths=lw,
                                     antialiaseds=(1,),
                                     linestyle=style,
                                     transOffset = ax.transData,
                                     )

    edge_collection.set_zorder(1)  # edges go behind nodes
    edge_collection.set_label(label)
    ax.add_collection(edge_collection)

    # Note: there was a bug in mpl regarding the handling of alpha values for
    # each line in a LineCollection.  It was fixed in matplotlib in r7184 and
    # r7189 (June 6 2009).  We should then not set the alpha value globally,
    # since the user can instead provide per-edge alphas now.  Only set it
    # globally if provided as a scalar.
    if cb.is_numlike(alpha):
        edge_collection.set_alpha(alpha)

    if edge_colors is None:
        if edge_cmap is not None:
            assert(isinstance(edge_cmap, Colormap))
        edge_collection.set_array(numpy.asarray(edge_color))
        edge_collection.set_cmap(edge_cmap)
        if edge_vmin is not None or edge_vmax is not None:
            edge_collection.set_clim(edge_vmin, edge_vmax)
        else:
            edge_collection.autoscale()

    arrow_collection = None

    if G.is_directed() and arrows:

        # a directed graph hack
        # draw thick line segments at head end of edge
        # waiting for someone else to implement arrows that will work
        arrow_colors = edge_colors
        a_pos = []
        p = 1.0-0.25  # make head segment 25 percent of edge length
        for src, dst in edge_pos:
            x1, y1 = src
            x2, y2 = dst
            dx = x2-x1   # x offset
            dy = y2-y1   # y offset
            d = numpy.sqrt(float(dx**2 + dy**2))  # length of edge
            if d == 0:   # source and target at same position
                continue
            if dx == 0:  # vertical edge
                xa = x2
                ya = dy*p+y1
            if dy == 0:  # horizontal edge
                ya = y2
                xa = dx*p+x1
            else:
                theta = numpy.arctan2(dy, dx)
                xa = p*d*numpy.cos(theta)+x1
                ya = p*d*numpy.sin(theta)+y1

            a_pos.append(((xa, ya), (x2, y2)))

        arrow_collection = LineCollection(a_pos,
                                colors=arrow_colors,
                                linewidths=[4*ww for ww in lw],
                                antialiaseds=(1,),
                                transOffset = ax.transData,
                                )

        arrow_collection.set_zorder(1)  # edges go behind nodes
        arrow_collection.set_label(label)
        ax.add_collection(arrow_collection)

    # update view
    minx = numpy.amin(numpy.ravel(edge_pos[:, :, 0]))
    maxx = numpy.amax(numpy.ravel(edge_pos[:, :, 0]))
    miny = numpy.amin(numpy.ravel(edge_pos[:, :, 1]))
    maxy = numpy.amax(numpy.ravel(edge_pos[:, :, 1]))

    w = maxx-minx
    h = maxy-miny
    padx,  pady = 0.05*w, 0.05*h
    corners = (minx-padx, miny-pady), (maxx+padx, maxy+pady)
    ax.update_datalim(corners)
    ax.autoscale_view()

#    if arrow_collection:

    return edge_collection

def draw_networkx_labels(G, pos,
                         labels=None,
                         font_size=12,
                         font_color='k',
                         font_family='sans-serif',
                         font_weight='normal',
                         alpha=1.0,
                         bbox=None,
                         ax=None,
                         **kwds):
    """Draw node labels on the graph G.

    Parameters
    ----------
    G : graph
       A networkx graph

    pos : dictionary
       A dictionary with nodes as keys and positions as values.
       Positions should be sequences of length 2.

    labels : dictionary, optional (default=None)
       Node labels in a dictionary keyed by node of text labels

    font_size : int
       Font size for text labels (default=12)

    font_color : string
       Font color string (default='k' black)

    font_family : string
       Font family (default='sans-serif')

    font_weight : string
       Font weight (default='normal')

    alpha : float
       The text transparency (default=1.0)

    ax : Matplotlib Axes object, optional
       Draw the graph in the specified Matplotlib axes.

    Returns
    -------
    dict
        `dict` of labels keyed on the nodes

    Examples
    --------
    >>> G=nx.dodecahedral_graph()
    >>> labels=nx.draw_networkx_labels(G,pos=nx.spring_layout(G))

    Also see the NetworkX drawing examples at
    http://networkx.readthedocs.io/en/latest/gallery.html

    See Also
    --------
    draw()
    draw_networkx()
    draw_networkx_nodes()
    draw_networkx_edges()
    draw_networkx_edge_labels()
    """
    try:
        import matplotlib.pyplot as plt
        import matplotlib.cbook as cb
    except ImportError:
        raise ImportError("Matplotlib required for draw()")
    except RuntimeError:
        print("Matplotlib unable to open display")
        raise

    if ax is None:
        ax = plt.gca()

    if labels is None:
        labels = dict((n, n) for n in G.nodes())

    # set optional alignment
    horizontalalignment = kwds.get('horizontalalignment', 'center')
    verticalalignment = kwds.get('verticalalignment', 'center')

    text_items = {}  # there is no text collection so we'll fake one
    for n, label in labels.items():
        (x, y) = pos[n]
        if not cb.is_string_like(label):
            label = str(label)  # this will cause "1" and 1 to be labeled the same
        t = ax.text(x, y,
                  label,
                  size=font_size,
                  color=font_color,
                  family=font_family,
                  weight=font_weight,
                  alpha=alpha,
                  horizontalalignment=horizontalalignment,
                  verticalalignment=verticalalignment,
                  transform=ax.transData,
                  bbox=bbox,
                  clip_on=True,
                  )
        text_items[n] = t

    return text_items

def draw_networkx_edge_labels(G, pos,
                              edge_labels=None,
                              label_pos=0.5,
                              font_size=10,
                              font_color='k',
                              font_family='sans-serif',
                              font_weight='normal',
                              alpha=1.0,
                              bbox=None,
                              ax=None,
                              rotate=True,
                              **kwds):
    """Draw edge labels.

    Parameters
    ----------
    G : graph
       A networkx graph

    pos : dictionary
       A dictionary with nodes as keys and positions as values.
       Positions should be sequences of length 2.

    ax : Matplotlib Axes object, optional
       Draw the graph in the specified Matplotlib axes.

    alpha : float
       The text transparency (default=1.0)

    edge_labels : dictionary
       Edge labels in a dictionary keyed by edge two-tuple of text
       labels (default=None). Only labels for the keys in the dictionary
       are drawn.

    label_pos : float
       Position of edge label along edge (0=head, 0.5=center, 1=tail)

    font_size : int
       Font size for text labels (default=12)

    font_color : string
       Font color string (default='k' black)

    font_weight : string
       Font weight (default='normal')

    font_family : string
       Font family (default='sans-serif')

    bbox : Matplotlib bbox
       Specify text box shape and colors.

    clip_on : bool
       Turn on clipping at axis boundaries (default=True)

    Returns
    -------
    dict
        `dict` of labels keyed on the edges

    Examples
    --------
    >>> G=nx.dodecahedral_graph()
    >>> edge_labels=nx.draw_networkx_edge_labels(G,pos=nx.spring_layout(G))

    Also see the NetworkX drawing examples at
    http://networkx.readthedocs.io/en/latest/gallery.html

    See Also
    --------
    draw()
    draw_networkx()
    draw_networkx_nodes()
    draw_networkx_edges()
    draw_networkx_labels()
    """
    try:
        import matplotlib.pyplot as plt
        import matplotlib.cbook as cb
        import numpy
    except ImportError:
        raise ImportError("Matplotlib required for draw()")
    except RuntimeError:
        print("Matplotlib unable to open display")
        raise

    if ax is None:
        ax = plt.gca()
    if edge_labels is None:
        labels = dict(((u, v), d) for u, v, d in G.edges(data=True))
    else:
        labels = edge_labels
    text_items = {}
    for (n1, n2), label in labels.items():
        (x1, y1) = pos[n1]
        (x2, y2) = pos[n2]
        (x, y) = (x1 * label_pos + x2 * (1.0 - label_pos),
                  y1 * label_pos + y2 * (1.0 - label_pos))

        if rotate:
            angle = numpy.arctan2(y2-y1, x2-x1)/(2.0*numpy.pi)*360  # degrees
            # make label orientation "right-side-up"
            if angle > 90:
                angle -= 180
            if angle < - 90:
                angle += 180
            # transform data coordinate angle to screen coordinate angle
            xy = numpy.array((x, y))
            trans_angle = ax.transData.transform_angles(numpy.array((angle,)),
                                                        xy.reshape((1, 2)))[0]
        else:
            trans_angle = 0.0
        # use default box of white with white border
        if bbox is None:
            bbox = dict(boxstyle='round',
                        ec=(1.0, 1.0, 1.0),
                        fc=(1.0, 1.0, 1.0),
                        )
        if not cb.is_string_like(label):
            label = str(label)  # this will cause "1" and 1 to be labeled the same

        # set optional alignment
        horizontalalignment = kwds.get('horizontalalignment', 'center')
        verticalalignment = kwds.get('verticalalignment', 'center')

        t = ax.text(x, y,
                    label,
                    size=font_size,
                    color=font_color,
                    family=font_family,
                    weight=font_weight,
                    alpha=alpha,
                    horizontalalignment=horizontalalignment,
                    verticalalignment=verticalalignment,
                    rotation=trans_angle,
                    transform=ax.transData,
                    bbox=bbox,
                    zorder=1,
                    clip_on=True,
                    )
        text_items[(n1, n2)] = t

    return text_items

def draw_circular(G, **kwargs):
    """Draw the graph G with a circular layout.

    Parameters
    ----------
    G : graph
       A networkx graph

    kwargs : optional keywords
       See networkx.draw_networkx() for a description of optional keywords,
       with the exception of the pos parameter which is not used by this
       function.
    """
    draw(G, circular_layout(G), **kwargs)

def draw_random(G, **kwargs):
    """Draw the graph G with a random layout.

    Parameters
    ----------
    G : graph
       A networkx graph

    kwargs : optional keywords
       See networkx.draw_networkx() for a description of optional keywords,
       with the exception of the pos parameter which is not used by this
       function.
    """
    draw(G, random_layout(G), **kwargs)

def draw_spectral(G, **kwargs):
    """Draw the graph G with a spectral layout.

    Parameters
    ----------
    G : graph
       A networkx graph

    kwargs : optional keywords
       See networkx.draw_networkx() for a description of optional keywords,
       with the exception of the pos parameter which is not used by this
       function.
    """
    draw(G, spectral_layout(G), **kwargs)

def draw_spring(G, **kwargs):
    """Draw the graph G with a spring layout.

    Parameters
    ----------
    G : graph
       A networkx graph

    kwargs : optional keywords
       See networkx.draw_networkx() for a description of optional keywords,
       with the exception of the pos parameter which is not used by this
       function.
    """
    draw(G, spring_layout(G), **kwargs)

def draw_shell(G, **kwargs):
    """Draw networkx graph with shell layout.

    Parameters
    ----------
    G : graph
       A networkx graph

    kwargs : optional keywords
       See networkx.draw_networkx() for a description of optional keywords,
       with the exception of the pos parameter which is not used by this
       function.
    """
    nlist = kwargs.get('nlist', None)
    if nlist is not None:
        del(kwargs['nlist'])
    draw(G, shell_layout(G, nlist=nlist), **kwargs)

def draw_nx(G, pos, **kwds):
    """For backward compatibility; use draw or draw_networkx."""
    draw(G, pos, **kwds)

def apply_alpha(colors, alpha, elem_list, cmap=None, vmin=None, vmax=None):
    """Apply an alpha (or list of alphas) to the colors provided.

    Parameters
    ----------

    color : color string, or array of floats
       Color of element. Can be a single color format string (default='r'),
       or a  sequence of colors with the same length as nodelist.
       If numeric values are specified they will be mapped to
       colors using the cmap and vmin,vmax parameters.  See
       matplotlib.scatter for more details.

    alpha : float or array of floats
       Alpha values for elements. This can be a single alpha value, in
       which case it will be applied to all the elements of color. Otherwise,
       if it is an array, the elements of alpha will be applied to the colors
       in order (cycling through alpha multiple times if necessary).

    elem_list : array of networkx objects
       The list of elements which are being colored. These could be nodes, edges
       or labels.

    cmap : matplotlib colormap
       Color map for use if colors is a list of floats corresponding to points on
       a color mapping.

    vmin, vmax : float
       Minimum and maximum values for normalizing colors if a color mapping is used.

    Returns
    -------

    rgba_colors : numpy ndarray
        Array containing RGBA format values for each of the node colours.

    """
    import numbers
    import itertools

    try:
        import numpy
        from matplotlib.colors import colorConverter
        import matplotlib.cm as cm
    except ImportError:
        raise ImportError("Matplotlib required for draw()")

    # If we have been provided with a list of numbers as long as elem_list, apply the color mapping.
    if len(colors) == len(elem_list) and isinstance(colors[0], numbers.Number):
        mapper = cm.ScalarMappable(cmap=cmap)
        mapper.set_clim(vmin, vmax)
        rgba_colors = mapper.to_rgba(colors)
    # Otherwise, convert colors to matplotlib's RGB using the colorConverter object.
    # These are converted to numpy ndarrays to be consistent with the to_rgba method of ScalarMappable.
    else:
        try:
            rgba_colors = numpy.array([colorConverter.to_rgba(colors)])
        except ValueError:
            rgba_colors = numpy.array([colorConverter.to_rgba(color) for color in colors])
    # Set the final column of the rgba_colors to have the relevant alpha values.
    try:
        # If alpha is longer than the number of colors, resize to the number of elements.
        # Also, if rgba_colors.size (the number of elements of rgba_colors) is the same as the number of
        # elements, resize the array, to avoid it being interpreted as a colormap by scatter()
        if len(alpha) > len(rgba_colors) or rgba_colors.size == len(elem_list):
            rgba_colors.resize((len(elem_list), 4))
            rgba_colors[1:, 0] = rgba_colors[0, 0]
            rgba_colors[1:, 1] = rgba_colors[0, 1]
            rgba_colors[1:, 2] = rgba_colors[0, 2]
        rgba_colors[:,  3] = list(itertools.islice(itertools.cycle(alpha), len(rgba_colors)))
    except TypeError:
        rgba_colors[:, -1] = alpha
    return rgba_colors

# fixture for nose tests
def setup_module(module):
    from nose import SkipTest
    try:
        import matplotlib as mpl
        mpl.use('PS', warn=False)
        import matplotlib.pyplot as plt
    except:
        raise SkipTest("matplotlib not available")


"""
An experimental module for reading dvi files output by TeX. Several
limitations make this not (currently) useful as a general-purpose dvi
preprocessor, but it is currently used by the pdf backend for
processing usetex text.

Interface::

  dvi = Dvi(filename, 72)
  # iterate over pages (but only one page is supported for now):
  for page in dvi:
      w, h, d = page.width, page.height, page.descent
      for x,y,font,glyph,width in page.text:
          fontname = font.texname
          pointsize = font.size
          ...
      for x,y,height,width in page.boxes:
          ...

"""
from __future__ import (absolute_import, division, print_function,
                        unicode_literals)

import six
from six.moves import xrange

import errno
import matplotlib
import matplotlib.cbook as mpl_cbook
from matplotlib.compat import subprocess
from matplotlib import rcParams
import numpy as np
import struct
import sys
import os

if six.PY3:
    def ord(x):
        return x

_dvistate = mpl_cbook.Bunch(pre=0, outer=1, inpage=2, post_post=3, finale=4)

class Dvi(object):
    """
    A dvi ("device-independent") file, as produced by TeX.
    The current implementation only reads the first page and does not
    even attempt to verify the postamble.
    """

    def __init__(self, filename, dpi):
        """
        Initialize the object. This takes the filename as input and
        opens the file; actually reading the file happens when
        iterating through the pages of the file.
        """
        matplotlib.verbose.report('Dvi: ' + filename, 'debug')
        self.file = open(filename, 'rb')
        self.dpi = dpi
        self.fonts = {}
        self.state = _dvistate.pre
        self.baseline = self._get_baseline(filename)

    def _get_baseline(self, filename):
        if rcParams['text.latex.preview']:
            base, ext = os.path.splitext(filename)
            baseline_filename = base + ".baseline"
            if os.path.exists(baseline_filename):
                with open(baseline_filename, 'rb') as fd:
                    l = fd.read().split()
                height, depth, width = l
                return float(depth)
        return None

    def __iter__(self):
        """
        Iterate through the pages of the file.

        Returns (text, boxes) pairs, where:
          text is a list of (x, y, fontnum, glyphnum, width) tuples
          boxes is a list of (x, y, height, width) tuples

        The coordinates are transformed into a standard Cartesian
        coordinate system at the dpi value given when initializing.
        The coordinates are floating point numbers, but otherwise
        precision is not lost and coordinate values are not clipped to
        integers.
        """
        while True:
            have_page = self._read()
            if have_page:
                yield self._output()
            else:
                break

    def close(self):
        """
        Close the underlying file if it is open.
        """
        if not self.file.closed:
            self.file.close()

    def _output(self):
        """
        Output the text and boxes belonging to the most recent page.
        page = dvi._output()
        """
        minx, miny, maxx, maxy = np.inf, np.inf, -np.inf, -np.inf
        maxy_pure = -np.inf
        for elt in self.text + self.boxes:
            if len(elt) == 4:   # box
                x,y,h,w = elt
                e = 0           # zero depth
            else:               # glyph
                x,y,font,g,w = elt
                h,e = font._height_depth_of(g)
            minx = min(minx, x)
            miny = min(miny, y - h)
            maxx = max(maxx, x + w)
            maxy = max(maxy, y + e)
            maxy_pure = max(maxy_pure, y)

        if self.dpi is None:
            # special case for ease of debugging: output raw dvi coordinates
            return mpl_cbook.Bunch(text=self.text, boxes=self.boxes,
                                   width=maxx-minx, height=maxy_pure-miny,
                                   descent=descent)

        d = self.dpi / (72.27 * 2**16) # from TeX's "scaled points" to dpi units
        if self.baseline is None:
            descent = (maxy - maxy_pure) * d
        else:
            descent = self.baseline

        text =  [ ((x-minx)*d, (maxy-y)*d - descent, f, g, w*d)
                  for (x,y,f,g,w) in self.text ]
        boxes = [ ((x-minx)*d, (maxy-y)*d - descent, h*d, w*d) for (x,y,h,w) in self.boxes ]

        return mpl_cbook.Bunch(text=text, boxes=boxes,
                               width=(maxx-minx)*d,
                               height=(maxy_pure-miny)*d,
                               descent=descent)

    def _read(self):
        """
        Read one page from the file. Return True if successful,
        False if there were no more pages.
        """
        while True:
            byte = ord(self.file.read(1)[0])
            self._dispatch(byte)
#             if self.state == _dvistate.inpage:
#                 matplotlib.verbose.report(
#                     'Dvi._read: after %d at %f,%f' %
#                     (byte, self.h, self.v),
#                     'debug-annoying')
            if byte == 140: # end of page
                return True
            if self.state == _dvistate.post_post: # end of file
                self.close()
                return False

    def _arg(self, nbytes, signed=False):
        """
        Read and return an integer argument *nbytes* long.
        Signedness is determined by the *signed* keyword.
        """
        str = self.file.read(nbytes)
        value = ord(str[0])
        if signed and value >= 0x80:
            value = value - 0x100
        for i in range(1, nbytes):
            value = 0x100*value + ord(str[i])
        return value

    def _dispatch(self, byte):
        """
        Based on the opcode *byte*, read the correct kinds of
        arguments from the dvi file and call the method implementing
        that opcode with those arguments.
        """
        if 0 <= byte <= 127: self._set_char(byte)
        elif byte == 128: self._set_char(self._arg(1))
        elif byte == 129: self._set_char(self._arg(2))
        elif byte == 130: self._set_char(self._arg(3))
        elif byte == 131: self._set_char(self._arg(4, True))
        elif byte == 132: self._set_rule(self._arg(4, True), self._arg(4, True))
        elif byte == 133: self._put_char(self._arg(1))
        elif byte == 134: self._put_char(self._arg(2))
        elif byte == 135: self._put_char(self._arg(3))
        elif byte == 136: self._put_char(self._arg(4, True))
        elif byte == 137: self._put_rule(self._arg(4, True), self._arg(4, True))
        elif byte == 138: self._nop()
        elif byte == 139: self._bop(*[self._arg(4, True) for i in range(11)])
        elif byte == 140: self._eop()
        elif byte == 141: self._push()
        elif byte == 142: self._pop()
        elif byte == 143: self._right(self._arg(1, True))
        elif byte == 144: self._right(self._arg(2, True))
        elif byte == 145: self._right(self._arg(3, True))
        elif byte == 146: self._right(self._arg(4, True))
        elif byte == 147: self._right_w(None)
        elif byte == 148: self._right_w(self._arg(1, True))
        elif byte == 149: self._right_w(self._arg(2, True))
        elif byte == 150: self._right_w(self._arg(3, True))
        elif byte == 151: self._right_w(self._arg(4, True))
        elif byte == 152: self._right_x(None)
        elif byte == 153: self._right_x(self._arg(1, True))
        elif byte == 154: self._right_x(self._arg(2, True))
        elif byte == 155: self._right_x(self._arg(3, True))
        elif byte == 156: self._right_x(self._arg(4, True))
        elif byte == 157: self._down(self._arg(1, True))
        elif byte == 158: self._down(self._arg(2, True))
        elif byte == 159: self._down(self._arg(3, True))
        elif byte == 160: self._down(self._arg(4, True))
        elif byte == 161: self._down_y(None)
        elif byte == 162: self._down_y(self._arg(1, True))
        elif byte == 163: self._down_y(self._arg(2, True))
        elif byte == 164: self._down_y(self._arg(3, True))
        elif byte == 165: self._down_y(self._arg(4, True))
        elif byte == 166: self._down_z(None)
        elif byte == 167: self._down_z(self._arg(1, True))
        elif byte == 168: self._down_z(self._arg(2, True))
        elif byte == 169: self._down_z(self._arg(3, True))
        elif byte == 170: self._down_z(self._arg(4, True))
        elif 171 <= byte <= 234: self._fnt_num(byte-171)
        elif byte == 235: self._fnt_num(self._arg(1))
        elif byte == 236: self._fnt_num(self._arg(2))
        elif byte == 237: self._fnt_num(self._arg(3))
        elif byte == 238: self._fnt_num(self._arg(4, True))
        elif 239 <= byte <= 242:
            len = self._arg(byte-238)
            special = self.file.read(len)
            self._xxx(special)
        elif 243 <= byte <= 246:
            k = self._arg(byte-242, byte==246)
            c, s, d, a, l = [ self._arg(x) for x in (4, 4, 4, 1, 1) ]
            n = self.file.read(a+l)
            self._fnt_def(k, c, s, d, a, l, n)
        elif byte == 247:
            i, num, den, mag, k = [ self._arg(x) for x in (1, 4, 4, 4, 1) ]
            x = self.file.read(k)
            self._pre(i, num, den, mag, x)
        elif byte == 248: self._post()
        elif byte == 249: self._post_post()
        else:
            raise ValueError("unknown command: byte %d"%byte)

    def _pre(self, i, num, den, mag, comment):
        if self.state != _dvistate.pre:
            raise ValueError("pre command in middle of dvi file")
        if i != 2:
            raise ValueError("Unknown dvi format %d"%i)
        if num != 25400000 or den != 7227 * 2**16:
            raise ValueError("nonstandard units in dvi file")
            # meaning: TeX always uses those exact values, so it
            # should be enough for us to support those
            # (There are 72.27 pt to an inch so 7227 pt =
            # 7227 * 2**16 sp to 100 in. The numerator is multiplied
            # by 10^5 to get units of 10**-7 meters.)
        if mag != 1000:
            raise ValueError("nonstandard magnification in dvi file")
            # meaning: LaTeX seems to frown on setting \mag, so
            # I think we can assume this is constant
        self.state = _dvistate.outer

    def _set_char(self, char):
        if self.state != _dvistate.inpage:
            raise ValueError("misplaced set_char in dvi file")
        self._put_char(char)
        self.h += self.fonts[self.f]._width_of(char)

    def _set_rule(self, a, b):
        if self.state != _dvistate.inpage:
            raise ValueError("misplaced set_rule in dvi file")
        self._put_rule(a, b)
        self.h += b

    def _put_char(self, char):
        if self.state != _dvistate.inpage:
            raise ValueError("misplaced put_char in dvi file")
        font = self.fonts[self.f]
        if font._vf is None:
            self.text.append((self.h, self.v, font, char,
                              font._width_of(char)))
#             matplotlib.verbose.report(
#                 'Dvi._put_char: %d,%d %d' %(self.h, self.v, char),
#                 'debug-annoying')
        else:
            scale = font._scale
            for x, y, f, g, w in font._vf[char].text:
                newf = DviFont(scale=_mul2012(scale, f._scale),
                               tfm=f._tfm, texname=f.texname, vf=f._vf)
                self.text.append((self.h + _mul2012(x, scale),
                                  self.v + _mul2012(y, scale),
                                  newf, g, newf._width_of(g)))
            self.boxes.extend([(self.h + _mul2012(x, scale),
                                self.v + _mul2012(y, scale),
                                _mul2012(a, scale), _mul2012(b, scale))
                               for x, y, a, b in font._vf[char].boxes])

    def _put_rule(self, a, b):
        if self.state != _dvistate.inpage:
            raise ValueError("misplaced put_rule in dvi file")
        if a > 0 and b > 0:
            self.boxes.append((self.h, self.v, a, b))
#             matplotlib.verbose.report(
#                 'Dvi._put_rule: %d,%d %d,%d' % (self.h, self.v, a, b),
#                 'debug-annoying')

    def _nop(self):
        pass

    def _bop(self, c0, c1, c2, c3, c4, c5, c6, c7, c8, c9, p):
        if self.state != _dvistate.outer:
            raise ValueError("misplaced bop in dvi file (state %d)" % self.state)
        self.state = _dvistate.inpage
        self.h, self.v, self.w, self.x, self.y, self.z = 0, 0, 0, 0, 0, 0
        self.stack = []
        self.text = []          # list of (x,y,fontnum,glyphnum)
        self.boxes = []         # list of (x,y,width,height)

    def _eop(self):
        if self.state != _dvistate.inpage:
            raise ValueError("misplaced eop in dvi file")
        self.state = _dvistate.outer
        del self.h, self.v, self.w, self.x, self.y, self.z, self.stack

    def _push(self):
        if self.state != _dvistate.inpage:
            raise ValueError("misplaced push in dvi file")
        self.stack.append((self.h, self.v, self.w, self.x, self.y, self.z))

    def _pop(self):
        if self.state != _dvistate.inpage:
            raise ValueError("misplaced pop in dvi file")
        self.h, self.v, self.w, self.x, self.y, self.z = self.stack.pop()

    def _right(self, b):
        if self.state != _dvistate.inpage:
            raise ValueError("misplaced right in dvi file")
        self.h += b

    def _right_w(self, new_w):
        if self.state != _dvistate.inpage:
            raise ValueError("misplaced w in dvi file")
        if new_w is not None:
            self.w = new_w
        self.h += self.w

    def _right_x(self, new_x):
        if self.state != _dvistate.inpage:
            raise ValueError("misplaced x in dvi file")
        if new_x is not None:
            self.x = new_x
        self.h += self.x

    def _down(self, a):
        if self.state != _dvistate.inpage:
            raise ValueError("misplaced down in dvi file")
        self.v += a

    def _down_y(self, new_y):
        if self.state != _dvistate.inpage:
            raise ValueError("misplaced y in dvi file")
        if new_y is not None:
            self.y = new_y
        self.v += self.y

    def _down_z(self, new_z):
        if self.state != _dvistate.inpage:
            raise ValueError("misplaced z in dvi file")
        if new_z is not None:
            self.z = new_z
        self.v += self.z

    def _fnt_num(self, k):
        if self.state != _dvistate.inpage:
            raise ValueError("misplaced fnt_num in dvi file")
        self.f = k

    def _xxx(self, special):
        if six.PY3:
            matplotlib.verbose.report(
                'Dvi._xxx: encountered special: %s'
                % ''.join([(32 <= ord(ch) < 127) and chr(ch)
                           or '<%02x>' % ord(ch)
                           for ch in special]),
                'debug')
        else:
            matplotlib.verbose.report(
                'Dvi._xxx: encountered special: %s'
                % ''.join([(32 <= ord(ch) < 127) and ch
                           or '<%02x>' % ord(ch)
                           for ch in special]),
                'debug')

    def _fnt_def(self, k, c, s, d, a, l, n):
        tfm = _tfmfile(n[-l:].decode('ascii'))
        if c != 0 and tfm.checksum != 0 and c != tfm.checksum:
            raise ValueError('tfm checksum mismatch: %s'%n)
        # It seems that the assumption behind the following check is incorrect:
        #if d != tfm.design_size:
        #    raise ValueError, 'tfm design size mismatch: %d in dvi, %d in %s'%\
        #        (d, tfm.design_size, n)

        vf = _vffile(n[-l:].decode('ascii'))

        self.fonts[k] = DviFont(scale=s, tfm=tfm, texname=n, vf=vf)

    def _post(self):
        if self.state != _dvistate.outer:
            raise ValueError("misplaced post in dvi file")
        self.state = _dvistate.post_post
        # TODO: actually read the postamble and finale?
        # currently post_post just triggers closing the file

    def _post_post(self):
        raise NotImplementedError

class DviFont(object):
    """
    Object that holds a font's texname and size, supports comparison,
    and knows the widths of glyphs in the same units as the AFM file.
    There are also internal attributes (for use by dviread.py) that
    are *not* used for comparison.

    The size is in Adobe points (converted from TeX points).

    .. attribute:: texname

       Name of the font as used internally by TeX and friends. This
       is usually very different from any external font names, and
       :class:`dviread.PsfontsMap` can be used to find the external
       name of the font.

    .. attribute:: size

       Size of the font in Adobe points, converted from the slightly
       smaller TeX points.

    .. attribute:: widths

       Widths of glyphs in glyph-space units, typically 1/1000ths of
       the point size.

    """
    __slots__ = ('texname', 'size', 'widths', '_scale', '_vf', '_tfm')

    def __init__(self, scale, tfm, texname, vf):
        if six.PY3 and isinstance(texname, bytes):
            texname = texname.decode('ascii')
        self._scale, self._tfm, self.texname, self._vf = \
            scale, tfm, texname, vf
        self.size = scale * (72.0 / (72.27 * 2**16))
        try:
            nchars = max(six.iterkeys(tfm.width)) + 1
        except ValueError:
            nchars = 0
        self.widths = [ (1000*tfm.width.get(char, 0)) >> 20
                        for char in xrange(nchars) ]

    def __eq__(self, other):
        return self.__class__ == other.__class__ and \
            self.texname == other.texname and self.size == other.size

    def __ne__(self, other):
        return not self.__eq__(other)

    def _width_of(self, char):
        """
        Width of char in dvi units. For internal use by dviread.py.
        """

        width = self._tfm.width.get(char, None)
        if width is not None:
            return _mul2012(width, self._scale)

        matplotlib.verbose.report(
            'No width for char %d in font %s' % (char, self.texname),
            'debug')
        return 0

    def _height_depth_of(self, char):
        """
        Height and depth of char in dvi units. For internal use by dviread.py.
        """

        result = []
        for metric,name in ((self._tfm.height, "height"),
                            (self._tfm.depth, "depth")):
            value = metric.get(char, None)
            if value is None:
                matplotlib.verbose.report(
                    'No %s for char %d in font %s' % (name, char, self.texname),
                    'debug')
                result.append(0)
            else:
                result.append(_mul2012(value, self._scale))
        return result

class Vf(Dvi):
    """
    A virtual font (\*.vf file) containing subroutines for dvi files.

    Usage::

      vf = Vf(filename)
      glyph = vf[code]
      glyph.text, glyph.boxes, glyph.width
    """

    def __init__(self, filename):
        Dvi.__init__(self, filename, 0)
        try:
            self._first_font = None
            self._chars = {}
            self._packet_ends = None
            self._read()
        finally:
            self.close()

    def __getitem__(self, code):
        return self._chars[code]

    def _dispatch(self, byte):
        # If we are in a packet, execute the dvi instructions
        if self.state == _dvistate.inpage:
            byte_at = self.file.tell()-1
            if byte_at == self._packet_ends:
                self._finalize_packet()
                # fall through
            elif byte_at > self._packet_ends:
                raise ValueError("Packet length mismatch in vf file")
            else:
                if byte in (139, 140) or byte >= 243:
                    raise ValueError("Inappropriate opcode %d in vf file" % byte)
                Dvi._dispatch(self, byte)
                return

        # We are outside a packet
        if byte < 242:          # a short packet (length given by byte)
            cc, tfm = self._arg(1), self._arg(3)
            self._init_packet(byte, cc, tfm)
        elif byte == 242:       # a long packet
            pl, cc, tfm = [ self._arg(x) for x in (4, 4, 4) ]
            self._init_packet(pl, cc, tfm)
        elif 243 <= byte <= 246:
            Dvi._dispatch(self, byte)
        elif byte == 247:       # preamble
            i, k = self._arg(1), self._arg(1)
            x = self.file.read(k)
            cs, ds = self._arg(4), self._arg(4)
            self._pre(i, x, cs, ds)
        elif byte == 248:       # postamble (just some number of 248s)
            self.state = _dvistate.post_post
        else:
            raise ValueError("unknown vf opcode %d" % byte)

    def _init_packet(self, pl, cc, tfm):
        if self.state != _dvistate.outer:
            raise ValueError("Misplaced packet in vf file")
        self.state = _dvistate.inpage
        self._packet_ends = self.file.tell() + pl
        self._packet_char = cc
        self._packet_width = tfm
        self.h, self.v, self.w, self.x, self.y, self.z = 0, 0, 0, 0, 0, 0
        self.stack, self.text, self.boxes = [], [], []
        self.f = self._first_font

    def _finalize_packet(self):
        self._chars[self._packet_char] = mpl_cbook.Bunch(
            text=self.text, boxes=self.boxes, width = self._packet_width)
        self.state = _dvistate.outer

    def _pre(self, i, x, cs, ds):
        if self.state != _dvistate.pre:
            raise ValueError("pre command in middle of vf file")
        if i != 202:
            raise ValueError("Unknown vf format %d" % i)
        if len(x):
            matplotlib.verbose.report('vf file comment: ' + x, 'debug')
        self.state = _dvistate.outer
        # cs = checksum, ds = design size

    def _fnt_def(self, k, *args):
        Dvi._fnt_def(self, k, *args)
        if self._first_font is None:
            self._first_font = k

def _fix2comp(num):
    """
    Convert from two's complement to negative.
    """
    assert 0 <= num < 2**32
    if num & 2**31:
        return num - 2**32
    else:
        return num

def _mul2012(num1, num2):
    """
    Multiply two numbers in 20.12 fixed point format.
    """
    # Separated into a function because >> has surprising precedence
    return (num1*num2) >> 20

class Tfm(object):
    """
    A TeX Font Metric file. This implementation covers only the bare
    minimum needed by the Dvi class.

    .. attribute:: checksum

       Used for verifying against the dvi file.

    .. attribute:: design_size

       Design size of the font (in what units?)

    .. attribute::  width

       Width of each character, needs to be scaled by the factor
       specified in the dvi file. This is a dict because indexing may
       not start from 0.

    .. attribute:: height

       Height of each character.

    .. attribute:: depth

       Depth of each character.
    """
    __slots__ = ('checksum', 'design_size', 'width', 'height', 'depth')

    def __init__(self, filename):
        matplotlib.verbose.report('opening tfm file ' + filename, 'debug')
        with open(filename, 'rb') as file:
            header1 = file.read(24)
            lh, bc, ec, nw, nh, nd = \
                struct.unpack(str('!6H'), header1[2:14])
            matplotlib.verbose.report(
                'lh=%d, bc=%d, ec=%d, nw=%d, nh=%d, nd=%d' % (
                    lh, bc, ec, nw, nh, nd), 'debug')
            header2 = file.read(4*lh)
            self.checksum, self.design_size = \
                struct.unpack(str('!2I'), header2[:8])
            # there is also encoding information etc.
            char_info = file.read(4*(ec-bc+1))
            widths = file.read(4*nw)
            heights = file.read(4*nh)
            depths = file.read(4*nd)

        self.width, self.height, self.depth = {}, {}, {}
        widths, heights, depths = \
            [ struct.unpack(str('!%dI') % (len(x)/4), x)
              for x in (widths, heights, depths) ]
        for idx, char in enumerate(xrange(bc, ec+1)):
            self.width[char] = _fix2comp(widths[ord(char_info[4*idx])])
            self.height[char] = _fix2comp(heights[ord(char_info[4*idx+1]) >> 4])
            self.depth[char] = _fix2comp(depths[ord(char_info[4*idx+1]) & 0xf])

class PsfontsMap(object):
    """
    A psfonts.map formatted file, mapping TeX fonts to PS fonts.
    Usage::

     >>> map = PsfontsMap(find_tex_file('pdftex.map'))
     >>> entry = map['ptmbo8r']
     >>> entry.texname
     'ptmbo8r'
     >>> entry.psname
     'Times-Bold'
     >>> entry.encoding
     '/usr/local/texlive/2008/texmf-dist/fonts/enc/dvips/base/8r.enc'
     >>> entry.effects
     {'slant': 0.16700000000000001}
     >>> entry.filename

    For historical reasons, TeX knows many Type-1 fonts by different
    names than the outside world. (For one thing, the names have to
    fit in eight characters.) Also, TeX's native fonts are not Type-1
    but Metafont, which is nontrivial to convert to PostScript except
    as a bitmap. While high-quality conversions to Type-1 format exist
    and are shipped with modern TeX distributions, we need to know
    which Type-1 fonts are the counterparts of which native fonts. For
    these reasons a mapping is needed from internal font names to font
    file names.

    A texmf tree typically includes mapping files called e.g.
    psfonts.map, pdftex.map, dvipdfm.map. psfonts.map is used by
    dvips, pdftex.map by pdfTeX, and dvipdfm.map by dvipdfm.
    psfonts.map might avoid embedding the 35 PostScript fonts (i.e.,
    have no filename for them, as in the Times-Bold example above),
    while the pdf-related files perhaps only avoid the "Base 14" pdf
    fonts. But the user may have configured these files differently.
    """
    __slots__ = ('_font',)

    def __init__(self, filename):
        self._font = {}
        with open(filename, 'rt') as file:
            self._parse(file)

    def __getitem__(self, texname):
        try:
            result = self._font[texname]
        except KeyError:
            result = self._font[texname.decode('ascii')]
        fn, enc = result.filename, result.encoding
        if fn is not None and not fn.startswith('/'):
            result.filename = find_tex_file(fn)
        if enc is not None and not enc.startswith('/'):
            result.encoding = find_tex_file(result.encoding)
        return result

    def _parse(self, file):
        """Parse each line into words."""
        for line in file:
            line = line.strip()
            if line == '' or line.startswith('%'):
                continue
            words, pos = [], 0
            while pos < len(line):
                if line[pos] == '"': # double quoted word
                    pos += 1
                    end = line.index('"', pos)
                    words.append(line[pos:end])
                    pos = end + 1
                else:                # ordinary word
                    end = line.find(' ', pos+1)
                    if end == -1: end = len(line)
                    words.append(line[pos:end])
                    pos = end
                while pos < len(line) and line[pos] == ' ':
                    pos += 1
            self._register(words)

    def _register(self, words):
        """Register a font described by "words".

        The format is, AFAIK: texname fontname [effects and filenames]
        Effects are PostScript snippets like ".177 SlantFont",
        filenames begin with one or two less-than signs. A filename
        ending in enc is an encoding file, other filenames are font
        files. This can be overridden with a left bracket: <[foobar
        indicates an encoding file named foobar.

        There is some difference between <foo.pfb and <<bar.pfb in
        subsetting, but I have no example of << in my TeX installation.
        """

        # If the map file specifies multiple encodings for a font, we
        # follow pdfTeX in choosing the last one specified. Such
        # entries are probably mistakes but they have occurred.
        # http://tex.stackexchange.com/questions/10826/
        # http://article.gmane.org/gmane.comp.tex.pdftex/4914

        texname, psname = words[:2]
        effects, encoding, filename = '', None, None
        for word in words[2:]:
            if not word.startswith('<'):
                effects = word
            else:
                word = word.lstrip('<')
                if word.startswith('[') or word.endswith('.enc'):
                    if encoding is not None:
                        matplotlib.verbose.report(
                            'Multiple encodings for %s = %s'
                            % (texname, psname), 'debug')
                    if word.startswith('['):
                        encoding = word[1:]
                    else:
                        encoding = word
                else:
                    assert filename is None
                    filename = word

        eff = effects.split()
        effects = {}
        try:
            effects['slant'] = float(eff[eff.index('SlantFont')-1])
        except ValueError:
            pass
        try:
            effects['extend'] = float(eff[eff.index('ExtendFont')-1])
        except ValueError:
            pass

        self._font[texname] = mpl_cbook.Bunch(
            texname=texname, psname=psname, effects=effects,
            encoding=encoding, filename=filename)

class Encoding(object):
    """
    Parses a \*.enc file referenced from a psfonts.map style file.
    The format this class understands is a very limited subset of
    PostScript.

    Usage (subject to change)::

      for name in Encoding(filename):
          whatever(name)
    """
    __slots__ = ('encoding',)

    def __init__(self, filename):
        with open(filename, 'rt') as file:
            matplotlib.verbose.report('Parsing TeX encoding ' + filename, 'debug-annoying')
            self.encoding = self._parse(file)
            matplotlib.verbose.report('Result: ' + repr(self.encoding), 'debug-annoying')

    def __iter__(self):
        for name in self.encoding:
            yield name

    def _parse(self, file):
        result = []

        state = 0
        for line in file:
            comment_start = line.find('%')
            if comment_start > -1:
                line = line[:comment_start]
            line = line.strip()

            if state == 0:
                # Expecting something like /FooEncoding [
                if '[' in line:
                    state = 1
                    line = line[line.index('[')+1:].strip()

            if state == 1:
                if ']' in line: # ] def
                    line = line[:line.index(']')]
                    state = 2
                words = line.split()
                for w in words:
                    if w.startswith('/'):
                        # Allow for /abc/def/ghi
                        subwords = w.split('/')
                        result.extend(subwords[1:])
                    else:
                        raise ValueError("Broken name in encoding file: " + w)

        return result

def find_tex_file(filename, format=None):
    """
    Call :program:`kpsewhich` to find a file in the texmf tree. If
    *format* is not None, it is used as the value for the
    :option:`--format` option.

    Apparently most existing TeX distributions on Unix-like systems
    use kpathsea. I hear MikTeX (a popular distribution on Windows)
    doesn't use kpathsea, so what do we do? (TODO)

    .. seealso::

      `Kpathsea documentation <http://www.tug.org/kpathsea/>`_
        The library that :program:`kpsewhich` is part of.
    """

    cmd = ['kpsewhich']
    if format is not None:
        cmd += ['--format=' + format]
    cmd += [filename]

    matplotlib.verbose.report('find_tex_file(%s): %s' \
                                  % (filename,cmd), 'debug')
    # stderr is unused, but reading it avoids a subprocess optimization
    # that breaks EINTR handling in some Python versions:
    # http://bugs.python.org/issue12493
    # https://github.com/matplotlib/matplotlib/issues/633
    pipe = subprocess.Popen(cmd, stdout=subprocess.PIPE,
                            stderr=subprocess.PIPE)
    result = pipe.communicate()[0].rstrip()
    matplotlib.verbose.report('find_tex_file result: %s' % result,
                              'debug')
    return result.decode('ascii')

# With multiple text objects per figure (e.g., tick labels) we may end
# up reading the same tfm and vf files many times, so we implement a
# simple cache. TODO: is this worth making persistent?

_tfmcache = {}
_vfcache = {}

def _fontfile(texname, class_, suffix, cache):
    try:
        return cache[texname]
    except KeyError:
        pass

    filename = find_tex_file(texname + suffix)
    if filename:
        result = class_(filename)
    else:
        result = None

    cache[texname] = result
    return result

def _tfmfile(texname):
    return _fontfile(texname, Tfm, '.tfm', _tfmcache)

def _vffile(texname):
    return _fontfile(texname, Vf, '.vf', _vfcache)

if __name__ == '__main__':
    import sys
    matplotlib.verbose.set_level('debug-annoying')
    fname = sys.argv[1]
    try: dpi = float(sys.argv[2])
    except IndexError: dpi = None
    dvi = Dvi(fname, dpi)
    fontmap = PsfontsMap(find_tex_file('pdftex.map'))
    for page in dvi:
        print('=== new page ===')
        fPrev = None
        for x,y,f,c,w in page.text:
            if f != fPrev:
                print('font', f.texname, 'scaled', f._scale/pow(2.0,20))
                fPrev = f
            print(x,y,c, 32 <= c < 128 and chr(c) or '.', w)
        for x,y,w,h in page.boxes:
            print(x,y,'BOX',w,h)


# -*- encoding:utf-8 -*-
import os
import sys
import logging
from copy import deepcopy
from matplotlib import pyplot as plt
import numpy as np
from sklearn.model_selection import train_test_split
reload(sys)

def drawModulation(dirpath, rownum=200):
    """信号文件绘图

    :param filepath: 需要显示绘图的信号文件路径
    :return: None
    """

    plt.figure(1)
    filepaths = os.listdir(dirpath)
    fileorder = 1
    useful_filepaths = [f for f in filepaths if f.startswith('parse_mod')]
    for filepath in useful_filepaths:
        count = np.random.randint(1, rownum + 1)
        with open(dirpath + '/' + filepath, 'rb') as fr:
            x = list()
            vals = list()
            name = filepath
            for i, line in enumerate(fr):
                if i < count:
                    continue
                if i > count:
                    break
                vals = line.strip().split('\t')
                vals = map(float, vals)
                x = range(len(vals))
            plt.subplot(2 * len(useful_filepaths), 1, fileorder * 2 - 1)
            plt.plot(x, vals, color = ((fileorder * 20 + 25) % 255 / 255.,
                                         (fileorder * 5 + 35) % 255 / 255.,
                                         (fileorder * 30 + 45) % 255 / 255.))
            plt.xlabel('symbol number')
            plt.ylabel('signal amplitude')
            plt.title(name)
        fileorder += 1
    plt.show()

def drawMixSignal(filepath, sample=5):
    """信号文件绘图

    :param filepath: 需要显示绘图的信号文件路径
    :return: None
    """

    plt.figure(1)
    with open(filepath, 'rb') as fr:
        rowNumber = sum(1 for _ in fr)
    with open(filepath, 'rb') as fr:
        sampleSignals = set(np.random.choice(range(rowNumber), sample, replace=False))
        rowOrder = 1
        for i, line in enumerate(fr):
            if i not in sampleSignals:
                continue
            vals = line.strip().split('\t')
            vals = map(float, vals)
            x = range(len(vals))
            plt.subplot(sample, 1, rowOrder)
            plt.plot(x, vals, color = ((rowOrder * 20 + 25) % 255 / 255.,
                                         (rowOrder * 5 + 35) % 255 / 255.,
                                         (rowOrder * 30 + 45) % 255 / 255.))
            rowOrder += 1
    plt.show()

def mixSignalAndTagging(dirpath='../data', savepath='../data/mixSignals.txt', modeSize=[]):
    """信号混叠和标注

    对已有的信号进行混叠.
    1-7分别对应：2ASK、QPSK、2FSK、2ASK+QPSK、2ASK+2FSK、QPSK+2FSK、2ASK+QPSK+2FSK

    :param dirpath: signal path
    :param modeSize: the sample size in each mode, from `1` to `n`
    :return: mixed signal
    """

    def tagger(tag):
        """

        给样本打标签,目前手动填写标签类型

        :param tag: like `1\t2`, `0\t2`, `0\t1\t2`
        :return: `int` from 1 to 7 representing label
        """

        if tag == '\t'.join(['0', ]):
            return 1
        elif tag == '\t'.join(['1', ]):
            return 2
        elif tag == '\t'.join(['2', ]):
            return 3
        elif tag == '\t'.join(['0', '1']):
            return 4
        elif tag == '\t'.join(['0', '2']):
            return 5
        elif tag == '\t'.join(['1', '2']):
            return 6
        elif tag == '\t'.join(['0', '1', '2']):
            return 7

    def C(n, m):
        def calcNext(count, point, l, r, res, pre):
            if(point > r):
                return
            if count == 1:
                for i in xrange(point, r + 1):
                    pre.append(i)
                    res.append(deepcopy(pre))
                    pre.pop()
            else:
                for i in xrange(point, r + 1):
                    pre.append(i)
                    calcNext(count - 1, i + 1, l, r, res, pre)
                    pre.pop()
        res = list()
        calcNext(m, 0, 0, n - 1, res, [])
        return res

    files = os.listdir(dirpath)
    signals = {}
    for filepath in files:
        if not filepath.startswith('parse_'):
            continue
        with open(dirpath + '/' + filepath, 'rb') as fr:
            modName = filepath.split('parse_mod_')[1].split('.txt')[0]
            signal = list()
            for line in fr:
                amps = line.strip().split('\t')
                amps = map(float, amps)
                signal.append(amps)
            # signal = zip(*signal)
            # signal = np.tile(signal, (20, 1))
            signals[modName] = signal

    modTypes = np.asarray(signals.keys())
    modeNum = len(modTypes)
    totalSignals = np.array([])
    totalTags = list()
    for mixNum in xrange(1, modeNum + 1):
        groupIndeces = C(modeNum, mixNum)
        groupNum = len(groupIndeces)
        sampleEachMod = modeSize[mixNum - 1] // groupNum
        groupSignals = np.array([])
        for groupInd in groupIndeces:
            mixSignals = np.array([])
            tag = '\t'.join(map(str, sorted(groupInd)))
            tag = str(tagger(tag))
            while len(mixSignals) < sampleEachMod:
                mixSignal = np.zeros([len(signals[modTypes[0]]), len(signals[modTypes[0]][0])])
                for ind in groupInd:
                    curSignal = np.asarray(signals[modTypes[ind]])
                    randomIndeces = np.random.choice(len(curSignal), len(curSignal), replace=False)
                    randSignal = curSignal[randomIndeces]
                    mixSignal += randSignal
                mixSignals = np.concatenate([mixSignals, mixSignal]) if mixSignals.shape[0] != 0 else mixSignal
            mixSignals = mixSignals[:sampleEachMod, :]
            totalTags.extend([tag] * sampleEachMod)
            groupSignals = np.concatenate([groupSignals, mixSignals]) if groupSignals.shape[0] != 0 else mixSignals
        totalSignals = np.concatenate([totalSignals, groupSignals]) if totalSignals.shape[0] != 0 else groupSignals

    assert len(totalTags) == sum(modeSize)
    assert len(totalSignals) == sum(modeSize)

    indeces = np.random.choice(len(totalSignals), len(totalSignals), replace=False)
    totalSignals = np.asarray(totalSignals)[indeces]
    totalTags = np.asarray(totalTags)[indeces]
    with open(savepath, 'wb') as fw:
        for i in xrange(len(totalTags)):
            signal = totalSignals[i]
            signal = map(str, signal)
            tag = totalTags[i]
            fw.write('\t'.join(['\t'.join(signal), tag]) + '\n')

def split(filepath):
    with open(filepath, 'rb') as fr:
        X = list()
        for line in fr:
            X.append(line.strip())
        X_train, X_test = train_test_split(X, test_size=0.2, random_state=42)
    filename = filepath.split('/')[-1]
    dirbase = filepath.split('/')[:-1]
    with open('/'.join(dirbase + ['train_' + filename]), 'wb') as fw:
        for line in X_train:
            fw.write(line + '\n')
    with open('/'.join(dirbase + ['test_' + filename]), 'wb') as fw:
        for line in X_test:
            fw.write(line + '\n')

if __name__ == '__main__':
    # drawModulation('../data/5dB')
    drawMixSignal('../data/50dB/mixSignals.txt')
    # mixSignalAndTagging('../data/5dB', '../data/5dB/mixSignals.txt', [600, 1500, 2000])
    # split('../data/5dB/mixSignals.txt')


from concurrent import futures
import json
import argparse
import time
import random
import itertools
import numpy as np
import os.path
import sys
import pandas as pd
import shutil
import requests
import requests.exceptions
import re
import logging
import json
from datetime import datetime, timedelta

LOGGER_NAME = 'fetch_cnpj'
TEMP_DATASET_PATH = os.path.join('data', 'companies-partial.xz')
INFO_DATASET_PATH = os.path.join('data', '{0}-{1}-{2}-companies.xz')
global logger, cnpj_list, num_threads, proxies_list

# source files mapped for extract cnpj

with open(os.path.join(os.path.dirname(os.path.realpath(__file__)),
                       'table_config.json')) as json_file:
    json_config = json.load(json_file)

datasets_cols = json_config['cnpj_cpf']

def configure_logger(verbosity):
    logger = logging.getLogger(LOGGER_NAME)
    logger.setLevel(verbosity)
    ch = logging.StreamHandler(sys.stdout)
    ch.setLevel(verbosity)
    logger.addHandler(ch)
    return logger

def transform_and_translate_data(json_data):
    """
    Transform main activity, secondary activity and partners list in
    multi columns and translate column names.
    """

    try:
        data = pd.DataFrame(columns=['atividade_principal',
                                     'data_situacao',
                                     'tipo',
                                     'nome',
                                     'telefone',
                                     'atividades_secundarias',
                                     'situacao',
                                     'bairro',
                                     'logradouro',
                                     'numero',
                                     'cep',
                                     'municipio',
                                     'uf',
                                     'abertura',
                                     'natureza_juridica',
                                     'fantasia',
                                     'cnpj',
                                     'ultima_atualizacao',
                                     'status',
                                     'complemento',
                                     'email',
                                     'efr',
                                     'motivo_situacao',
                                     'situacao_especial',
                                     'data_situacao_especial',
                                     'qsa'])

        data = data.append(json_data, ignore_index=True)
    except Exception as e:
        logger.error("Error trying to transform and translate data:")
        logger.error(json_data)
        raise e

    def decompose_main_activity(value):
        struct = value
        if struct:
            return pd.Series(struct[0]). \
                rename_axis({'code': 'main_activity_code',
                             'text': 'main_activity'})
        else:
            return pd.Series({}, index=['main_activity_code', 'main_activity'])

    def decompose_secondary_activities(value):
        struct = value
        if struct and struct[0].get('text') != 'Não informada':
            new_attributes = [pd.Series(activity).
                              rename_axis({'code': 'secondary_activity_%i_code' % (index + 1),
                                           'text': 'secondary_activity_%i' % (index + 1)})
                              for index, activity in enumerate(struct)]
            return pd.concat(new_attributes)
        else:
            return pd.Series()

    def decompose_partners_list(value):
        struct = value
        if struct and len(struct) > 0:
            new_attributes = [pd.Series(partner).
                              rename_axis({
                                  'nome_rep_legal': 'partner_%i_legal_representative_name' % (index + 1),
                                  'qual_rep_legal': 'partner_%i_legal_representative_qualification' % (index + 1),
                                  'pais_origem': 'partner_%i_contry_origin' % (index + 1),
                                  'nome': 'partner_%i_name' % (index + 1),
                                  'qual': 'partner_%i_qualification' % (index + 1)})
                              for index, partner in enumerate(struct)]
            return pd.concat(new_attributes)
        else:
            return pd.Series()

    data = data.rename(columns={
        'abertura': 'opening',
        'atividade_principal': 'main_activity',
        'atividades_secundarias': 'secondary_activities',
        'bairro': 'neighborhood',
        'cep': 'zip_code',
        'complemento': 'additional_address_details',
        'data_situacao_especial': 'special_situation_date',
        'data_situacao': 'situation_date',
        'efr': 'responsible_federative_entity',
        'fantasia': 'trade_name',
        'logradouro': 'address',
        'motivo_situacao': 'situation_reason',
        'municipio': 'city',
        'natureza_juridica': 'legal_entity',
        'nome': 'name',
        'numero': 'number',
        'situacao_especial': 'special_situation',
        'situacao': 'situation',
        'telefone': 'phone',
        'tipo': 'type',
        'uf': 'state',
        'ultima_atualizacao': 'last_updated',
    })
    data['main_activity'] = data['main_activity'].fillna('{}')
    data['secondary_activities'] = data['secondary_activities'].fillna('{}')
    data['qsa'] = data['qsa'].fillna('{}')
    data = pd.concat([
        data.drop(['main_activity', 'secondary_activities', 'qsa'], axis=1),
        data['main_activity'].apply(decompose_main_activity),
        data['secondary_activities'].apply(decompose_secondary_activities),
        data['qsa'].apply(decompose_partners_list)],
        axis=1)
    return data

def load_temp_dataset():
    if os.path.exists(TEMP_DATASET_PATH):
        return pd.read_csv(TEMP_DATASET_PATH, low_memory=False)
    else:
        return pd.DataFrame(columns=['cnpj'])

def read_cnpj_source_files(cnpj_source_files):
    """
    Read the files passed as arguments and extract CNPJs from them.
    The file needs to be mapped in datasets_cols.
    """

    def extract_file_name_from_args(filepath):
        date = re.compile('\d+-\d+-\d+-').findall(os.path.basename(filepath))
        if date:
            filename_without_date = os.path.basename(
                filepath).replace(date[0], '')
        else:
            filename_without_date = os.path.basename(filepath)
        return filename_without_date[:filename_without_date.rfind('.')]

    def read_cnpj_list_to_import(filename, column):
        cnpj_list = pd.read_csv(filename,
                                usecols=([column]),
                                dtype={column: np.str}
                                )[column]
        cnpj_list = cnpj_list.map(lambda cnpj:
                                  str(cnpj).replace(r'[./-]', '')).where(cnpj_list.str.len() == 14).unique()
        return list(cnpj_list)

    filesNotFound = list(filter(lambda file: not os.path.exists(file) or
                                datasets_cols.get(
                                    extract_file_name_from_args(file.lower())) is None,
                                cnpj_source_files))
    filesFound = list(filter(lambda file: os.path.exists(file) and
                             datasets_cols.get(
                                 extract_file_name_from_args(file.lower())),
                             cnpj_source_files))
    cnpj_list_to_import = list(itertools.chain.from_iterable(
        map(lambda file: read_cnpj_list_to_import(file, datasets_cols.get(extract_file_name_from_args(file.lower()))),
            filesFound)))

    return cnpj_list_to_import, filesFound, filesNotFound

def remaining_cnpjs(cnpj_list_to_import, temp_dataset):
    cnpj_list = set(cnpj_list_to_import)
    already_fetched = set(temp_dataset['cnpj'].str.replace(r'[./-]', ''))
    return list(cnpj_list - already_fetched)

def fetch_cnpj_info(cnpj, timeout=60):
    url = 'http://receitaws.com.br/v1/cnpj/%s' % cnpj
    try:
        result = requests.get(url,
                              timeout=timeout,
                              proxies={'http': random.choice(proxies_list + [None])})
        if result.status_code == 200:
            cnpj_list.remove(cnpj)
            json_return = json.loads(result.text.replace(
                '\n', '').replace('\r', '').replace(';', ''))
            return json_return
        elif result.status_code == 429:
            logger.debug('Sleeping 60 seconds to try again.')
            logger.debug(result.text)
            time.sleep(60)
            logger.debug('Thread starting fetch again. {} CNPJs remaining.'.format(
                len(cnpj_list)))
        else:
            logger.debug(result.text)
    except requests.exceptions.Timeout as e:
        logger.debug(e)
    except requests.exceptions.ConnectionError as e:
        logger.debug(e)

parser = argparse.ArgumentParser(epilog="ex: python fetch_cnpj_info.py \
    ./data/2016-12-10-reimbursements.xz 2016-12-14-amendments.xz \
    -p 177.67.84.135:8080 177.67.82.80:8080 -t 10")
parser.add_argument('-v', '--verbosity', default='INFO',
                    help='level of logging messages.')
parser.add_argument('-t', '--threads', type=int, default=10,
                    help='number of threads to use in fetch process')
parser.add_argument('-p', '--proxies', nargs='*',
                    help='proxies list to distribuite requests in fetch process')
parser.add_argument('args', nargs='+',
                    help='source files from where to get CNPJs to fetch')
args = parser.parse_args()

if args.args:
    logger = configure_logger(args.verbosity)
    if args.proxies:
        proxies_list = ['http://' + proxy for proxy in args.proxies]
    else:
        proxies_list = []
    num_threads = args.threads
    cnpj_list_to_import, filesFound, filesNotFound = read_cnpj_source_files(
        args.args)
    temp_dataset = load_temp_dataset()
    cnpj_list = remaining_cnpjs(cnpj_list_to_import, temp_dataset)

    print('%i CNPJ\'s to be fetched' % len(cnpj_list))
    print('Starting fetch. {0} worker threads and {1} http proxies'.format(
        num_threads, len(proxies_list)))

    # Try again in case of error during fetch_cnpj_info
    while len(cnpj_list) > 0:
        with futures.ThreadPoolExecutor(max_workers=num_threads) as executor:
            future_to_cnpj_info = dict((executor.submit(fetch_cnpj_info, cnpj), cnpj)
                                       for cnpj in cnpj_list)
            last_saving_point = 0
            for future in futures.as_completed(future_to_cnpj_info):
                cnpj = future_to_cnpj_info[future]
                if future.exception() is None and future.result() is not None and future.result()['status'] == 'OK':
                    result_translated = transform_and_translate_data(
                        future.result())
                    temp_dataset = pd.concat([temp_dataset, result_translated])
                    if last_saving_point < divmod(len(temp_dataset.index), 100)[0]:
                        last_saving_point = divmod(len(temp_dataset.index), 100)[0]
                        print('###################################')
                        print('Saving information already fetched. {0} records'.format(
                            len(temp_dataset.index)))
                        temp_dataset.to_csv(TEMP_DATASET_PATH,
                                            compression='xz',
                                            encoding='utf-8',
                                            index=False)

    temp_dataset.to_csv(TEMP_DATASET_PATH,
                        compression='xz',
                        encoding='utf-8',
                        index=False)
    os.rename(TEMP_DATASET_PATH, INFO_DATASET_PATH.format(
        datetime.today().strftime("%Y"),
        datetime.today().strftime("%m"),
        datetime.today().strftime("%d")))

    if len(filesNotFound) > 0:
        print('The following files were not found:')
        for file in filesNotFound:
            print(file)
        print('Maybe they were misspelled or the CNPJ\'s columns are not mapped:')
        for file in datasets_cols:
            print('File: %s | Column: %s' % (file, datasets_cols[file]))

    print('%i CNPJ\'s listed in file(s)' % len(set(cnpj_list_to_import)))
    cnpj_list = remaining_cnpjs(cnpj_list_to_import, temp_dataset)
    print('%i CNPJ\'s remaining' % len(cnpj_list))
else:
    print('no files to fetch CNPJ\'s')
    print('type python fetch_cnpj_info.py -h for help')