# coding: utf-8
# Copyright (c) Pymatgen Development Team.
# Distributed under the terms of the MIT License.

"""
This module implements input and output processing from Gaussian.
"""

import re
import warnings

import numpy as np
import scipy.constants as cst
from monty.io import zopen

from pymatgen.core.composition import Composition
from pymatgen.core.periodic_table import Element
from pymatgen.core.structure import Molecule
from pymatgen.core.operations import SymmOp
from pymatgen.core.units import Ha_to_eV
from pymatgen.electronic_structure.core import Spin
from pymatgen.util.coord import get_angle

__author__ = "Shyue Ping Ong, Germain  Salvato-Vallverdu, Xin Chen"
__copyright__ = "Copyright 2013, The Materials Virtual Lab"
__version__ = "0.1"
__maintainer__ = "Shyue Ping Ong"
__email__ = "ongsp@ucsd.edu"
__date__ = "8/1/15"

float_patt = re.compile(r"\s*([+-]?\d+\.\d+)")

def read_route_line(route):
    """
    read route line in gaussian input/output and return functional basis_set
    and a dictionary of other route parameters

    Args:
        route (str) : the route line

    return
        functional (str) : the method (HF, PBE ...)
        basis_set (str) : the basis set
        route (dict) : dictionary of parameters
    """
    scrf_patt = re.compile(r"^([sS][cC][rR][fF])\s*=\s*(.+)")
    multi_params_patt = re.compile(r"^([A-z]+[0-9]*)[\s=]+\((.*)\)$")
    functional = None
    basis_set = None
    route_params = {}
    dieze_tag = None
    if route:
        if "/" in route:
            tok = route.split("/")
            functional = tok[0].split()[-1]
            basis_set = tok[1].split()[0]
            for tok in [functional, basis_set, "/"]:
                route = route.replace(tok, "")

        for tok in route.split():
            if scrf_patt.match(tok):
                m = scrf_patt.match(tok)
                route_params[m.group(1)] = m.group(2)
            elif tok.upper() in ["#", "#N", "#P", "#T"]:
                # does not store # in route to avoid error in input
                if tok == "#":
                    dieze_tag = "#N"
                else:
                    dieze_tag = tok
                continue
            else:
                m = re.match(multi_params_patt, tok.strip("#"))
                if m:
                    pars = {}
                    for par in m.group(2).split(","):
                        p = par.split("=")
                        pars[p[0]] = None if len(p) == 1 else p[1]
                    route_params[m.group(1)] = pars
                else:
                    d = tok.strip("#").split("=")
                    route_params[d[0]] = None if len(d) == 1 else d[1]

    return functional, basis_set, route_params, dieze_tag

class GaussianInput:
    """
    An object representing a Gaussian input file.
    """

    # Commonly used regex patterns
    _zmat_patt = re.compile(r"^(\w+)*([\s,]+(\w+)[\s,]+(\w+))*[\-\.\s,\w]*$")
    _xyz_patt = re.compile(r"^(\w+)[\s,]+([\d\.eE\-]+)[\s,]+([\d\.eE\-]+)[\s,]+" r"([\d\.eE\-]+)[\-\.\s,\w.]*$")

    def __init__(
        self,
        mol,
        charge=None,
        spin_multiplicity=None,
        title=None,
        functional="HF",
        basis_set="6-31G(d)",
        route_parameters=None,
        input_parameters=None,
        link0_parameters=None,
        dieze_tag="#P",
        gen_basis=None,
    ):
        """
        Args:
            mol: Input molecule. It can either be a Molecule object,
                a string giving the geometry in a format supported by Guassian,
                or ``None``. If the molecule is ``None``, you will need to use
                read it in from a checkpoint. Consider adding ``CHK`` to the
                ``link0_parameters``.
            charge: Charge of the molecule. If None, charge on molecule is used.
                Defaults to None. This allows the input file to be set a
                charge independently from the molecule itself.
                If ``mol`` is not a Molecule object, then you must specify a charge.
            spin_multiplicity: Spin multiplicity of molecule. Defaults to None,
                which means that the spin multiplicity is set to 1 if the
                molecule has no unpaired electrons and to 2 if there are
                unpaired electrons. If ``mol`` is not a Molecule object, then you
                 must specify the multiplicity
            title: Title for run. Defaults to formula of molecule if None.
            functional: Functional for run.
            basis_set: Basis set for run.
            route_parameters: Additional route parameters as a dict. For example,
                {'SP':"", "SCF":"Tight"}
            input_parameters: Additional input parameters for run as a dict. Used
                for example, in PCM calculations.  E.g., {"EPS":12}
            link0_parameters: Link0 parameters as a dict. E.g., {"%mem": "1000MW"}
            dieze_tag: # preceding the route line. E.g. "#p"
            gen_basis: allows a user-specified basis set to be used in a Gaussian
                calculation. If this is not None, the attribute ``basis_set`` will
                be set to "Gen".
        """
        self._mol = mol

        # Determine multiplicity and charge settings
        if isinstance(mol, Molecule):
            self.charge = charge if charge is not None else mol.charge
            nelectrons = mol.charge + mol.nelectrons - self.charge
            if spin_multiplicity is not None:
                self.spin_multiplicity = spin_multiplicity
                if (nelectrons + spin_multiplicity) % 2 != 1:
                    raise ValueError(
                        "Charge of {} and spin multiplicity of {} is"
                        " not possible for this molecule".format(self.charge, spin_multiplicity)
                    )
            else:
                self.spin_multiplicity = 1 if nelectrons % 2 == 0 else 2

            # Get a title from the molecule name
            self.title = title if title else self._mol.composition.formula
        else:
            self.charge = charge
            self.spin_multiplicity = spin_multiplicity

            # Set a title
            self.title = title if title else "Restart"

        # Store the remaining settings
        self.functional = functional
        self.basis_set = basis_set
        self.link0_parameters = link0_parameters if link0_parameters else {}
        self.route_parameters = route_parameters if route_parameters else {}
        self.input_parameters = input_parameters if input_parameters else {}
        self.dieze_tag = dieze_tag if dieze_tag[0] == "#" else "#" + dieze_tag
        self.gen_basis = gen_basis
        if gen_basis is not None:
            self.basis_set = "Gen"

    @property
    def molecule(self):
        """
        Returns molecule associated with this GaussianInput.
        """
        return self._mol

    @staticmethod
    def _parse_coords(coord_lines):
        """
        Helper method to parse coordinates.
        """
        paras = {}
        var_pattern = re.compile(r"^([A-Za-z]+\S*)[\s=,]+([\d\-\.]+)$")
        for l in coord_lines:
            m = var_pattern.match(l.strip())
            if m:
                paras[m.group(1).strip("=")] = float(m.group(2))

        species = []
        coords = []
        # Stores whether a Zmatrix format is detected. Once a zmatrix format
        # is detected, it is assumed for the remaining of the parsing.
        zmode = False
        for l in coord_lines:
            l = l.strip()
            if not l:
                break
            if (not zmode) and GaussianInput._xyz_patt.match(l):
                m = GaussianInput._xyz_patt.match(l)
                species.append(m.group(1))
                toks = re.split(r"[,\s]+", l.strip())
                if len(toks) > 4:
                    coords.append([float(i) for i in toks[2:5]])
                else:
                    coords.append([float(i) for i in toks[1:4]])
            elif GaussianInput._zmat_patt.match(l):
                zmode = True
                toks = re.split(r"[,\s]+", l.strip())
                species.append(toks[0])
                toks.pop(0)
                if len(toks) == 0:
                    coords.append(np.array([0, 0, 0]))
                else:
                    nn = []
                    parameters = []
                    while len(toks) > 1:
                        ind = toks.pop(0)
                        data = toks.pop(0)
                        try:
                            nn.append(int(ind))
                        except ValueError:
                            nn.append(species.index(ind) + 1)
                        try:
                            val = float(data)
                            parameters.append(val)
                        except ValueError:
                            if data.startswith("-"):
                                parameters.append(-paras[data[1:]])
                            else:
                                parameters.append(paras[data])
                    if len(nn) == 1:
                        coords.append(np.array([0, 0, parameters[0]]))
                    elif len(nn) == 2:
                        coords1 = coords[nn[0] - 1]
                        coords2 = coords[nn[1] - 1]
                        bl = parameters[0]
                        angle = parameters[1]
                        axis = [0, 1, 0]
                        op = SymmOp.from_origin_axis_angle(coords1, axis, angle, False)
                        coord = op.operate(coords2)
                        vec = coord - coords1
                        coord = vec * bl / np.linalg.norm(vec) + coords1
                        coords.append(coord)
                    elif len(nn) == 3:
                        coords1 = coords[nn[0] - 1]
                        coords2 = coords[nn[1] - 1]
                        coords3 = coords[nn[2] - 1]
                        bl = parameters[0]
                        angle = parameters[1]
                        dih = parameters[2]
                        v1 = coords3 - coords2
                        v2 = coords1 - coords2
                        axis = np.cross(v1, v2)
                        op = SymmOp.from_origin_axis_angle(coords1, axis, angle, False)
                        coord = op.operate(coords2)
                        v1 = coord - coords1
                        v2 = coords1 - coords2
                        v3 = np.cross(v1, v2)
                        adj = get_angle(v3, axis)
                        axis = coords1 - coords2
                        op = SymmOp.from_origin_axis_angle(coords1, axis, dih - adj, False)
                        coord = op.operate(coord)
                        vec = coord - coords1
                        coord = vec * bl / np.linalg.norm(vec) + coords1
                        coords.append(coord)

        def _parse_species(sp_str):
            """
            The species specification can take many forms. E.g.,
            simple integers representing atomic numbers ("8"),
            actual species string ("C") or a labelled species ("C1").
            Sometimes, the species string is also not properly capitalized,
            e.g, ("c1"). This method should take care of these known formats.
            """
            try:
                return int(sp_str)
            except ValueError:
                sp = re.sub(r"\d", "", sp_str)
                return sp.capitalize()

        species = [_parse_species(sp) for sp in species]

        return Molecule(species, coords)

    @staticmethod
    def from_string(contents):
        """
        Creates GaussianInput from a string.

        Args:
            contents: String representing an Gaussian input file.

        Returns:
            GaussianInput object
        """
        lines = [l.strip() for l in contents.split("\n")]

        link0_patt = re.compile(r"^(%.+)\s*=\s*(.+)")
        link0_dict = {}
        for i, l in enumerate(lines):
            if link0_patt.match(l):
                m = link0_patt.match(l)
                link0_dict[m.group(1).strip("=")] = m.group(2)

        route_patt = re.compile(r"^#[sSpPnN]*.*")
        route = ""
        route_index = None
        for i, l in enumerate(lines):
            if route_patt.match(l):
                route += " " + l
                route_index = i
            # This condition allows for route cards spanning multiple lines
            elif (l == "" or l.isspace()) and route_index:
                break
        functional, basis_set, route_paras, dieze_tag = read_route_line(route)
        ind = 2
        title = []
        while lines[route_index + ind].strip():
            title.append(lines[route_index + ind].strip())
            ind += 1
        title = " ".join(title)
        ind += 1
        toks = re.split(r"[,\s]+", lines[route_index + ind])
        charge = int(float(toks[0]))
        spin_mult = int(toks[1])
        coord_lines = []
        spaces = 0
        input_paras = {}
        ind += 1
        for i in range(route_index + ind, len(lines)):
            if lines[i].strip() == "":
                spaces += 1
            if spaces >= 2:
                d = lines[i].split("=")
                if len(d) == 2:
                    input_paras[d[0]] = d[1]
            else:
                coord_lines.append(lines[i].strip())
        mol = GaussianInput._parse_coords(coord_lines)
        mol.set_charge_and_spin(charge, spin_mult)

        return GaussianInput(
            mol,
            charge=charge,
            spin_multiplicity=spin_mult,
            title=title,
            functional=functional,
            basis_set=basis_set,
            route_parameters=route_paras,
            input_parameters=input_paras,
            link0_parameters=link0_dict,
            dieze_tag=dieze_tag,
        )

    @staticmethod
    def from_file(filename):
        """
        Creates GaussianInput from a file.

        Args:
            filename: Gaussian input filename

        Returns:
            GaussianInput object
        """
        with zopen(filename, "r") as f:
            return GaussianInput.from_string(f.read())

    def _find_nn_pos_before_site(self, siteindex):
        """
        Returns index of nearest neighbor atoms.
        """
        alldist = [(self._mol.get_distance(siteindex, i), i) for i in range(siteindex)]
        alldist = sorted(alldist, key=lambda x: x[0])
        return [d[1] for d in alldist]

    def get_zmatrix(self):
        """
        Returns a z-matrix representation of the molecule.
        """
        output = []
        outputvar = []
        for i, site in enumerate(self._mol):
            if i == 0:
                output.append("{}".format(site.specie))
            elif i == 1:
                nn = self._find_nn_pos_before_site(i)
                bondlength = self._mol.get_distance(i, nn[0])
                output.append("{} {} B{}".format(self._mol[i].specie, nn[0] + 1, i))
                outputvar.append("B{}={:.6f}".format(i, bondlength))
            elif i == 2:
                nn = self._find_nn_pos_before_site(i)
                bondlength = self._mol.get_distance(i, nn[0])
                angle = self._mol.get_angle(i, nn[0], nn[1])
                output.append("{} {} B{} {} A{}".format(self._mol[i].specie, nn[0] + 1, i, nn[1] + 1, i))
                outputvar.append("B{}={:.6f}".format(i, bondlength))
                outputvar.append("A{}={:.6f}".format(i, angle))
            else:
                nn = self._find_nn_pos_before_site(i)
                bondlength = self._mol.get_distance(i, nn[0])
                angle = self._mol.get_angle(i, nn[0], nn[1])
                dih = self._mol.get_dihedral(i, nn[0], nn[1], nn[2])
                output.append(
                    "{} {} B{} {} A{} {} D{}".format(self._mol[i].specie, nn[0] + 1, i, nn[1] + 1, i, nn[2] + 1, i)
                )
                outputvar.append("B{}={:.6f}".format(i, bondlength))
                outputvar.append("A{}={:.6f}".format(i, angle))
                outputvar.append("D{}={:.6f}".format(i, dih))
        return "\n".join(output) + "\n\n" + "\n".join(outputvar)

    def get_cart_coords(self):
        """
        Return the cartesian coordinates of the molecule
        """

        def to_s(x):
            return "%0.6f" % x

        outs = []
        for i, site in enumerate(self._mol):
            outs.append(" ".join([site.species_string, " ".join([to_s(j) for j in site.coords])]))
        return "\n".join(outs)

    def __str__(self):
        return self.to_string()

    def to_string(self, cart_coords=False):
        """
        Return GaussianInput string

        Option: whe cart_coords sets to True return the cartesian coordinates
                instead of the z-matrix

        """

        def para_dict_to_string(para, joiner=" "):
            para_str = []
            # sorted is only done to make unittests work reliably
            for par, val in sorted(para.items()):
                if val is None or val == "":
                    para_str.append(par)
                elif isinstance(val, dict):
                    val_str = para_dict_to_string(val, joiner=",")
                    para_str.append("{}=({})".format(par, val_str))
                else:
                    para_str.append("{}={}".format(par, val))
            return joiner.join(para_str)

        output = []
        if self.link0_parameters:
            output.append(para_dict_to_string(self.link0_parameters, "\n"))

        # Handle functional or basis set set to None, empty string or whitespace
        func_str = "" if self.functional is None else self.functional.strip()
        bset_str = "" if self.basis_set is None else self.basis_set.strip()

        if func_str != "" and bset_str != "":
            func_bset_str = " {}/{}".format(func_str, bset_str)
        else:
            # don't use the slash if either or both are set as empty
            func_bset_str = " {}{}".format(func_str, bset_str).rstrip()

        output.append(
            "{diez}{func_bset} {route}".format(
                diez=self.dieze_tag,
                func_bset=func_bset_str,
                route=para_dict_to_string(self.route_parameters),
            )
        )
        output.append("")
        output.append(self.title)
        output.append("")

        charge_str = "" if self.charge is None else "%d" % self.charge
        multip_str = "" if self.spin_multiplicity is None else " %d" % self.spin_multiplicity
        output.append("{}{}".format(charge_str, multip_str))

        if isinstance(self._mol, Molecule):
            if cart_coords is True:
                output.append(self.get_cart_coords())
            else:
                output.append(self.get_zmatrix())
        elif self._mol is not None:
            output.append(str(self._mol))
        output.append("")
        if self.gen_basis is not None:
            output.append("{:s}\n".format(self.gen_basis))
        output.append(para_dict_to_string(self.input_parameters, "\n"))
        output.append("\n")
        return "\n".join(output)

    def write_file(self, filename, cart_coords=False):
        """
        Write the input string into a file

        Option: see __str__ method
        """
        with zopen(filename, "w") as f:
            f.write(self.to_string(cart_coords))

    def as_dict(self):
        """
        :return: MSONable dict
        """
        return {
            "@module": self.__class__.__module__,
            "@class": self.__class__.__name__,
            "molecule": self.molecule.as_dict(),
            "functional": self.functional,
            "basis_set": self.basis_set,
            "route_parameters": self.route_parameters,
            "title": self.title,
            "charge": self.charge,
            "spin_multiplicity": self.spin_multiplicity,
            "input_parameters": self.input_parameters,
            "link0_parameters": self.link0_parameters,
            "dieze_tag": self.dieze_tag,
        }

    @classmethod
    def from_dict(cls, d):
        """
        :param d: dict
        :return: GaussianInput
        """
        return GaussianInput(
            mol=Molecule.from_dict(d["molecule"]),
            functional=d["functional"],
            basis_set=d["basis_set"],
            route_parameters=d["route_parameters"],
            title=d["title"],
            charge=d["charge"],
            spin_multiplicity=d["spin_multiplicity"],
            input_parameters=d["input_parameters"],
            link0_parameters=d["link0_parameters"],
        )

class GaussianOutput:
    """
    Parser for Gaussian output files.

    .. note::

        Still in early beta.

    Attributes:

    .. attribute:: structures

        All structures from the calculation in the standard orientation. If the
        symmetry is not considered, the standard orientation is not printed out
        and the input orientation is used instead. Check the `standard_orientation`
        attribute.

    .. attribute:: structures_input_orientation

        All structures from the calculation in the input orientation or the
        Z-matrix orientation (if an opt=z-matrix was requested).

    .. attribute:: opt_structures

        All optimized structures from the calculation in the standard orientation,
        if the attribute 'standard_orientation' is True, otherwise in the input
        or the Z-matrix orientation.

    .. attribute:: energies

        All energies from the calculation.

    .. attribute:: eigenvalues

        List of eigenvalues for the last geometry

    .. attribute:: MO_coefficients

        Matrix of MO coefficients for the last geometry

    .. attribute:: cart_forces

        All cartesian forces from the calculation.

    .. attribute:: frequencies

        A list for each freq calculation and for each mode of a dict with
        {
            "frequency": freq in cm-1,
            "symmetry": symmetry tag
            "r_mass": Reduce mass,
            "f_constant": force constant,
            "IR_intensity": IR Intensity,
            "mode": normal mode
         }

        The normal mode is a 1D vector of dx, dy dz of each atom.

    .. attribute:: hessian

        Matrix of second derivatives of the energy with respect to cartesian
        coordinates in the **input orientation** frame. Need #P in the
        route section in order to be in the output.

    .. attribute:: properly_terminated

        True if run has properly terminated

    .. attribute:: is_pcm

        True if run is a PCM run.

    .. attribute:: is_spin

        True if it is an unrestricted run

    .. attribute:: stationary_type

        If it is a relaxation run, indicates whether it is a minimum (Minimum)
        or a saddle point ("Saddle").

    .. attribute:: corrections

        Thermochemical corrections if this run is a Freq run as a dict. Keys
        are "Zero-point", "Thermal", "Enthalpy" and "Gibbs Free Energy"

    .. attribute:: functional

        Functional used in the run.

    .. attribute:: basis_set

        Basis set used in the run

    .. attribute:: route

        Additional route parameters as a dict. For example,
            {'SP':"", "SCF":"Tight"}

    .. attribute:: dieze_tag

        # preceding the route line, e.g. "#P"

    .. attribute:: link0

        Link0 parameters as a dict. E.g., {"%mem": "1000MW"}

    .. attribute:: charge

        Charge for structure

    .. attribute:: spin_multiplicity

        Spin multiplicity for structure

    .. attribute:: num_basis_func

        Number of basis functions in the run.

    .. attribute:: electrons

        number of alpha and beta electrons as (N alpha, N beta)

    .. attribute:: pcm

        PCM parameters and output if available.

    .. attribute:: errors

        error if not properly terminated (list to be completed in error_defs)

    .. attribute:: Mulliken_charges

        Mulliken atomic charges

    .. attribute:: eigenvectors

        Matrix of shape (num_basis_func, num_basis_func). Each column is an
        eigenvectors and contains AO coefficients of an MO.

        eigenvectors[Spin] = mat(num_basis_func, num_basis_func)

    .. attribute:: molecular_orbital

        MO development coefficients on AO in a more convenient array dict
        for each atom and basis set label.

        mo[Spin][OM j][atom i] = {AO_k: coeff, AO_k: coeff ... }

    .. attribute:: atom_basis_labels

        Labels of AO for each atoms. These labels are those used in the output
        of molecular orbital coefficients (POP=Full) and in the
        molecular_orbital array dict.

        atom_basis_labels[iatom] = [AO_k, AO_k, ...]

    .. attribute:: resumes

        List of gaussian data resume given at the end of the output file before
        the quotation. The resumes are given as string.

    .. attribute:: title

        Title of the gaussian run.

    .. attribute:: standard_orientation

        If True, the geometries stored in the structures are in the standard
        orientation. Else, the geometries are in the input orientation.

    .. attribute:: bond_orders

        Dict of bond order values read in the output file such as:
        {(0, 1): 0.8709, (1, 6): 1.234, ...}

        The keys are the atom indexes and the values are the Wiberg bond indexes
        that are printed using `pop=NBOREAD` and `$nbo bndidx $end`.

    Methods:

    .. method:: to_input()

        Return a GaussianInput object using the last geometry and the same
        calculation parameters.

    .. method:: read_scan()

        Read a potential energy surface from a gaussian scan calculation.

    .. method:: get_scan_plot()

        Get a matplotlib plot of the potential energy surface

    .. method:: save_scan_plot()

        Save a matplotlib plot of the potential energy surface to a file

    """

    def __init__(self, filename):
        """
        Args:
            filename: Filename of Gaussian output file.
        """
        self.filename = filename
        self._parse(filename)

    @property
    def final_energy(self):
        """
        :return: Final energy in Gaussian output.
        """
        return self.energies[-1]

    @property
    def final_structure(self):
        """
        :return: Final structure in Gaussian output.
        """
        return self.structures[-1]

    def _parse(self, filename):
        start_patt = re.compile(r" \(Enter \S+l101\.exe\)")
        route_patt = re.compile(r" #[pPnNtT]*.*")
        link0_patt = re.compile(r"^\s(%.+)\s*=\s*(.+)")
        charge_mul_patt = re.compile(r"Charge\s+=\s*([-\d]+)\s+" r"Multiplicity\s+=\s*(\d+)")
        num_basis_func_patt = re.compile(r"([0-9]+)\s+basis functions")
        num_elec_patt = re.compile(r"(\d+)\s+alpha electrons\s+(\d+)\s+beta electrons")
        pcm_patt = re.compile(r"Polarizable Continuum Model")
        stat_type_patt = re.compile(r"imaginary frequencies")
        scf_patt = re.compile(r"E\(.*\)\s*=\s*([-\.\d]+)\s+")
        mp2_patt = re.compile(r"EUMP2\s*=\s*(.*)")
        oniom_patt = re.compile(r"ONIOM:\s+extrapolated energy\s*=\s*(.*)")
        termination_patt = re.compile(r"(Normal|Error) termination")
        error_patt = re.compile(r"(! Non-Optimized Parameters !|Convergence failure)")
        mulliken_patt = re.compile(r"^\s*(Mulliken charges|Mulliken atomic charges)")
        mulliken_charge_patt = re.compile(r"^\s+(\d+)\s+([A-Z][a-z]?)\s*(\S*)")
        end_mulliken_patt = re.compile(r"(Sum of Mulliken )(.*)(charges)\s*=\s*(\D)")
        std_orientation_patt = re.compile(r"Standard orientation")
        input_orientation_patt = re.compile(r"Input orientation|Z-Matrix orientation")
        orbital_patt = re.compile(r"(Alpha|Beta)\s*\S+\s*eigenvalues --(.*)")
        thermo_patt = re.compile(r"(Zero-point|Thermal) correction(.*)=" r"\s+([\d\.-]+)")
        forces_on_patt = re.compile(r"Center\s+Atomic\s+Forces\s+\(Hartrees/Bohr\)")
        forces_off_patt = re.compile(r"Cartesian\s+Forces:\s+Max.*RMS.*")
        forces_patt = re.compile(r"\s+(\d+)\s+(\d+)\s+([0-9\.-]+)\s+([0-9\.-]+)\s+([0-9\.-]+)")

        freq_on_patt = re.compile(r"Harmonic\sfrequencies\s+\(cm\*\*-1\),\sIR\sintensities.*Raman.*")

        normal_mode_patt = re.compile(r"\s+(\d+)\s+(\d+)\s+([0-9\.-]{4,5})\s+([0-9\.-]{4,5}).*")

        mo_coeff_patt = re.compile(r"Molecular Orbital Coefficients:")
        mo_coeff_name_patt = re.compile(r"\d+\s((\d+|\s+)\s+([a-zA-Z]{1,2}|\s+))\s+(\d+\S+)")

        hessian_patt = re.compile(r"Force constants in Cartesian coordinates:")
        resume_patt = re.compile(r"^\s1\\1\\GINC-\S*")
        resume_end_patt = re.compile(r"^\s.*\\\\@")

        bond_order_patt = re.compile(r"Wiberg bond index matrix in the NAO basis:")

        self.properly_terminated = False
        self.is_pcm = False
        self.stationary_type = "Minimum"
        self.corrections = {}
        self.energies = []
        self.pcm = None
        self.errors = []
        self.Mulliken_charges = {}
        self.link0 = {}
        self.cart_forces = []
        self.frequencies = []
        self.eigenvalues = []
        self.is_spin = False
        self.hessian = None
        self.resumes = []
        self.title = None
        self.bond_orders = {}

        read_coord = 0
        read_mulliken = False
        read_eigen = False
        eigen_txt = []
        parse_stage = 0
        num_basis_found = False
        terminated = False
        parse_forces = False
        forces = []
        parse_freq = False
        frequencies = []
        read_mo = False
        parse_hessian = False
        routeline = ""
        standard_orientation = False
        parse_bond_order = False
        input_structures = list()
        std_structures = list()
        geom_orientation = None
        opt_structures = list()

        with zopen(filename) as f:
            for line in f:
                if parse_stage == 0:
                    if start_patt.search(line):
                        parse_stage = 1
                    elif link0_patt.match(line):
                        m = link0_patt.match(line)
                        self.link0[m.group(1)] = m.group(2)
                    elif route_patt.search(line) or routeline != "":
                        if set(line.strip()) == {"-"}:
                            params = read_route_line(routeline)
                            self.functional = params[0]
                            self.basis_set = params[1]
                            self.route_parameters = params[2]
                            route_lower = {k.lower(): v for k, v in self.route_parameters.items()}
                            self.dieze_tag = params[3]
                            parse_stage = 1
                        else:
                            routeline += line.strip()
                elif parse_stage == 1:
                    if set(line.strip()) == {"-"} and self.title is None:
                        self.title = ""
                    elif self.title == "":
                        self.title = line.strip()
                    elif charge_mul_patt.search(line):
                        m = charge_mul_patt.search(line)
                        self.charge = int(m.group(1))
                        self.spin_multiplicity = int(m.group(2))
                        parse_stage = 2
                elif parse_stage == 2:

                    if self.is_pcm:
                        self._check_pcm(line)

                    if "freq" in route_lower and thermo_patt.search(line):
                        m = thermo_patt.search(line)
                        if m.group(1) == "Zero-point":
                            self.corrections["Zero-point"] = float(m.group(3))
                        else:
                            key = m.group(2).strip(" to ")
                            self.corrections[key] = float(m.group(3))

                    if read_coord:
                        [f.readline() for i in range(3)]
                        line = f.readline()
                        sp = []
                        coords = []
                        while set(line.strip()) != {"-"}:
                            toks = line.split()
                            sp.append(Element.from_Z(int(toks[1])))
                            coords.append([float(x) for x in toks[3:6]])
                            line = f.readline()

                        read_coord = False
                        if geom_orientation == "input":
                            input_structures.append(Molecule(sp, coords))
                        elif geom_orientation == "standard":
                            std_structures.append(Molecule(sp, coords))

                    if parse_forces:
                        m = forces_patt.search(line)
                        if m:
                            forces.extend([float(_v) for _v in m.groups()[2:5]])
                        elif forces_off_patt.search(line):
                            self.cart_forces.append(forces)
                            forces = []
                            parse_forces = False

                    # read molecular orbital eigenvalues
                    if read_eigen:
                        m = orbital_patt.search(line)
                        if m:
                            eigen_txt.append(line)
                        else:
                            read_eigen = False
                            self.eigenvalues = {Spin.up: []}
                            for eigenline in eigen_txt:
                                if "Alpha" in eigenline:
                                    self.eigenvalues[Spin.up] += [float(e) for e in float_patt.findall(eigenline)]
                                elif "Beta" in eigenline:
                                    if Spin.down not in self.eigenvalues:
                                        self.eigenvalues[Spin.down] = []
                                    self.eigenvalues[Spin.down] += [float(e) for e in float_patt.findall(eigenline)]
                            eigen_txt = []

                    # read molecular orbital coefficients
                    if (not num_basis_found) and num_basis_func_patt.search(line):
                        m = num_basis_func_patt.search(line)
                        self.num_basis_func = int(m.group(1))
                        num_basis_found = True
                    elif read_mo:
                        # build a matrix with all coefficients
                        all_spin = [Spin.up]
                        if self.is_spin:
                            all_spin.append(Spin.down)

                        mat_mo = {}
                        for spin in all_spin:
                            mat_mo[spin] = np.zeros((self.num_basis_func, self.num_basis_func))
                            nMO = 0
                            end_mo = False
                            while nMO < self.num_basis_func and not end_mo:
                                f.readline()
                                f.readline()
                                self.atom_basis_labels = []
                                for i in range(self.num_basis_func):
                                    line = f.readline()

                                    # identify atom and OA labels
                                    m = mo_coeff_name_patt.search(line)
                                    if m.group(1).strip() != "":
                                        iat = int(m.group(2)) - 1
                                        # atname = m.group(3)
                                        self.atom_basis_labels.append([m.group(4)])
                                    else:
                                        self.atom_basis_labels[iat].append(m.group(4))

                                    # MO coefficients
                                    coeffs = [float(c) for c in float_patt.findall(line)]
                                    for j, c in enumerate(coeffs):
                                        mat_mo[spin][i, nMO + j] = c

                                nMO += len(coeffs)
                                line = f.readline()
                                # manage pop=regular case (not all MO)
                                if nMO < self.num_basis_func and (
                                    "Density Matrix:" in line or mo_coeff_patt.search(line)
                                ):
                                    end_mo = True
                                    warnings.warn("POP=regular case, matrix " "coefficients not complete")
                            f.readline()

                        self.eigenvectors = mat_mo
                        read_mo = False

                        # build a more convenient array dict with MO
                        # coefficient of each atom in each MO.
                        # mo[Spin][OM j][atom i] =
                        # {AO_k: coeff, AO_k: coeff ... }
                        mo = {}
                        for spin in all_spin:
                            mo[spin] = [
                                [{} for iat in range(len(self.atom_basis_labels))] for j in range(self.num_basis_func)
                            ]
                            for j in range(self.num_basis_func):
                                i = 0
                                for iat in range(len(self.atom_basis_labels)):
                                    for label in self.atom_basis_labels[iat]:
                                        mo[spin][j][iat][label] = self.eigenvectors[spin][i, j]
                                        i += 1

                        self.molecular_orbital = mo

                    elif parse_freq:
                        while line.strip() != "":  #  blank line
                            ifreqs = [int(val) - 1 for val in line.split()]
                            for ifreq in ifreqs:
                                frequencies.append(
                                    {
                                        "frequency": None,
                                        "r_mass": None,
                                        "f_constant": None,
                                        "IR_intensity": None,
                                        "symmetry": None,
                                        "mode": [],
                                    }
                                )
                            # read freq, intensity, masses, symmetry ...
                            while "Atom  AN" not in line:
                                if "Frequencies --" in line:
                                    freqs = map(float, float_patt.findall(line))
                                    for ifreq, freq in zip(ifreqs, freqs):
                                        frequencies[ifreq]["frequency"] = freq
                                elif "Red. masses --" in line:
                                    r_masses = map(float, float_patt.findall(line))
                                    for ifreq, r_mass in zip(ifreqs, r_masses):
                                        frequencies[ifreq]["r_mass"] = r_mass
                                elif "Frc consts  --" in line:
                                    f_consts = map(float, float_patt.findall(line))
                                    for ifreq, f_const in zip(ifreqs, f_consts):
                                        frequencies[ifreq]["f_constant"] = f_const
                                elif "IR Inten    --" in line:
                                    IR_intens = map(float, float_patt.findall(line))
                                    for ifreq, intens in zip(ifreqs, IR_intens):
                                        frequencies[ifreq]["IR_intensity"] = intens
                                else:
                                    syms = line.split()[:3]
                                    for ifreq, sym in zip(ifreqs, syms):
                                        frequencies[ifreq]["symmetry"] = sym
                                line = f.readline()

                            #  read normal modes
                            line = f.readline()
                            while normal_mode_patt.search(line):
                                values = list(map(float, float_patt.findall(line)))
                                for i, ifreq in zip(range(0, len(values), 3), ifreqs):
                                    frequencies[ifreq]["mode"].extend(values[i : i + 3])
                                line = f.readline()

                        parse_freq = False
                        self.frequencies.append(frequencies)
                        frequencies = []

                    elif parse_hessian:
                        #  read Hessian matrix under "Force constants in Cartesian coordinates"
                        #  Hessian matrix is in the input  orientation framework
                        # WARNING : need #P in the route line
                        parse_hessian = False
                        ndf = 3 * len(input_structures[0])
                        self.hessian = np.zeros((ndf, ndf))
                        j_indices = range(5)
                        jndf = 0
                        while jndf < ndf:
                            for i in range(jndf, ndf):
                                line = f.readline()
                                vals = re.findall(r"\s*([+-]?\d+\.\d+[eEdD]?[+-]\d+)", line)
                                vals = [float(val.replace("D", "E")) for val in vals]
                                for jval, val in enumerate(vals):
                                    j = j_indices[jval]
                                    self.hessian[i, j] = val
                                    self.hessian[j, i] = val
                            jndf += len(vals)
                            line = f.readline()
                            j_indices = [j + 5 for j in j_indices]

                    elif parse_bond_order:
                        # parse Wiberg bond order
                        line = f.readline()
                        line = f.readline()
                        nat = len(input_structures[0])
                        matrix = list()
                        for iat in range(nat):
                            line = f.readline()
                            matrix.append([float(v) for v in line.split()[2:]])

                        self.bond_orders = dict()
                        for iat in range(nat):
                            for jat in range(iat + 1, nat):
                                self.bond_orders[(iat, jat)] = matrix[iat][jat]
                        parse_bond_order = False

                    elif termination_patt.search(line):
                        m = termination_patt.search(line)
                        if m.group(1) == "Normal":
                            self.properly_terminated = True
                            terminated = True
                    elif error_patt.search(line):
                        error_defs = {
                            "! Non-Optimized Parameters !": "Optimization " "error",
                            "Convergence failure": "SCF convergence error",
                        }
                        m = error_patt.search(line)
                        self.errors.append(error_defs[m.group(1)])
                    elif num_elec_patt.search(line):
                        m = num_elec_patt.search(line)
                        self.electrons = (int(m.group(1)), int(m.group(2)))
                    elif (not self.is_pcm) and pcm_patt.search(line):
                        self.is_pcm = True
                        self.pcm = {}
                    elif "freq" in route_lower and "opt" in route_lower and stat_type_patt.search(line):
                        self.stationary_type = "Saddle"
                    elif mp2_patt.search(line):
                        m = mp2_patt.search(line)
                        self.energies.append(float(m.group(1).replace("D", "E")))
                    elif oniom_patt.search(line):
                        m = oniom_patt.matcher(line)
                        self.energies.append(float(m.group(1)))
                    elif scf_patt.search(line):
                        m = scf_patt.search(line)
                        self.energies.append(float(m.group(1)))
                    elif std_orientation_patt.search(line):
                        standard_orientation = True
                        geom_orientation = "standard"
                        read_coord = True
                    elif input_orientation_patt.search(line):
                        geom_orientation = "input"
                        read_coord = True
                    elif "Optimization completed." in line:
                        line = f.readline()
                        if " -- Stationary point found." not in line:
                            warnings.warn(
                                "\n" + self.filename + ": Optimization complete but this is not a stationary point"
                            )
                        if standard_orientation:
                            opt_structures.append(std_structures[-1])
                        else:
                            opt_structures.append(input_structures[-1])
                    elif not read_eigen and orbital_patt.search(line):
                        eigen_txt.append(line)
                        read_eigen = True
                    elif mulliken_patt.search(line):
                        mulliken_txt = []
                        read_mulliken = True
                    elif not parse_forces and forces_on_patt.search(line):
                        parse_forces = True
                    elif freq_on_patt.search(line):
                        parse_freq = True
                        [f.readline() for i in range(3)]
                    elif mo_coeff_patt.search(line):
                        if "Alpha" in line:
                            self.is_spin = True
                        read_mo = True
                    elif hessian_patt.search(line):
                        parse_hessian = True
                    elif resume_patt.search(line):
                        resume = []
                        while not resume_end_patt.search(line):
                            resume.append(line)
                            line = f.readline()
                            #  security if \\@ not in one line !
                            if line == "\n":
                                break
                        resume.append(line)
                        resume = "".join([r.strip() for r in resume])
                        self.resumes.append(resume)
                    elif bond_order_patt.search(line):
                        parse_bond_order = True

                    if read_mulliken:
                        if not end_mulliken_patt.search(line):
                            mulliken_txt.append(line)
                        else:
                            m = end_mulliken_patt.search(line)
                            mulliken_charges = {}
                            for line in mulliken_txt:
                                if mulliken_charge_patt.search(line):
                                    m = mulliken_charge_patt.search(line)
                                    dic = {int(m.group(1)): [m.group(2), float(m.group(3))]}
                                    mulliken_charges.update(dic)
                            read_mulliken = False
                            self.Mulliken_charges = mulliken_charges

        # store the structures. If symmetry is considered, the standard orientation
        # is used. Else the input orientation is used.
        if standard_orientation:
            self.structures = std_structures
            self.structures_input_orientation = input_structures
        else:
            self.structures = input_structures
            self.structures_input_orientation = input_structures
        # store optimized structure in input orientation
        self.opt_structures = opt_structures

        if not terminated:
            warnings.warn("\n" + self.filename + ": Termination error or bad Gaussian output file !")

    def _check_pcm(self, line):
        energy_patt = re.compile(r"(Dispersion|Cavitation|Repulsion) energy" r"\s+\S+\s+=\s+(\S*)")
        total_patt = re.compile(r"with all non electrostatic terms\s+\S+\s+" r"=\s+(\S*)")
        parameter_patt = re.compile(r"(Eps|Numeral density|RSolv|Eps" r"\(inf[inity]*\))\s+=\s*(\S*)")

        if energy_patt.search(line):
            m = energy_patt.search(line)
            self.pcm["{} energy".format(m.group(1))] = float(m.group(2))
        elif total_patt.search(line):
            m = total_patt.search(line)
            self.pcm["Total energy"] = float(m.group(1))
        elif parameter_patt.search(line):
            m = parameter_patt.search(line)
            self.pcm[m.group(1)] = float(m.group(2))

    def as_dict(self):
        """
        Json-serializable dict representation.
        """
        structure = self.final_structure
        d = {
            "has_gaussian_completed": self.properly_terminated,
            "nsites": len(structure),
        }
        comp = structure.composition
        d["unit_cell_formula"] = comp.as_dict()
        d["reduced_cell_formula"] = Composition(comp.reduced_formula).as_dict()
        d["pretty_formula"] = comp.reduced_formula
        d["is_pcm"] = self.is_pcm
        d["errors"] = self.errors
        d["Mulliken_charges"] = self.Mulliken_charges

        unique_symbols = sorted(list(d["unit_cell_formula"].keys()))
        d["elements"] = unique_symbols
        d["nelements"] = len(unique_symbols)
        d["charge"] = self.charge
        d["spin_multiplicity"] = self.spin_multiplicity

        vin = {
            "route": self.route_parameters,
            "functional": self.functional,
            "basis_set": self.basis_set,
            "nbasisfunctions": self.num_basis_func,
            "pcm_parameters": self.pcm,
        }

        d["input"] = vin

        nsites = len(self.final_structure)

        vout = {
            "energies": self.energies,
            "final_energy": self.final_energy,
            "final_energy_per_atom": self.final_energy / nsites,
            "molecule": structure.as_dict(),
            "stationary_type": self.stationary_type,
            "corrections": self.corrections,
        }

        d["output"] = vout
        d["@module"] = self.__class__.__module__
        d["@class"] = self.__class__.__name__

        return d

    def read_scan(self):
        """
        Read a potential energy surface from a gaussian scan calculation.

        Returns:

            A dict: {"energies": [ values ],
                     "coords": {"d1": [ values ], "A2", [ values ], ... }}

            "energies" are the energies of all points of the potential energy
            surface. "coords" are the internal coordinates used to compute the
            potential energy surface and the internal coordinates optimized,
            labelled by their name as defined in the calculation.
        """

        def floatList(l):
            """return a list of float from a list of string"""
            return [float(v) for v in l]

        scan_patt = re.compile(r"^\sSummary of the potential surface scan:")
        optscan_patt = re.compile(r"^\sSummary of Optimized Potential Surface Scan")
        coord_patt = re.compile(r"^\s*(\w+)((\s*[+-]?\d+\.\d+)+)")

        # data dict return
        data = {"energies": list(), "coords": dict()}

        # read in file
        with zopen(self.filename, "r") as f:
            line = f.readline()

            while line != "":
                if optscan_patt.match(line):
                    f.readline()
                    line = f.readline()
                    endScan = False
                    while not endScan:
                        data["energies"] += floatList(float_patt.findall(line))
                        line = f.readline()
                        while coord_patt.match(line):
                            icname = line.split()[0].strip()
                            if icname in data["coords"]:
                                data["coords"][icname] += floatList(float_patt.findall(line))
                            else:
                                data["coords"][icname] = floatList(float_patt.findall(line))
                            line = f.readline()
                        if not re.search(r"^\s+((\s*\d+)+)", line):
                            endScan = True
                        else:
                            line = f.readline()

                elif scan_patt.match(line):
                    line = f.readline()
                    data["coords"] = {icname: list() for icname in line.split()[1:-1]}
                    f.readline()
                    line = f.readline()
                    while not re.search(r"^\s-+", line):
                        values = floatList(line.split())
                        data["energies"].append(values[-1])
                        for i, icname in enumerate(data["coords"]):
                            data["coords"][icname].append(values[i + 1])
                        line = f.readline()
                else:
                    line = f.readline()

        return data

    def get_scan_plot(self, coords=None):
        """
        Get a matplotlib plot of the potential energy surface.

        Args:
            coords: internal coordinate name to use as abcissa.
        """
        from pymatgen.util.plotting import pretty_plot

        plt = pretty_plot(12, 8)

        d = self.read_scan()

        if coords and coords in d["coords"]:
            x = d["coords"][coords]
            plt.xlabel(coords)
        else:
            x = range(len(d["energies"]))
            plt.xlabel("points")

        plt.ylabel("Energy (eV)")

        e_min = min(d["energies"])
        y = [(e - e_min) * Ha_to_eV for e in d["energies"]]

        plt.plot(x, y, "ro--")
        return plt

    def save_scan_plot(self, filename="scan.pdf", img_format="pdf", coords=None):
        """
        Save matplotlib plot of the potential energy surface to a file.

        Args:
            filename: Filename to write to.
            img_format: Image format to use. Defaults to EPS.
            coords: internal coordinate name to use as abcissa.
        """
        plt = self.get_scan_plot(coords)
        plt.savefig(filename, format=img_format)

    def read_excitation_energies(self):
        """
        Read a excitation energies after a TD-DFT calculation.

        Returns:

            A list: A list of tuple for each transition such as
                    [(energie (eV), lambda (nm), oscillatory strength), ... ]
        """

        transitions = list()

        # read in file
        with zopen(self.filename, "r") as f:
            line = f.readline()
            td = False
            while line != "":
                if re.search(r"^\sExcitation energies and oscillator strengths:", line):
                    td = True

                if td:
                    if re.search(r"^\sExcited State\s*\d", line):
                        val = [float(v) for v in float_patt.findall(line)]
                        transitions.append(tuple(val[0:3]))
                line = f.readline()
        return transitions

    def get_spectre_plot(self, sigma=0.05, step=0.01):
        """
        Get a matplotlib plot of the UV-visible xas. Transition are plotted
        as vertical lines and as a sum of normal functions with sigma with. The
        broadening is applied in energy and the xas is plotted as a function
        of the wavelength.

        Args:
            sigma: Full width at half maximum in eV for normal functions.
            step: bin interval in eV

        Returns:
            A dict: {"energies": values, "lambda": values, "xas": values}
                    where values are lists of abscissa (energies, lamba) and
                    the sum of gaussian functions (xas).
            A matplotlib plot.
        """
        from scipy.stats import norm

        from pymatgen.util.plotting import pretty_plot

        plt = pretty_plot(12, 8)

        transitions = self.read_excitation_energies()

        minval = min([val[0] for val in transitions]) - 5.0 * sigma
        maxval = max([val[0] for val in transitions]) + 5.0 * sigma
        npts = int((maxval - minval) / step) + 1

        eneval = np.linspace(minval, maxval, npts)  # in eV
        lambdaval = [cst.h * cst.c / (val * cst.e) * 1.0e9 for val in eneval]  # in nm

        # sum of gaussian functions
        spectre = np.zeros(npts)
        for trans in transitions:
            spectre += trans[2] * norm(eneval, trans[0], sigma)
        spectre /= spectre.max()
        plt.plot(lambdaval, spectre, "r-", label="spectre")

        data = {"energies": eneval, "lambda": lambdaval, "xas": spectre}

        # plot transitions as vlines
        plt.vlines(
            [val[1] for val in transitions],
            0.0,
            [val[2] for val in transitions],
            color="blue",
            label="transitions",
            linewidth=2,
        )

        plt.xlabel("$\\lambda$ (nm)")
        plt.ylabel("Arbitrary unit")
        plt.legend()

        return data, plt

    def save_spectre_plot(self, filename="spectre.pdf", img_format="pdf", sigma=0.05, step=0.01):
        """
        Save matplotlib plot of the spectre to a file.

        Args:
            filename: Filename to write to.
            img_format: Image format to use. Defaults to EPS.
            sigma: Full width at half maximum in eV for normal functions.
            step: bin interval in eV
        """
        d, plt = self.get_spectre_plot(sigma, step)
        plt.savefig(filename, format=img_format)

    def to_input(
        self,
        mol=None,
        charge=None,
        spin_multiplicity=None,
        title=None,
        functional=None,
        basis_set=None,
        route_parameters=None,
        input_parameters=None,
        link0_parameters=None,
        dieze_tag=None,
        cart_coords=False,
    ):
        """
        Create a new input object using by default the last geometry read in
        the output file and with the same calculation parameters. Arguments
        are the same as GaussianInput class.

        Returns
            gaunip (GaussianInput) : the gaussian input object
        """
        if not mol:
            mol = self.final_structure

        if charge is None:
            charge = self.charge

        if spin_multiplicity is None:
            spin_multiplicity = self.spin_multiplicity

        if not title:
            title = self.title

        if not functional:
            functional = self.functional

        if not basis_set:
            basis_set = self.basis_set

        if not route_parameters:
            route_parameters = self.route_parameters

        if not link0_parameters:
            link0_parameters = self.link0

        if not dieze_tag:
            dieze_tag = self.dieze_tag

        return GaussianInput(
            mol=mol,
            charge=charge,
            spin_multiplicity=spin_multiplicity,
            title=title,
            functional=functional,
            basis_set=basis_set,
            route_parameters=route_parameters,
            input_parameters=input_parameters,
            link0_parameters=link0_parameters,
            dieze_tag=dieze_tag,
        )


#coding:utf8

import numpy as np
import math
import pylab as pl
import matplotlib.cm as cm
import matplotlib.mlab as mlab
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D

import json

class GraphTree:
    def __init__(self):
        self.jsonobj = {}
        self.leafNode = dict(boxstyle = 'round4',fc = '0.8')
        self.branchNode = dict(boxstyle = 'sawtooth',fc = '0.8')
        self.arrow = dict(arrowstyle = '<-')
        self.depth = 0
        self.leafcount = 0

    def get_depth_leafcount(self,root):
        current_node = root.keys()[0] #name of choice node(string)
        branch_dict = root[current_node]
        maxdepth, thisdepth, thisleafcount = 0,0,0
        for current_node in branch_dict.keys():
            # print current_node,type(branch_dict[current_node]).__name__
            if type(branch_dict[current_node]).__name__ == 'dict':
                temp = self.get_depth_leafcount(branch_dict[current_node])
                thisdepth = 1 + temp[0]
                thisleafcount += temp[1]
            else:
                thisdepth = 1
                thisleafcount += 1
            if thisdepth > maxdepth:
                maxdepth = thisdepth
        return maxdepth,thisleafcount

    def load(self,strjson):
        self.jsonobj = dict(strjson)
        self.depth,self.leafcount = self.get_depth_leafcount(self.jsonobj)

    def plotMidText(self, cntrPt, parentPt, txtString):
        xMid = (parentPt[0]  - cntrPt[0]) / 2.0 + cntrPt[0]
        yMid = (parentPt[1]  - cntrPt[1]) / 2.0 + cntrPt[1]
        self.ax1.text(xMid, yMid, txtString)

    def plotNode(self, nodeTxt, cntrPt, parentPt, nodeType):
        self.ax1.annotate(nodeTxt, xy = parentPt, xycoords = 'axes fraction', xytext = cntrPt, \
             textcoords = 'axes fraction', va = 'center', ha = 'center', bbox = nodeType, arrowprops = self.arrow)

    def plotTree(self, myTree, parentPt, nodeTxt):
        depth, leaves = self.get_depth_leafcount(myTree)
        current_node = myTree.keys()[0]
        cntrPt = (self.xOff + (1.0 + leaves) / 2.0 / self.leafcount, self.yOff)
        self.plotMidText(cntrPt, parentPt, nodeTxt)
        self.plotNode(current_node, cntrPt, parentPt, self.branchNode)
        branch_dict = myTree[current_node]
        self.yOff -= 1.0 / self.depth
        for current_node in branch_dict.keys():
            if type(branch_dict[current_node]).__name__ == 'dict':
                self.plotTree(branch_dict[current_node], cntrPt, str(current_node))
            else:
                self.xOff += 1.0 / self.leafcount
                self.plotNode(branch_dict[current_node], (self.xOff, self.yOff), cntrPt, self.leafNode)
                self.plotMidText((self.xOff, self.yOff), cntrPt, str(current_node))
        self.yOff += 1.0 / self.depth

    def createPlot(self, show = True, save = ''):
        fig = plt.figure(1, facecolor = 'white')
        fig.clf()
        axprops = dict(xticks = [], yticks = [])
        self.ax1 = plt.subplot(111,frameon = False, **axprops)
        self.xOff, self.yOff = -0.5 / self.leafcount, 1.0
        self.plotTree(self.jsonobj, (0.5,1.0), '')
        import StringIO, urllib, base64
        if show:
            plt.show()
        else:
            imgdata = StringIO.StringIO()
            fig.savefig(imgdata, format='png')
            imgdata.seek(0)  # rewind the data
            uri = 'data:image/png;base64,' + urllib.quote(base64.b64encode(imgdata.buf))
            imgdata.close()
            return uri

    def showPlot(self):
        plt.show()

if __name__ == '__main__':
    tr = GraphTree()
    # aa = '{"no surfacing":{"0":"no","1":{"flippers":{"0":"no","1":"yes"}}}}'
    # tr.load(json.loads(aa))
    #JSON can't have non-string key
    aa = {"aged":{"0":"no","1":{"male":{"0":"no","1":"yes"}}}}
    # aa = {'water': {0: 1, 1: {'foot': {0: "'no'", 1: "'yes'"}}}}
    print dict(aa)
    # aa = {"no surfacing":{0:"no",1:{"flippers":{0:"no",1:"yes"}}}}
    # print dict(aa)
    tr.load(aa)
    print tr.leafcount,tr.depth
    tr.createPlot(show=True)


#! /usr/bin/env python

import sys
from aubio import pvoc, source
from numpy import zeros, hstack

def get_waveform_plot(filename, samplerate = 0, block_size = 4096, ax = None, downsample = 2**4):
    import matplotlib.pyplot as plt
    if not ax:
        fig = plt.figure()
        ax = fig.add_subplot(111)
    hop_s = block_size

    allsamples_max = zeros(0,)
    downsample = downsample  # to plot n samples / hop_s

    a = source(filename, samplerate, hop_s)            # source file
    if samplerate == 0: samplerate = a.samplerate

    total_frames = 0
    while True:
        samples, read = a()
        # keep some data to plot it later
        new_maxes = (abs(samples.reshape(hop_s/downsample, downsample))).max(axis=0)
        allsamples_max = hstack([allsamples_max, new_maxes])
        total_frames += read
        if read < hop_s: break
    allsamples_max = (allsamples_max > 0) * allsamples_max
    allsamples_max_times = [ ( float (t) / downsample ) * hop_s for t in range(len(allsamples_max)) ]

    ax.plot(allsamples_max_times,  allsamples_max, '-b')
    ax.plot(allsamples_max_times, -allsamples_max, '-b')
    ax.axis(xmin = allsamples_max_times[0], xmax = allsamples_max_times[-1])

    set_xlabels_sample2time(ax, allsamples_max_times[-1], samplerate)
    return ax

def set_xlabels_sample2time(ax, latest_sample, samplerate):
    ax.axis(xmin = 0, xmax = latest_sample)
    if latest_sample / float(samplerate) > 60:
        ax.set_xlabel('time (mm:ss)')
        ax.set_xticklabels([ "%02d:%02d" % (t/float(samplerate)/60, (t/float(samplerate))%60) for t in ax.get_xticks()[:-1]], rotation = 50)
    else:
        ax.set_xlabel('time (ss.mm)')
        ax.set_xticklabels([ "%02d.%02d" % (t/float(samplerate), 100*((t/float(samplerate))%1) ) for t in ax.get_xticks()[:-1]], rotation = 50)

if __name__ == '__main__':
    import matplotlib.pyplot as plt
    if len(sys.argv) < 2:
        print "Usage: %s <filename>" % sys.argv[0]
    else:
        for soundfile in sys.argv[1:]:
            get_waveform_plot(soundfile)
            # display graph
            plt.show()


import pytest

from warnings import catch_warnings
import os
import datetime
import numpy as np
import sys
from distutils.version import LooseVersion

from pandas import compat
from pandas.compat import u, PY3
from pandas import (Series, DataFrame, Panel, MultiIndex, bdate_range,
                    date_range, period_range, Index, Categorical)
from pandas.errors import PerformanceWarning
from pandas.io.packers import to_msgpack, read_msgpack
import pandas.util.testing as tm
from pandas.util.testing import (ensure_clean,
                                 assert_categorical_equal,
                                 assert_frame_equal,
                                 assert_index_equal,
                                 assert_series_equal,
                                 patch)
from pandas.tests.test_panel import assert_panel_equal

import pandas
from pandas import Timestamp, NaT
from pandas._libs.tslib import iNaT

nan = np.nan

try:
    import blosc  # NOQA
except ImportError:
    _BLOSC_INSTALLED = False
else:
    _BLOSC_INSTALLED = True

try:
    import zlib  # NOQA
except ImportError:
    _ZLIB_INSTALLED = False
else:
    _ZLIB_INSTALLED = True

@pytest.fixture(scope='module')
def current_packers_data():
    # our current version packers data
    from pandas.tests.io.generate_legacy_storage_files import (
        create_msgpack_data)
    return create_msgpack_data()

@pytest.fixture(scope='module')
def all_packers_data():
    # our all of our current version packers data
    from pandas.tests.io.generate_legacy_storage_files import (
        create_data)
    return create_data()

def check_arbitrary(a, b):

    if isinstance(a, (list, tuple)) and isinstance(b, (list, tuple)):
        assert(len(a) == len(b))
        for a_, b_ in zip(a, b):
            check_arbitrary(a_, b_)
    elif isinstance(a, Panel):
        assert_panel_equal(a, b)
    elif isinstance(a, DataFrame):
        assert_frame_equal(a, b)
    elif isinstance(a, Series):
        assert_series_equal(a, b)
    elif isinstance(a, Index):
        assert_index_equal(a, b)
    elif isinstance(a, Categorical):
        # Temp,
        # Categorical.categories is changed from str to bytes in PY3
        # maybe the same as GH 13591
        if PY3 and b.categories.inferred_type == 'string':
            pass
        else:
            tm.assert_categorical_equal(a, b)
    elif a is NaT:
        assert b is NaT
    elif isinstance(a, Timestamp):
        assert a == b
        assert a.freq == b.freq
    else:
        assert(a == b)

class TestPackers(object):

    def setup_method(self, method):
        self.path = '__%s__.msg' % tm.rands(10)

    def teardown_method(self, method):
        pass

    def encode_decode(self, x, compress=None, **kwargs):
        with ensure_clean(self.path) as p:
            to_msgpack(p, x, compress=compress, **kwargs)
            return read_msgpack(p, **kwargs)

class TestAPI(TestPackers):

    def test_string_io(self):

        df = DataFrame(np.random.randn(10, 2))
        s = df.to_msgpack(None)
        result = read_msgpack(s)
        tm.assert_frame_equal(result, df)

        s = df.to_msgpack()
        result = read_msgpack(s)
        tm.assert_frame_equal(result, df)

        s = df.to_msgpack()
        result = read_msgpack(compat.BytesIO(s))
        tm.assert_frame_equal(result, df)

        s = to_msgpack(None, df)
        result = read_msgpack(s)
        tm.assert_frame_equal(result, df)

        with ensure_clean(self.path) as p:

            s = df.to_msgpack()
            fh = open(p, 'wb')
            fh.write(s)
            fh.close()
            result = read_msgpack(p)
            tm.assert_frame_equal(result, df)

    def test_path_pathlib(self):
        df = tm.makeDataFrame()
        result = tm.round_trip_pathlib(df.to_msgpack, read_msgpack)
        tm.assert_frame_equal(df, result)

    def test_path_localpath(self):
        df = tm.makeDataFrame()
        result = tm.round_trip_localpath(df.to_msgpack, read_msgpack)
        tm.assert_frame_equal(df, result)

    def test_iterator_with_string_io(self):

        dfs = [DataFrame(np.random.randn(10, 2)) for i in range(5)]
        s = to_msgpack(None, *dfs)
        for i, result in enumerate(read_msgpack(s, iterator=True)):
            tm.assert_frame_equal(result, dfs[i])

    def test_invalid_arg(self):
        # GH10369
        class A(object):

            def __init__(self):
                self.read = 0

        pytest.raises(ValueError, read_msgpack, path_or_buf=None)
        pytest.raises(ValueError, read_msgpack, path_or_buf={})
        pytest.raises(ValueError, read_msgpack, path_or_buf=A())

class TestNumpy(TestPackers):

    def test_numpy_scalar_float(self):
        x = np.float32(np.random.rand())
        x_rec = self.encode_decode(x)
        tm.assert_almost_equal(x, x_rec)

    def test_numpy_scalar_complex(self):
        x = np.complex64(np.random.rand() + 1j * np.random.rand())
        x_rec = self.encode_decode(x)
        assert np.allclose(x, x_rec)

    def test_scalar_float(self):
        x = np.random.rand()
        x_rec = self.encode_decode(x)
        tm.assert_almost_equal(x, x_rec)

    def test_scalar_complex(self):
        x = np.random.rand() + 1j * np.random.rand()
        x_rec = self.encode_decode(x)
        assert np.allclose(x, x_rec)

    def test_list_numpy_float(self):
        x = [np.float32(np.random.rand()) for i in range(5)]
        x_rec = self.encode_decode(x)
        # current msgpack cannot distinguish list/tuple
        tm.assert_almost_equal(tuple(x), x_rec)

        x_rec = self.encode_decode(tuple(x))
        tm.assert_almost_equal(tuple(x), x_rec)

    def test_list_numpy_float_complex(self):
        if not hasattr(np, 'complex128'):
            pytest.skip('numpy cant handle complex128')

        x = [np.float32(np.random.rand()) for i in range(5)] + \
            [np.complex128(np.random.rand() + 1j * np.random.rand())
             for i in range(5)]
        x_rec = self.encode_decode(x)
        assert np.allclose(x, x_rec)

    def test_list_float(self):
        x = [np.random.rand() for i in range(5)]
        x_rec = self.encode_decode(x)
        # current msgpack cannot distinguish list/tuple
        tm.assert_almost_equal(tuple(x), x_rec)

        x_rec = self.encode_decode(tuple(x))
        tm.assert_almost_equal(tuple(x), x_rec)

    def test_list_float_complex(self):
        x = [np.random.rand() for i in range(5)] + \
            [(np.random.rand() + 1j * np.random.rand()) for i in range(5)]
        x_rec = self.encode_decode(x)
        assert np.allclose(x, x_rec)

    def test_dict_float(self):
        x = {'foo': 1.0, 'bar': 2.0}
        x_rec = self.encode_decode(x)
        tm.assert_almost_equal(x, x_rec)

    def test_dict_complex(self):
        x = {'foo': 1.0 + 1.0j, 'bar': 2.0 + 2.0j}
        x_rec = self.encode_decode(x)
        tm.assert_dict_equal(x, x_rec)

        for key in x:
            tm.assert_class_equal(x[key], x_rec[key], obj="complex value")

    def test_dict_numpy_float(self):
        x = {'foo': np.float32(1.0), 'bar': np.float32(2.0)}
        x_rec = self.encode_decode(x)
        tm.assert_almost_equal(x, x_rec)

    def test_dict_numpy_complex(self):
        x = {'foo': np.complex128(1.0 + 1.0j),
             'bar': np.complex128(2.0 + 2.0j)}
        x_rec = self.encode_decode(x)
        tm.assert_dict_equal(x, x_rec)

        for key in x:
            tm.assert_class_equal(x[key], x_rec[key], obj="numpy complex128")

    def test_numpy_array_float(self):

        # run multiple times
        for n in range(10):
            x = np.random.rand(10)
            for dtype in ['float32', 'float64']:
                x = x.astype(dtype)
                x_rec = self.encode_decode(x)
                tm.assert_almost_equal(x, x_rec)

    def test_numpy_array_complex(self):
        x = (np.random.rand(5) + 1j * np.random.rand(5)).astype(np.complex128)
        x_rec = self.encode_decode(x)
        assert (all(map(lambda x, y: x == y, x, x_rec)) and
                x.dtype == x_rec.dtype)

    def test_list_mixed(self):
        x = [1.0, np.float32(3.5), np.complex128(4.25), u('foo')]
        x_rec = self.encode_decode(x)
        # current msgpack cannot distinguish list/tuple
        tm.assert_almost_equal(tuple(x), x_rec)

        x_rec = self.encode_decode(tuple(x))
        tm.assert_almost_equal(tuple(x), x_rec)

class TestBasic(TestPackers):

    def test_timestamp(self):

        for i in [Timestamp(
            '20130101'), Timestamp('20130101', tz='US/Eastern'),
                Timestamp('201301010501')]:
            i_rec = self.encode_decode(i)
            assert i == i_rec

    def test_nat(self):
        nat_rec = self.encode_decode(NaT)
        assert NaT is nat_rec

    def test_datetimes(self):

        # fails under 2.6/win32 (np.datetime64 seems broken)

        if LooseVersion(sys.version) < '2.7':
            pytest.skip('2.6 with np.datetime64 is broken')

        for i in [datetime.datetime(2013, 1, 1),
                  datetime.datetime(2013, 1, 1, 5, 1),
                  datetime.date(2013, 1, 1),
                  np.datetime64(datetime.datetime(2013, 1, 5, 2, 15))]:
            i_rec = self.encode_decode(i)
            assert i == i_rec

    def test_timedeltas(self):

        for i in [datetime.timedelta(days=1),
                  datetime.timedelta(days=1, seconds=10),
                  np.timedelta64(1000000)]:
            i_rec = self.encode_decode(i)
            assert i == i_rec

class TestIndex(TestPackers):

    def setup_method(self, method):
        super(TestIndex, self).setup_method(method)

        self.d = {
            'string': tm.makeStringIndex(100),
            'date': tm.makeDateIndex(100),
            'int': tm.makeIntIndex(100),
            'rng': tm.makeRangeIndex(100),
            'float': tm.makeFloatIndex(100),
            'empty': Index([]),
            'tuple': Index(zip(['foo', 'bar', 'baz'], [1, 2, 3])),
            'period': Index(period_range('2012-1-1', freq='M', periods=3)),
            'date2': Index(date_range('2013-01-1', periods=10)),
            'bdate': Index(bdate_range('2013-01-02', periods=10)),
            'cat': tm.makeCategoricalIndex(100)
        }

        self.mi = {
            'reg': MultiIndex.from_tuples([('bar', 'one'), ('baz', 'two'),
                                           ('foo', 'two'),
                                           ('qux', 'one'), ('qux', 'two')],
                                          names=['first', 'second']),
        }

    def test_basic_index(self):

        for s, i in self.d.items():
            i_rec = self.encode_decode(i)
            tm.assert_index_equal(i, i_rec)

        # datetime with no freq (GH5506)
        i = Index([Timestamp('20130101'), Timestamp('20130103')])
        i_rec = self.encode_decode(i)
        tm.assert_index_equal(i, i_rec)

        # datetime with timezone
        i = Index([Timestamp('20130101 9:00:00'), Timestamp(
            '20130103 11:00:00')]).tz_localize('US/Eastern')
        i_rec = self.encode_decode(i)
        tm.assert_index_equal(i, i_rec)

    def test_multi_index(self):

        for s, i in self.mi.items():
            i_rec = self.encode_decode(i)
            tm.assert_index_equal(i, i_rec)

    def test_unicode(self):
        i = tm.makeUnicodeIndex(100)

        i_rec = self.encode_decode(i)
        tm.assert_index_equal(i, i_rec)

    def categorical_index(self):
        # GH15487
        df = DataFrame(np.random.randn(10, 2))
        df = df.astype({0: 'category'}).set_index(0)
        result = self.encode_decode(df)
        tm.assert_frame_equal(result, df)

class TestSeries(TestPackers):

    def setup_method(self, method):
        super(TestSeries, self).setup_method(method)

        self.d = {}

        s = tm.makeStringSeries()
        s.name = 'string'
        self.d['string'] = s

        s = tm.makeObjectSeries()
        s.name = 'object'
        self.d['object'] = s

        s = Series(iNaT, dtype='M8[ns]', index=range(5))
        self.d['date'] = s

        data = {
            'A': [0., 1., 2., 3., np.nan],
            'B': [0, 1, 0, 1, 0],
            'C': ['foo1', 'foo2', 'foo3', 'foo4', 'foo5'],
            'D': date_range('1/1/2009', periods=5),
            'E': [0., 1, Timestamp('20100101'), 'foo', 2.],
            'F': [Timestamp('20130102', tz='US/Eastern')] * 2 +
                 [Timestamp('20130603', tz='CET')] * 3,
            'G': [Timestamp('20130102', tz='US/Eastern')] * 5,
            'H': Categorical([1, 2, 3, 4, 5]),
            'I': Categorical([1, 2, 3, 4, 5], ordered=True),
        }

        self.d['float'] = Series(data['A'])
        self.d['int'] = Series(data['B'])
        self.d['mixed'] = Series(data['E'])
        self.d['dt_tz_mixed'] = Series(data['F'])
        self.d['dt_tz'] = Series(data['G'])
        self.d['cat_ordered'] = Series(data['H'])
        self.d['cat_unordered'] = Series(data['I'])

    def test_basic(self):

        # run multiple times here
        for n in range(10):
            for s, i in self.d.items():
                i_rec = self.encode_decode(i)
                assert_series_equal(i, i_rec)

class TestCategorical(TestPackers):

    def setup_method(self, method):
        super(TestCategorical, self).setup_method(method)

        self.d = {}

        self.d['plain_str'] = Categorical(['a', 'b', 'c', 'd', 'e'])
        self.d['plain_str_ordered'] = Categorical(['a', 'b', 'c', 'd', 'e'],
                                                  ordered=True)

        self.d['plain_int'] = Categorical([5, 6, 7, 8])
        self.d['plain_int_ordered'] = Categorical([5, 6, 7, 8], ordered=True)

    def test_basic(self):

        # run multiple times here
        for n in range(10):
            for s, i in self.d.items():
                i_rec = self.encode_decode(i)
                assert_categorical_equal(i, i_rec)

class TestNDFrame(TestPackers):

    def setup_method(self, method):
        super(TestNDFrame, self).setup_method(method)

        data = {
            'A': [0., 1., 2., 3., np.nan],
            'B': [0, 1, 0, 1, 0],
            'C': ['foo1', 'foo2', 'foo3', 'foo4', 'foo5'],
            'D': date_range('1/1/2009', periods=5),
            'E': [0., 1, Timestamp('20100101'), 'foo', 2.],
            'F': [Timestamp('20130102', tz='US/Eastern')] * 5,
            'G': [Timestamp('20130603', tz='CET')] * 5,
            'H': Categorical(['a', 'b', 'c', 'd', 'e']),
            'I': Categorical(['a', 'b', 'c', 'd', 'e'], ordered=True),
        }

        self.frame = {
            'float': DataFrame(dict(A=data['A'], B=Series(data['A']) + 1)),
            'int': DataFrame(dict(A=data['B'], B=Series(data['B']) + 1)),
            'mixed': DataFrame(data)}

        with catch_warnings(record=True):
            self.panel = {
                'float': Panel(dict(ItemA=self.frame['float'],
                                    ItemB=self.frame['float'] + 1))}

    def test_basic_frame(self):

        for s, i in self.frame.items():
            i_rec = self.encode_decode(i)
            assert_frame_equal(i, i_rec)

    def test_basic_panel(self):

        with catch_warnings(record=True):
            for s, i in self.panel.items():
                i_rec = self.encode_decode(i)
                assert_panel_equal(i, i_rec)

    def test_multi(self):

        i_rec = self.encode_decode(self.frame)
        for k in self.frame.keys():
            assert_frame_equal(self.frame[k], i_rec[k])

        l = tuple([self.frame['float'], self.frame['float'].A,
                   self.frame['float'].B, None])
        l_rec = self.encode_decode(l)
        check_arbitrary(l, l_rec)

        # this is an oddity in that packed lists will be returned as tuples
        l = [self.frame['float'], self.frame['float']
             .A, self.frame['float'].B, None]
        l_rec = self.encode_decode(l)
        assert isinstance(l_rec, tuple)
        check_arbitrary(l, l_rec)

    def test_iterator(self):

        l = [self.frame['float'], self.frame['float']
             .A, self.frame['float'].B, None]

        with ensure_clean(self.path) as path:
            to_msgpack(path, *l)
            for i, packed in enumerate(read_msgpack(path, iterator=True)):
                check_arbitrary(packed, l[i])

    def tests_datetimeindex_freq_issue(self):

        # GH 5947
        # inferring freq on the datetimeindex
        df = DataFrame([1, 2, 3], index=date_range('1/1/2013', '1/3/2013'))
        result = self.encode_decode(df)
        assert_frame_equal(result, df)

        df = DataFrame([1, 2], index=date_range('1/1/2013', '1/2/2013'))
        result = self.encode_decode(df)
        assert_frame_equal(result, df)

    def test_dataframe_duplicate_column_names(self):

        # GH 9618
        expected_1 = DataFrame(columns=['a', 'a'])
        expected_2 = DataFrame(columns=[1] * 100)
        expected_2.loc[0] = np.random.randn(100)
        expected_3 = DataFrame(columns=[1, 1])
        expected_3.loc[0] = ['abc', np.nan]

        result_1 = self.encode_decode(expected_1)
        result_2 = self.encode_decode(expected_2)
        result_3 = self.encode_decode(expected_3)

        assert_frame_equal(result_1, expected_1)
        assert_frame_equal(result_2, expected_2)
        assert_frame_equal(result_3, expected_3)

class TestSparse(TestPackers):

    def _check_roundtrip(self, obj, comparator, **kwargs):

        # currently these are not implemetned
        # i_rec = self.encode_decode(obj)
        # comparator(obj, i_rec, **kwargs)
        pytest.raises(NotImplementedError, self.encode_decode, obj)

    def test_sparse_series(self):

        s = tm.makeStringSeries()
        s[3:5] = np.nan
        ss = s.to_sparse()
        self._check_roundtrip(ss, tm.assert_series_equal,
                              check_series_type=True)

        ss2 = s.to_sparse(kind='integer')
        self._check_roundtrip(ss2, tm.assert_series_equal,
                              check_series_type=True)

        ss3 = s.to_sparse(fill_value=0)
        self._check_roundtrip(ss3, tm.assert_series_equal,
                              check_series_type=True)

    def test_sparse_frame(self):

        s = tm.makeDataFrame()
        s.loc[3:5, 1:3] = np.nan
        s.loc[8:10, -2] = np.nan
        ss = s.to_sparse()

        self._check_roundtrip(ss, tm.assert_frame_equal,
                              check_frame_type=True)

        ss2 = s.to_sparse(kind='integer')
        self._check_roundtrip(ss2, tm.assert_frame_equal,
                              check_frame_type=True)

        ss3 = s.to_sparse(fill_value=0)
        self._check_roundtrip(ss3, tm.assert_frame_equal,
                              check_frame_type=True)

class TestCompression(TestPackers):
    """See https://github.com/pandas-dev/pandas/pull/9783
    """

    def setup_method(self, method):
        try:
            from sqlalchemy import create_engine
            self._create_sql_engine = create_engine
        except ImportError:
            self._SQLALCHEMY_INSTALLED = False
        else:
            self._SQLALCHEMY_INSTALLED = True

        super(TestCompression, self).setup_method(method)
        data = {
            'A': np.arange(1000, dtype=np.float64),
            'B': np.arange(1000, dtype=np.int32),
            'C': list(100 * 'abcdefghij'),
            'D': date_range(datetime.datetime(2015, 4, 1), periods=1000),
            'E': [datetime.timedelta(days=x) for x in range(1000)],
        }
        self.frame = {
            'float': DataFrame(dict((k, data[k]) for k in ['A', 'A'])),
            'int': DataFrame(dict((k, data[k]) for k in ['B', 'B'])),
            'mixed': DataFrame(data),
        }

    def test_plain(self):
        i_rec = self.encode_decode(self.frame)
        for k in self.frame.keys():
            assert_frame_equal(self.frame[k], i_rec[k])

    def _test_compression(self, compress):
        i_rec = self.encode_decode(self.frame, compress=compress)
        for k in self.frame.keys():
            value = i_rec[k]
            expected = self.frame[k]
            assert_frame_equal(value, expected)
            # make sure that we can write to the new frames
            for block in value._data.blocks:
                assert block.values.flags.writeable

    def test_compression_zlib(self):
        if not _ZLIB_INSTALLED:
            pytest.skip('no zlib')
        self._test_compression('zlib')

    def test_compression_blosc(self):
        if not _BLOSC_INSTALLED:
            pytest.skip('no blosc')
        self._test_compression('blosc')

    def _test_compression_warns_when_decompress_caches(self, compress):
        not_garbage = []
        control = []  # copied data

        compress_module = globals()[compress]
        real_decompress = compress_module.decompress

        def decompress(ob):
            """mock decompress function that delegates to the real
            decompress but caches the result and a copy of the result.
            """
            res = real_decompress(ob)
            not_garbage.append(res)  # hold a reference to this bytes object
            control.append(bytearray(res))  # copy the data here to check later
            return res

        # types mapped to values to add in place.
        rhs = {
            np.dtype('float64'): 1.0,
            np.dtype('int32'): 1,
            np.dtype('object'): 'a',
            np.dtype('datetime64[ns]'): np.timedelta64(1, 'ns'),
            np.dtype('timedelta64[ns]'): np.timedelta64(1, 'ns'),
        }

        with patch(compress_module, 'decompress', decompress), \
                tm.assert_produces_warning(PerformanceWarning) as ws:

            i_rec = self.encode_decode(self.frame, compress=compress)
            for k in self.frame.keys():

                value = i_rec[k]
                expected = self.frame[k]
                assert_frame_equal(value, expected)
                # make sure that we can write to the new frames even though
                # we needed to copy the data
                for block in value._data.blocks:
                    assert block.values.flags.writeable
                    # mutate the data in some way
                    block.values[0] += rhs[block.dtype]

        for w in ws:
            # check the messages from our warnings
            assert str(w.message) == ('copying data after decompressing; '
                                      'this may mean that decompress is '
                                      'caching its result')

        for buf, control_buf in zip(not_garbage, control):
            # make sure none of our mutations above affected the
            # original buffers
            assert buf == control_buf

    def test_compression_warns_when_decompress_caches_zlib(self):
        if not _ZLIB_INSTALLED:
            pytest.skip('no zlib')
        self._test_compression_warns_when_decompress_caches('zlib')

    def test_compression_warns_when_decompress_caches_blosc(self):
        if not _BLOSC_INSTALLED:
            pytest.skip('no blosc')
        self._test_compression_warns_when_decompress_caches('blosc')

    def _test_small_strings_no_warn(self, compress):
        empty = np.array([], dtype='uint8')
        with tm.assert_produces_warning(None):
            empty_unpacked = self.encode_decode(empty, compress=compress)

        tm.assert_numpy_array_equal(empty_unpacked, empty)
        assert empty_unpacked.flags.writeable

        char = np.array([ord(b'a')], dtype='uint8')
        with tm.assert_produces_warning(None):
            char_unpacked = self.encode_decode(char, compress=compress)

        tm.assert_numpy_array_equal(char_unpacked, char)
        assert char_unpacked.flags.writeable
        # if this test fails I am sorry because the interpreter is now in a
        # bad state where b'a' points to 98 == ord(b'b').
        char_unpacked[0] = ord(b'b')

        # we compare the ord of bytes b'a' with unicode u'a' because the should
        # always be the same (unless we were able to mutate the shared
        # character singleton in which case ord(b'a') == ord(b'b').
        assert ord(b'a') == ord(u'a')
        tm.assert_numpy_array_equal(
            char_unpacked,
            np.array([ord(b'b')], dtype='uint8'),
        )

    def test_small_strings_no_warn_zlib(self):
        if not _ZLIB_INSTALLED:
            pytest.skip('no zlib')
        self._test_small_strings_no_warn('zlib')

    def test_small_strings_no_warn_blosc(self):
        if not _BLOSC_INSTALLED:
            pytest.skip('no blosc')
        self._test_small_strings_no_warn('blosc')

    def test_readonly_axis_blosc(self):
        # GH11880
        if not _BLOSC_INSTALLED:
            pytest.skip('no blosc')
        df1 = DataFrame({'A': list('abcd')})
        df2 = DataFrame(df1, index=[1., 2., 3., 4.])
        assert 1 in self.encode_decode(df1['A'], compress='blosc')
        assert 1. in self.encode_decode(df2['A'], compress='blosc')

    def test_readonly_axis_zlib(self):
        # GH11880
        df1 = DataFrame({'A': list('abcd')})
        df2 = DataFrame(df1, index=[1., 2., 3., 4.])
        assert 1 in self.encode_decode(df1['A'], compress='zlib')
        assert 1. in self.encode_decode(df2['A'], compress='zlib')

    def test_readonly_axis_blosc_to_sql(self):
        # GH11880
        if not _BLOSC_INSTALLED:
            pytest.skip('no blosc')
        if not self._SQLALCHEMY_INSTALLED:
            pytest.skip('no sqlalchemy')
        expected = DataFrame({'A': list('abcd')})
        df = self.encode_decode(expected, compress='blosc')
        eng = self._create_sql_engine("sqlite:///:memory:")
        df.to_sql('test', eng, if_exists='append')
        result = pandas.read_sql_table('test', eng, index_col='index')
        result.index.names = [None]
        assert_frame_equal(expected, result)

    def test_readonly_axis_zlib_to_sql(self):
        # GH11880
        if not _ZLIB_INSTALLED:
            pytest.skip('no zlib')
        if not self._SQLALCHEMY_INSTALLED:
            pytest.skip('no sqlalchemy')
        expected = DataFrame({'A': list('abcd')})
        df = self.encode_decode(expected, compress='zlib')
        eng = self._create_sql_engine("sqlite:///:memory:")
        df.to_sql('test', eng, if_exists='append')
        result = pandas.read_sql_table('test', eng, index_col='index')
        result.index.names = [None]
        assert_frame_equal(expected, result)

class TestEncoding(TestPackers):

    def setup_method(self, method):
        super(TestEncoding, self).setup_method(method)
        data = {
            'A': [compat.u('\u2019')] * 1000,
            'B': np.arange(1000, dtype=np.int32),
            'C': list(100 * 'abcdefghij'),
            'D': date_range(datetime.datetime(2015, 4, 1), periods=1000),
            'E': [datetime.timedelta(days=x) for x in range(1000)],
            'G': [400] * 1000
        }
        self.frame = {
            'float': DataFrame(dict((k, data[k]) for k in ['A', 'A'])),
            'int': DataFrame(dict((k, data[k]) for k in ['B', 'B'])),
            'mixed': DataFrame(data),
        }
        self.utf_encodings = ['utf8', 'utf16', 'utf32']

    def test_utf(self):
        # GH10581
        for encoding in self.utf_encodings:
            for frame in compat.itervalues(self.frame):
                result = self.encode_decode(frame, encoding=encoding)
                assert_frame_equal(result, frame)

    def test_default_encoding(self):
        for frame in compat.itervalues(self.frame):
            result = frame.to_msgpack()
            expected = frame.to_msgpack(encoding='utf8')
            assert result == expected
            result = self.encode_decode(frame)
            assert_frame_equal(result, frame)

def legacy_packers_versions():
    # yield the packers versions
    path = tm.get_data_path('legacy_msgpack')
    for v in os.listdir(path):
        p = os.path.join(path, v)
        if os.path.isdir(p):
            yield v

class TestMsgpack(object):
    """
    How to add msgpack tests:

    1. Install pandas version intended to output the msgpack.
TestPackers
    2. Execute "generate_legacy_storage_files.py" to create the msgpack.
    $ python generate_legacy_storage_files.py <output_dir> msgpack

    3. Move the created pickle to "data/legacy_msgpack/<version>" directory.
    """

    minimum_structure = {'series': ['float', 'int', 'mixed',
                                    'ts', 'mi', 'dup'],
                         'frame': ['float', 'int', 'mixed', 'mi'],
                         'panel': ['float'],
                         'index': ['int', 'date', 'period'],
                         'mi': ['reg2']}

    def check_min_structure(self, data, version):
        for typ, v in self.minimum_structure.items():
            assert typ in data, '"{0}" not found in unpacked data'.format(typ)
            for kind in v:
                msg = '"{0}" not found in data["{1}"]'.format(kind, typ)
                assert kind in data[typ], msg

    def compare(self, current_data, all_data, vf, version):
        # GH12277 encoding default used to be latin-1, now utf-8
        if LooseVersion(version) < '0.18.0':
            data = read_msgpack(vf, encoding='latin-1')
        else:
            data = read_msgpack(vf)
        self.check_min_structure(data, version)
        for typ, dv in data.items():
            assert typ in all_data, ('unpacked data contains '
                                     'extra key "{0}"'
                                     .format(typ))
            for dt, result in dv.items():
                assert dt in current_data[typ], ('data["{0}"] contains extra '
                                                 'key "{1}"'.format(typ, dt))
                try:
                    expected = current_data[typ][dt]
                except KeyError:
                    continue

                # use a specific comparator
                # if available
                comp_method = "compare_{typ}_{dt}".format(typ=typ, dt=dt)
                comparator = getattr(self, comp_method, None)
                if comparator is not None:
                    comparator(result, expected, typ, version)
                else:
                    check_arbitrary(result, expected)

        return data

    def compare_series_dt_tz(self, result, expected, typ, version):
        # 8260
        # dtype is object < 0.17.0
        if LooseVersion(version) < '0.17.0':
            expected = expected.astype(object)
            tm.assert_series_equal(result, expected)
        else:
            tm.assert_series_equal(result, expected)

    def compare_frame_dt_mixed_tzs(self, result, expected, typ, version):
        # 8260
        # dtype is object < 0.17.0
        if LooseVersion(version) < '0.17.0':
            expected = expected.astype(object)
            tm.assert_frame_equal(result, expected)
        else:
            tm.assert_frame_equal(result, expected)

    @pytest.mark.parametrize('version', legacy_packers_versions())
    def test_msgpacks_legacy(self, current_packers_data, all_packers_data,
                             version):

        pth = tm.get_data_path('legacy_msgpack/{0}'.format(version))
        n = 0
        for f in os.listdir(pth):
            # GH12142 0.17 files packed in P2 can't be read in P3
            if (compat.PY3 and version.startswith('0.17.') and
                    f.split('.')[-4][-1] == '2'):
                continue
            vf = os.path.join(pth, f)
            try:
                with catch_warnings(record=True):
                    self.compare(current_packers_data, all_packers_data,
                                 vf, version)
            except ImportError:
                # blosc not installed
                continue
            n += 1
        assert n > 0, 'Msgpack files are not tested'