import sys
from IPython.core.interactiveshell import InteractiveShell
import pandasjson as json
import StringIO

if __name__=="__main__":
    mode = "ipython"
    line = sys.stdin.readline()
    shell = InteractiveShell()
    while line:
        # explicitly write to stdout
        sys.stdout.write(line)
        sys.stdout.flush()
        # handle incoming data, parse it, and redirect
        # stdout so it doesn't interfere
        line = sys.stdin.readline()
        data = json.loads(line)
        codeOut = StringIO.StringIO()
        sys.stdout = codeOut
        try:
            code = data["code"]
            if data.get("autocomplete")==True:
                _, completions = shell.complete(code)
                print json.dumps(completions)
            elif code.startswith("print"):
                #exec(code)
                shell.ex(code)
            else:
                try:
                    #print repr(eval(code))
                    print repr(shell.ev(code))
                except:
                    #exec(code)
                    shell.ex(code)
        except Exception, e:
            pass

        sys.stdout = sys.__stdout__
        data["result"] = codeOut.getvalue()
        sys.stdout.write(json.dumps(data) + "\n")
        sys.stdout.flush()


import sys, os, json, time
import pandas as pd

BOROCODE = {'61' : '1', '05' : '2', '47': '3', '81' : '4', '85': '5'}

if (len(sys.argv) < 2):
    print ("Usage: cfcensus.py census.csv districts.json")
    exit()

censusfile = sys.argv[1]
councilfile = sys.argv[2]
TRACTCOL = 'BoroCT' # rename this for 2000 census

def boroCT (id2):
    boro = BOROCODE[str(id2)[3:5]]
    tract = str(id2)[5:]
    return boro + tract

for (f) in ([censusfile, councilfile]):
    if (not os.path.isfile(f)):
        print ("File " + f + " is not readable")
        exit()

try:
    vehDf = pd.read_csv(
         censusfile,
         skiprows=[1]
            )
except Exception as e:
    print ("Unable to read census file " + censusfile + ": {0}".format(e))
    exit()

try:
    with open(councilfile) as councilfo:
        councilData = json.load(councilfo)

except Exception as e:
    print ("Unable to read council file " + councilfile+": {0}".format(e))
    exit()

vehDf['pctNoVeh'] = vehDf['HD01_VD03'].astype('int') / vehDf['HD01_VD01'].astype('int')
vehDf[TRACTCOL] = vehDf['GEO.id2'].apply(boroCT)

vehDf2 = pd.DataFrame(vehDf[[TRACTCOL, 'HD01_VD01', 'HD01_VD03', 'pctNoVeh']]).set_index(TRACTCOL)

f = 0
total = {}
noVeh = {}
councilDistricts = set()
for (t, c) in councilData.items():
    for (d) in c:
        councilDistricts.add(d)
        try:
            total[d] = total.get(d, 0) + c[d] * vehDf2.loc[str(t)]['HD01_VD01']
            noVeh[d] = noVeh.get(d, 0) + c[d] * vehDf2.loc[str(t)]['HD01_VD03']
        except KeyError as e:
            print("No entry for census tract " + str(t))

for (d) in sorted(councilDistricts, key=int):
    print (','.join([
                d,
                str(int(total[d])),
                str(int(noVeh[d])),
                str(round((noVeh[d] / total[d]), 3))
                ]))


#!/usr/bin/python
# Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Do all the steps required to build and test against nacl."""

import optparse
import os.path
import re
import shutil
import subprocess
import sys

import find_chrome

THIS_DIR = os.path.abspath(os.path.dirname(__file__))
CHROMIUM_DIR = os.path.abspath(os.path.join(THIS_DIR, '..', '..', '..'))
sys.path.append(os.path.join(CHROMIUM_DIR, 'build'))
import detect_host_arch

# Copied from buildbot/buildbot_lib.py
def TryToCleanContents(path, file_name_filter=lambda fn: True):
  """
  Remove the contents of a directory without touching the directory itself.
  Ignores all failures.
  """
  if os.path.exists(path):
    for fn in os.listdir(path):
      TryToCleanPath(os.path.join(path, fn), file_name_filter)

# Copied from buildbot/buildbot_lib.py
def TryToCleanPath(path, file_name_filter=lambda fn: True):
  """
  Removes a file or directory.
  Ignores all failures.
  """
  if os.path.exists(path):
    if file_name_filter(path):
      print 'Trying to remove %s' % path
      if os.path.isdir(path):
        shutil.rmtree(path, ignore_errors=True)
      else:
        try:
          os.remove(path)
        except Exception:
          pass
    else:
      print 'Skipping %s' % path

# TODO(ncbray): this is somewhat unsafe.  We should fix the underlying problem.
def CleanTempDir():
  # Only delete files and directories like:
  # a) C:\temp\83C4.tmp
  # b) /tmp/.org.chromium.Chromium.EQrEzl
  file_name_re = re.compile(
      r'[\\/]([0-9a-fA-F]+\.tmp|\.org\.chrom\w+\.Chrom\w+\..+)$')
  file_name_filter = lambda fn: file_name_re.search(fn) is not None

  path = os.environ.get('TMP', os.environ.get('TEMP', '/tmp'))
  if len(path) >= 4 and os.path.isdir(path):
    print
    print "Cleaning out the temp directory."
    print
    TryToCleanContents(path, file_name_filter)
  else:
    print
    print "Cannot find temp directory, not cleaning it."
    print

def RunCommand(cmd, cwd, env):
  sys.stdout.write('\nRunning %s\n\n' % ' '.join(cmd))
  sys.stdout.flush()
  retcode = subprocess.call(cmd, cwd=cwd, env=env)
  if retcode != 0:
    sys.stdout.write('\nFailed: %s\n\n' % ' '.join(cmd))
    sys.exit(retcode)

def RunTests(name, cmd, nacl_dir, env):
  sys.stdout.write('\n\nBuilding files needed for %s testing...\n\n' % name)
  RunCommand(cmd + ['do_not_run_tests=1', '-j8'], nacl_dir, env)
  sys.stdout.write('\n\nRunning %s tests...\n\n' % name)
  RunCommand(cmd, nacl_dir, env)

def BuildAndTest(options):
  # Refuse to run under cygwin.
  if sys.platform == 'cygwin':
    raise Exception('I do not work under cygwin, sorry.')

  # By default, use the version of Python is being used to run this script.
  python = sys.executable
  if sys.platform == 'darwin':
    # Mac 10.5 bots tend to use a particularlly old version of Python, look for
    # a newer version.
    macpython27 = '/Library/Frameworks/Python.framework/Versions/2.7/bin/python'
    if os.path.exists(macpython27):
      python = macpython27

  script_dir = os.path.dirname(os.path.abspath(__file__))
  src_dir = os.path.dirname(os.path.dirname(os.path.dirname(script_dir)))
  nacl_dir = os.path.join(src_dir, 'native_client')

  # Decide platform specifics.
  if options.browser_path:
    chrome_filename = options.browser_path
  else:
    chrome_filename = find_chrome.FindChrome(src_dir, [options.mode])
    if chrome_filename is None:
      raise Exception('Cannot find a chrome binary - specify one with '
                      '--browser_path?')

  env = dict(os.environ)
  if sys.platform in ['win32', 'cygwin']:
    if options.bits == 64:
      bits = 64
    elif options.bits == 32:
      bits = 32
    elif '64' in os.environ.get('PROCESSOR_ARCHITECTURE', '') or \
         '64' in os.environ.get('PROCESSOR_ARCHITEW6432', ''):
      bits = 64
    else:
      bits = 32
    msvs_path = ';'.join([
        r'c:\Program Files\Microsoft Visual Studio 9.0\VC',
        r'c:\Program Files (x86)\Microsoft Visual Studio 9.0\VC',
        r'c:\Program Files\Microsoft Visual Studio 9.0\Common7\Tools',
        r'c:\Program Files (x86)\Microsoft Visual Studio 9.0\Common7\Tools',
        r'c:\Program Files\Microsoft Visual Studio 8\VC',
        r'c:\Program Files (x86)\Microsoft Visual Studio 8\VC',
        r'c:\Program Files\Microsoft Visual Studio 8\Common7\Tools',
        r'c:\Program Files (x86)\Microsoft Visual Studio 8\Common7\Tools',
    ])
    env['PATH'] += ';' + msvs_path
    scons = [python, 'scons.py']
  elif sys.platform == 'darwin':
    if options.bits == 64:
      bits = 64
    elif options.bits == 32:
      bits = 32
    else:
      p = subprocess.Popen(['file', chrome_filename], stdout=subprocess.PIPE)
      (p_stdout, _) = p.communicate()
      assert p.returncode == 0
      if p_stdout.find('executable x86_64') >= 0:
        bits = 64
      else:
        bits = 32
    scons = [python, 'scons.py']
  else:
    if options.bits == 64:
      bits = 64
    elif options.bits == 32:
      bits = 32
    elif '64' in detect_host_arch.HostArch():
      bits = 64
    else:
      bits = 32
    # xvfb-run has a 2-second overhead per invocation, so it is cheaper to wrap
    # the entire build step rather than each test (browser_headless=1).
    # We also need to make sure that there are at least 24 bits per pixel.
    # https://code.google.com/p/chromium/issues/detail?id=316687
    scons = [
        'xvfb-run',
        '--auto-servernum',
        '--server-args', '-screen 0 1024x768x24',
        python, 'scons.py',
    ]

  if options.jobs > 1:
    scons.append('-j%d' % options.jobs)

  scons.append('disable_tests=%s' % options.disable_tests)

  if options.buildbot is not None:
    scons.append('buildbot=%s' % (options.buildbot,))

  # Clean the output of the previous build.
  # Incremental builds can get wedged in weird ways, so we're trading speed
  # for reliability.
  shutil.rmtree(os.path.join(nacl_dir, 'scons-out'), True)

  # check that the HOST (not target) is 64bit
  # this is emulating what msvs_env.bat is doing
  if '64' in os.environ.get('PROCESSOR_ARCHITECTURE', '') or \
     '64' in os.environ.get('PROCESSOR_ARCHITEW6432', ''):
    # 64bit HOST
    env['VS90COMNTOOLS'] = ('c:\\Program Files (x86)\\'
                            'Microsoft Visual Studio 9.0\\Common7\\Tools\\')
    env['VS80COMNTOOLS'] = ('c:\\Program Files (x86)\\'
                            'Microsoft Visual Studio 8.0\\Common7\\Tools\\')
  else:
    # 32bit HOST
    env['VS90COMNTOOLS'] = ('c:\\Program Files\\Microsoft Visual Studio 9.0\\'
                            'Common7\\Tools\\')
    env['VS80COMNTOOLS'] = ('c:\\Program Files\\Microsoft Visual Studio 8.0\\'
                            'Common7\\Tools\\')

  # Run nacl/chrome integration tests.
  # Note that we have to add nacl_irt_test to --mode in order to get
  # inbrowser_test_runner to run.
  # TODO(mseaborn): Change it so that inbrowser_test_runner is not a
  # special case.
  cmd = scons + ['--verbose', '-k', 'platform=x86-%d' % bits,
      '--mode=opt-host,nacl,nacl_irt_test',
      'chrome_browser_path=%s' % chrome_filename,
  ]
  if not options.integration_bot and not options.morenacl_bot:
    cmd.append('disable_flaky_tests=1')
  cmd.append('chrome_browser_tests')

  # Propagate path to JSON output if present.
  # Note that RunCommand calls sys.exit on errors, so potential errors
  # from one command won't be overwritten by another one. Overwriting
  # a successful results file with either success or failure is fine.
  if options.json_build_results_output_file:
    cmd.append('json_build_results_output_file=%s' %
               options.json_build_results_output_file)

  # Download the toolchain(s).
  pkg_ver_dir = os.path.join(nacl_dir, 'build', 'package_version')
  RunCommand([python, os.path.join(pkg_ver_dir, 'package_version.py'),
              '--exclude', 'arm_trusted',
              '--exclude', 'pnacl_newlib',
              '--exclude', 'nacl_arm_newlib',
              'sync', '--extract'],
             nacl_dir, os.environ)

  CleanTempDir()

  if options.enable_newlib:
    RunTests('nacl-newlib', cmd, nacl_dir, env)

  if options.enable_glibc:
    RunTests('nacl-glibc', cmd + ['--nacl_glibc'], nacl_dir, env)

def MakeCommandLineParser():
  parser = optparse.OptionParser()
  parser.add_option('-m', '--mode', dest='mode', default='Debug',
                    help='Debug/Release mode')
  parser.add_option('-j', dest='jobs', default=1, type='int',
                    help='Number of parallel jobs')

  parser.add_option('--enable_newlib', dest='enable_newlib', default=-1,
                    type='int', help='Run newlib tests?')
  parser.add_option('--enable_glibc', dest='enable_glibc', default=-1,
                    type='int', help='Run glibc tests?')

  parser.add_option('--json_build_results_output_file',
                    help='Path to a JSON file for machine-readable output.')

  # Deprecated, but passed to us by a script in the Chrome repo.
  # Replaced by --enable_glibc=0
  parser.add_option('--disable_glibc', dest='disable_glibc',
                    action='store_true', default=False,
                    help='Do not test using glibc.')

  parser.add_option('--disable_tests', dest='disable_tests',
                    type='string', default='',
                    help='Comma-separated list of tests to omit')
  builder_name = os.environ.get('BUILDBOT_BUILDERNAME', '')
  is_integration_bot = 'nacl-chrome' in builder_name
  parser.add_option('--integration_bot', dest='integration_bot',
                    type='int', default=int(is_integration_bot),
                    help='Is this an integration bot?')
  is_morenacl_bot = (
      'More NaCl' in builder_name or
      'naclmore' in builder_name)
  parser.add_option('--morenacl_bot', dest='morenacl_bot',
                    type='int', default=int(is_morenacl_bot),
                    help='Is this a morenacl bot?')

  # Not used on the bots, but handy for running the script manually.
  parser.add_option('--bits', dest='bits', action='store',
                    type='int', default=None,
                    help='32/64')
  parser.add_option('--browser_path', dest='browser_path', action='store',
                    type='string', default=None,
                    help='Path to the chrome browser.')
  parser.add_option('--buildbot', dest='buildbot', action='store',
                    type='string', default=None,
                    help='Value passed to scons as buildbot= option.')
  return parser

def Main():
  parser = MakeCommandLineParser()
  options, args = parser.parse_args()
  if options.integration_bot and options.morenacl_bot:
    parser.error('ERROR: cannot be both an integration bot and a morenacl bot')

  # Set defaults for enabling newlib.
  if options.enable_newlib == -1:
    options.enable_newlib = 1

  # Set defaults for enabling glibc.
  if options.enable_glibc == -1:
    if options.integration_bot or options.morenacl_bot:
      options.enable_glibc = 1
    else:
      options.enable_glibc = 0

  if args:
    parser.error('ERROR: invalid argument')
  BuildAndTest(options)

if __name__ == '__main__':
  Main()


import time

import pandas as pd

from warnings import warn

from carto.auth import APIKeyAuthClient
from carto.datasets import DatasetManager
from carto.exceptions import CartoException, CartoRateLimitException
from carto.sql import SQLClient, BatchSQLClient, CopySQLClient
from pyrestcli.exceptions import NotFoundException

from ..dataset_info import DatasetInfo
from ... import __version__
from ...auth.defaults import get_default_credentials
from ...utils.logger import log
from ...utils.geom_utils import encode_geometry_ewkb
from ...utils.utils import (is_sql_query, check_credentials, encode_row, map_geom_type, PG_NULL, double_quote,
                            create_tmp_name)
from ...utils.columns import (get_dataframe_columns_info, get_query_columns_info, obtain_converters, date_columns_names,
                              normalize_name)

DEFAULT_RETRY_TIMES = 3
BATCH_API_PAYLOAD_THRESHOLD = 12000

def retry_copy(func):
    def wrapper(*args, **kwargs):
        m_retry_times = kwargs.get('retry_times', DEFAULT_RETRY_TIMES)
        while m_retry_times >= 1:
            try:
                return func(*args, **kwargs)
            except CartoRateLimitException as err:
                m_retry_times -= 1

                if m_retry_times <= 0:
                    warn(('Read call was rate-limited. '
                          'This usually happens when there are multiple queries being read at the same time.'))
                    raise err

                warn('Read call rate limited. Waiting {s} seconds'.format(s=err.retry_after))
                time.sleep(err.retry_after)
                warn('Retrying...')
        return func(*args, **kwargs)
    return wrapper

def not_found(func):
    def decorator_func(*args, **kwargs):
        try:
            return func(*args, **kwargs)

        except CartoException as e:
            if hasattr(e, 'args') and isinstance(e.args, (list, tuple)) and type(e.args[0]) == NotFoundException:
                raise Exception('Resource not found') from None

            else:
                raise e

    return decorator_func

class ContextManager:

    def __init__(self, credentials):
        self.credentials = credentials or get_default_credentials()
        check_credentials(self.credentials)

        self.auth_client = _create_auth_client(self.credentials)
        self.sql_client = SQLClient(self.auth_client)
        self.copy_client = CopySQLClient(self.auth_client)
        self.batch_sql_client = BatchSQLClient(self.auth_client)

    @not_found
    def execute_query(self, query, parse_json=True, do_post=True, format=None, **request_args):
        return self.sql_client.send(query.strip(), parse_json, do_post, format, **request_args)

    @not_found
    def execute_long_running_query(self, query):
        return self.batch_sql_client.create_and_wait_for_completion(query.strip())

    def copy_to(self, source, schema=None, limit=None, retry_times=DEFAULT_RETRY_TIMES):
        query = self.compute_query(source, schema)
        columns = self._get_query_columns_info(query)
        copy_query = self._get_copy_query(query, columns, limit)
        return self._copy_to(copy_query, columns, retry_times)

    def copy_from(self, gdf, table_name, if_exists='fail', cartodbfy=True,
                  retry_times=DEFAULT_RETRY_TIMES):
        schema = self.get_schema()
        table_name = self.normalize_table_name(table_name)
        df_columns = get_dataframe_columns_info(gdf)

        if self.has_table(table_name, schema):
            if if_exists == 'replace':
                table_query = self._compute_query_from_table(table_name, schema)
                table_columns = self._get_query_columns_info(table_query)

                if self._compare_columns(df_columns, table_columns):
                    # Equal columns: truncate table
                    self._truncate_table(table_name, schema)
                else:
                    # Diff columns: truncate table and drop + add columns
                    self._truncate_and_drop_add_columns(
                        table_name, schema, df_columns, table_columns)

            elif if_exists == 'fail':
                raise Exception('Table "{schema}.{table_name}" already exists in your CARTO account. '
                                'Please choose a different `table_name` or use '
                                'if_exists="replace" to overwrite it.'.format(
                                    table_name=table_name, schema=schema))
            else:  # 'append'
                cartodbfy = False
        else:
            self._create_table_from_columns(table_name, schema, df_columns)

        self._copy_from(gdf, table_name, df_columns, retry_times)

        if cartodbfy is True:
            cartodbfy_query = _cartodbfy_query(table_name, schema)
            self.execute_long_running_query(cartodbfy_query)

        return table_name

    def create_table_from_query(self, query, table_name, if_exists):
        schema = self.get_schema()
        table_name = self.normalize_table_name(table_name)

        if self.has_table(table_name, schema):
            if if_exists == 'replace':
                # TODO: review logic copy_from
                self._drop_create_table_from_query(table_name, schema, query)
            elif if_exists == 'fail':
                raise Exception('Table "{schema}.{table_name}" already exists in your CARTO account. '
                                'Please choose a different `table_name` or use '
                                'if_exists="replace" to overwrite it.'.format(
                                    table_name=table_name, schema=schema))
            else:  # 'append'
                pass
        else:
            self._drop_create_table_from_query(table_name, schema, query)

        return table_name

    def list_tables(self, schema=None):
        datasets = DatasetManager(self.auth_client).filter(
            show_table_size_and_row_count='false',
            show_table='false',
            show_stats='false',
            show_likes='false',
            show_liked='false',
            show_permission='false',
            show_uses_builder_features='false',
            show_synchronization='false',
            load_totals='false'
        )
        datasets.sort(key=lambda x: x.updated_at, reverse=True)
        return pd.DataFrame([dataset.name for dataset in datasets], columns=['tables'])

    def has_table(self, table_name, schema=None):
        query = self.compute_query(table_name, schema)
        return self._check_exists(query)

    def delete_table(self, table_name):
        query = _drop_table_query(table_name)
        output = self.execute_query(query)
        return not('notices' in output and 'does not exist' in output['notices'][0])

    def _delete_function(self, function_name):
        query = _drop_function_query(function_name)
        self.execute_query(query)
        return function_name

    def _create_function(self, schema, statement,
                         function_name=None, columns_types=None, return_value='VOID', language='plpgsql'):
        function_name = function_name or create_tmp_name(base='tmp_func')
        safe_schema = double_quote(schema)
        query, qualified_func_name = _create_function_query(
            schema=safe_schema,
            function_name=function_name,
            statement=statement,
            columns_types=columns_types or '',
            return_value=return_value,
            language=language)
        self.execute_query(query)
        return qualified_func_name

    def rename_table(self, table_name, new_table_name, if_exists='fail'):
        new_table_name = self.normalize_table_name(new_table_name)

        if table_name == new_table_name:
            raise ValueError('Table names are equal. Please choose a different table name.')

        if not self.has_table(table_name):
            raise Exception('Table "{table_name}" does not exist in your CARTO account.'.format(
                                table_name=table_name))

        if self.has_table(new_table_name):
            if if_exists == 'replace':
                log.debug('Removing table "{}"'.format(new_table_name))
                self.delete_table(new_table_name)
            elif if_exists == 'fail':
                raise Exception('Table "{new_table_name}" already exists in your CARTO account. '
                                'Please choose a different `new_table_name` or use '
                                'if_exists="replace" to overwrite it.'.format(
                                    new_table_name=new_table_name))

        self._rename_table(table_name, new_table_name)
        return new_table_name

    def update_privacy_table(self, table_name, privacy=None):
        DatasetInfo(self.auth_client, table_name).update_privacy(privacy)

    def get_privacy(self, table_name):
        return DatasetInfo(self.auth_client, table_name).privacy

    def get_schema(self):
        """Get user schema from current credentials"""
        query = 'SELECT current_schema()'
        result = self.execute_query(query, do_post=False)
        schema = result['rows'][0]['current_schema']
        log.debug('schema: {}'.format(schema))
        return schema

    def get_geom_type(self, query):
        """Fetch geom type of a remote table or query"""
        distict_query = '''
            SELECT distinct ST_GeometryType(the_geom) AS geom_type
            FROM ({}) q
            LIMIT 5
        '''.format(query)
        response = self.execute_query(distict_query, do_post=False)
        if response and response.get('rows') and len(response.get('rows')) > 0:
            st_geom_type = response.get('rows')[0].get('geom_type')
            if st_geom_type:
                return map_geom_type(st_geom_type[3:])
        return None

    def get_num_rows(self, query):
        """Get the number of rows in the query"""
        result = self.execute_query('SELECT COUNT(*) FROM ({query}) _query'.format(query=query))
        return result.get('rows')[0].get('count')

    def get_bounds(self, query):
        extent_query = '''
            SELECT ARRAY[
                ARRAY[st_xmin(geom_env), st_ymin(geom_env)],
                ARRAY[st_xmax(geom_env), st_ymax(geom_env)]
            ] bounds FROM (
                SELECT ST_Extent(the_geom) geom_env
                FROM ({}) q
            ) q;
        '''.format(query)
        response = self.execute_query(extent_query, do_post=False)
        if response and response.get('rows') and len(response.get('rows')) > 0:
            return response.get('rows')[0].get('bounds')
        return None

    def get_column_names(self, source, schema=None, exclude=None):
        query = self.compute_query(source, schema)
        columns = [c.name for c in self._get_query_columns_info(query)]

        if exclude and isinstance(exclude, list):
            columns = list(set(columns) - set(exclude))

        return columns

    def is_public(self, query):
        # Used to detect public tables in queries in the publication,
        # because privacy only works for tables.
        public_auth_client = _create_auth_client(self.credentials, public=True)
        public_sql_client = SQLClient(public_auth_client)
        exists_query = 'EXPLAIN {}'.format(query)
        try:
            public_sql_client.send(exists_query, do_post=False)
            return True
        except CartoException:
            return False

    def get_table_names(self, query):
        # Used to detect tables in queries in the publication.
        query = 'SELECT CDB_QueryTablesText($q${}$q$) as tables'.format(query)
        result = self.execute_query(query)
        tables = []
        if result['total_rows'] > 0 and result['rows'][0]['tables']:
            # Dataset_info only works with tables without schema
            tables = [table.split('.')[1] if '.' in table else table for table in result['rows'][0]['tables']]
        return tables

    def _compare_columns(self, a, b):
        a_copy = [i for i in a if _not_reserved(i.name)]
        b_copy = [i for i in b if _not_reserved(i.name)]

        a_copy.sort()
        b_copy.sort()

        return a_copy == b_copy

    def _drop_create_table_from_query(self, table_name, schema, query):
        log.debug('DROP + CREATE table "{}"'.format(table_name))
        query = 'BEGIN; {drop}; {create}; COMMIT;'.format(
            drop=_drop_table_query(table_name),
            create=_create_table_from_query_query(table_name, query))
        self.execute_long_running_query(query)

    def _create_table_from_columns(self, table_name, schema, columns):
        log.debug('CREATE table "{}"'.format(table_name))
        query = 'BEGIN; {create}; COMMIT;'.format(
            create=_create_table_from_columns_query(table_name, columns))
        self.execute_query(query)

    def _truncate_table(self, table_name, schema):
        log.debug('TRUNCATE table "{}"'.format(table_name))
        query = 'BEGIN; {truncate}; COMMIT;'.format(
            truncate=_truncate_table_query(table_name))
        self.execute_query(query)

    def _truncate_and_drop_add_columns(self, table_name, schema, df_columns, table_columns):
        log.debug('TRUNCATE AND DROP + ADD columns table "{}"'.format(table_name))
        drop_columns = _drop_columns_query(table_name, table_columns)
        add_columns = _add_columns_query(table_name, df_columns)

        drop_add_columns = 'ALTER TABLE {table_name} {drop_columns},{add_columns};'.format(
            table_name=table_name, drop_columns=drop_columns, add_columns=add_columns)

        query = '{regenerate}; BEGIN; {truncate}; {drop_add_columns}; COMMIT;'.format(
            regenerate=_regenerate_table_query(table_name, schema) if self._check_regenerate_table_exists() else '',
            truncate=_truncate_table_query(table_name),
            drop_add_columns=drop_add_columns)

        query_length_over_threshold = len(query) > BATCH_API_PAYLOAD_THRESHOLD

        if query_length_over_threshold:
            qualified_func_name = self._create_function(
                schema=schema, statement=drop_add_columns)
            drop_add_func_sql = 'SELECT {}'.format(qualified_func_name)
            query = '''
                {regenerate};
                BEGIN;
                {truncate};
                {drop_add_func_sql};
                COMMIT;'''.format(
                regenerate=_regenerate_table_query(
                    table_name, schema) if self._check_regenerate_table_exists() else '',
                truncate=_truncate_table_query(table_name),
                drop_add_func_sql=drop_add_func_sql)
        try:
            self.execute_long_running_query(query)
        finally:
            if query_length_over_threshold:
                self._delete_function(qualified_func_name)

    def compute_query(self, source, schema=None):
        if is_sql_query(source):
            return source
        schema = schema or self.get_schema()
        return self._compute_query_from_table(source, schema)

    def _compute_query_from_table(self, table_name, schema):
        return 'SELECT * FROM "{schema}"."{table_name}"'.format(
            schema=schema or 'public',
            table_name=table_name
        )

    def _check_exists(self, query):
        exists_query = 'EXPLAIN {}'.format(query)
        try:
            self.execute_query(exists_query, do_post=False)
            return True
        except CartoException:
            return False

    def _check_regenerate_table_exists(self):
        query = '''
            SELECT 1
            FROM pg_catalog.pg_proc p
            LEFT JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace
            WHERE p.proname = 'cdb_regeneratetable' AND n.nspname = 'cartodb';
        '''
        result = self.execute_query(query)
        return len(result['rows']) > 0

    def _get_query_columns_info(self, query):
        query = 'SELECT * FROM ({}) _q LIMIT 0'.format(query)
        table_info = self.execute_query(query)
        return get_query_columns_info(table_info['fields'])

    def _get_copy_query(self, query, columns, limit):
        query_columns = [
            double_quote(column.name) for column in columns
            if (column.name != 'the_geom_webmercator')
        ]

        query = 'SELECT {columns} FROM ({query}) _q'.format(
            query=query,
            columns=','.join(query_columns))

        if limit is not None:
            if isinstance(limit, int) and (limit >= 0):
                query += ' LIMIT {limit}'.format(limit=limit)
            else:
                raise ValueError("`limit` parameter must an integer >= 0")

        return query

    @retry_copy
    def _copy_to(self, query, columns, retry_times=DEFAULT_RETRY_TIMES):
        log.debug('COPY TO')
        copy_query = "COPY ({0}) TO stdout WITH (FORMAT csv, HEADER true, NULL '{1}')".format(query, PG_NULL)

        raw_result = self.copy_client.copyto_stream(copy_query)

        converters = obtain_converters(columns)
        parse_dates = date_columns_names(columns)

        df = pd.read_csv(
            raw_result,
            converters=converters,
            parse_dates=parse_dates)

        return df

    @retry_copy
    def _copy_from(self, dataframe, table_name, columns, retry_times=DEFAULT_RETRY_TIMES):
        log.debug('COPY FROM')
        query = """
            COPY {table_name}({columns}) FROM stdin WITH (FORMAT csv, DELIMITER '|', NULL '{null}');
        """.format(
            table_name=table_name, null=PG_NULL,
            columns=','.join(double_quote(column.dbname) for column in columns)).strip()
        data = _compute_copy_data(dataframe, columns)

        self.copy_client.copyfrom(query, data)

    def _rename_table(self, table_name, new_table_name):
        query = _rename_table_query(table_name, new_table_name)
        self.execute_query(query)

    def normalize_table_name(self, table_name):
        norm_table_name = normalize_name(table_name)
        if norm_table_name != table_name:
            log.debug('Table name normalized: "{}"'.format(norm_table_name))
        return norm_table_name

def _drop_table_query(table_name, if_exists=True):
    return 'DROP TABLE {if_exists} {table_name}'.format(
        table_name=table_name,
        if_exists='IF EXISTS' if if_exists else '')

def _drop_function_query(function_name, columns_types=None, if_exists=True):
    if columns_types and not isinstance(columns_types, dict):
        raise ValueError('The columns_types parameter should be a dictionary of column names and types.')
    columns_types = columns_types or {}
    columns = ['{0} {1}'.format(cname, ctype) for cname, ctype in columns_types.items()]
    columns_str = ','.join(columns)
    return 'DROP FUNCTION {if_exists} {function_name}{columns_str_call}'.format(
        function_name=function_name,
        if_exists='IF EXISTS' if if_exists else '',
        columns_str_call='({columns_str})'.format(columns_str=columns_str) if columns else '')

def _truncate_table_query(table_name):
    return 'TRUNCATE TABLE {table_name}'.format(
        table_name=table_name)

def _create_function_query(schema, function_name, statement, columns_types, return_value, language):
    if columns_types and not isinstance(columns_types, dict):
        raise ValueError('The columns_types parameter should be a dictionary of column names and types.')
    columns_types = columns_types or {}
    columns = ['{0} {1}'.format(cname, ctype) for cname, ctype in columns_types.items()]
    columns_str = ','.join(columns) if columns else ''
    function_query = '''
        CREATE FUNCTION {schema}.{function_name}({columns_str})
        RETURNS {return_value} AS $$
        BEGIN
        {statement}
        END;
        $$ LANGUAGE {language}
    '''.format(schema=schema,
               function_name=function_name,
               statement=statement,
               columns_str=columns_str,
               return_value=return_value,
               language=language)
    qualified_func_name = '{schema}.{function_name}({columns_str})'.format(
            schema=schema, function_name=function_name, columns_str=columns_str)
    return function_query, qualified_func_name

def _drop_columns_query(table_name, columns):
    columns = ['DROP COLUMN {name}'.format(name=double_quote(c.dbname))
               for c in columns if _not_reserved(c.dbname)]
    return ','.join(columns)

def _add_columns_query(table_name, columns):
    columns = ['ADD COLUMN {name} {type}'.format(name=double_quote(c.dbname), type=c.dbtype)
               for c in columns if _not_reserved(c.dbname)]
    return ','.join(columns)

def _not_reserved(column):
    RESERVED_COLUMNS = ['cartodb_id', 'the_geom', 'the_geom_webmercator']
    return column not in RESERVED_COLUMNS

def _create_table_from_columns_query(table_name, columns):
    columns = ['{name} {type}'.format(name=double_quote(c.dbname), type=c.dbtype) for c in columns]
    return 'CREATE TABLE {table_name} ({columns})'.format(
        table_name=table_name,
        columns=','.join(columns))

def _create_table_from_query_query(table_name, query):
    return 'CREATE TABLE {table_name} AS ({query})'.format(table_name=table_name, query=query)

def _cartodbfy_query(table_name, schema):
    return "SELECT CDB_CartodbfyTable('{schema}', '{table_name}')".format(
        schema=schema, table_name=table_name)

def _regenerate_table_query(table_name, schema):
    return "SELECT CDB_RegenerateTable('{schema}.{table_name}'::regclass)".format(
        schema=schema, table_name=table_name)

def _rename_table_query(table_name, new_table_name):
    return 'ALTER TABLE {table_name} RENAME TO {new_table_name};'.format(
        table_name=table_name, new_table_name=new_table_name)

def _create_auth_client(credentials, public=False):
    return APIKeyAuthClient(
        base_url=credentials.base_url,
        api_key='default_public' if public else credentials.api_key,
        session=credentials.session,
        client_id='cartoframes_{}'.format(__version__),
        user_agent='cartoframes_{}'.format(__version__))

def _compute_copy_data(df, columns):
    for index in df.index:
        row_data = []
        for column in columns:
            val = df.at[index, column.name]

            if column.is_geom:
                val = encode_geometry_ewkb(val)

            row_data.append(encode_row(val))

        csv_row = b'|'.join(row_data)
        csv_row += b'\n'

        yield csv_row


"""
======================================
Sparse inverse covariance estimation
======================================

Using the GraphLasso estimator to learn a covariance and sparse precision
from a small number of samples.

To estimate a probabilistic model (e.g. a Gaussian model), estimating the
precision matrix, that is the inverse covariance matrix, is as important
as estimating the covariance matrix. Indeed a Gaussian model is
parametrized by the precision matrix.

To be in favorable recovery conditions, we sample the data from a model
with a sparse inverse covariance matrix. In addition, we ensure that the
data is not too much correlated (limiting the largest coefficient of the
precision matrix) and that there a no small coefficients in the
precision matrix that cannot be recovered. In addition, with a small
number of observations, it is easier to recover a correlation matrix
rather than a covariance, thus we scale the time series.

Here, the number of samples is slightly larger than the number of
dimensions, thus the empirical covariance is still invertible. However,
as the observations are strongly correlated, the empirical covariance
matrix is ill-conditioned and as a result its inverse --the empirical
precision matrix-- is very far from the ground truth.

If we use l2 shrinkage, as with the Ledoit-Wolf estimator, as the number
of samples is small, we need to shrink a lot. As a result, the
Ledoit-Wolf precision is fairly close to the ground truth precision, that
is not far from being diagonal, but the off-diagonal structure is lost.

The l1-penalized estimator can recover part of this off-diagonal
structure. It learns a sparse precision. It is not able to
recover the exact sparsity pattern: it detects too many non-zero
coefficients. However, the highest non-zero coefficients of the l1
estimated correspond to the non-zero coefficients in the ground truth.
Finally, the coefficients of the l1 precision estimate are biased toward
zero: because of the penalty, they are all smaller than the corresponding
ground truth value, as can be seen on the figure.

Note that, the color range of the precision matrices is tweaked to
improve readability of the figure. The full range of values of the
empirical precision is not displayed.

The alpha parameter of the GraphLasso setting the sparsity of the model is
set by internal cross-validation in the GraphLassoCV. As can be
seen on figure 2, the grid to compute the cross-validation score is
iteratively refined in the neighborhood of the maximum.
"""
print(__doc__)
# author: Gael Varoquaux <gael.varoquaux@inria.fr>
# License: BSD 3 clause
# Copyright: INRIA

import numpy as np
from scipy import linalg
from sklearn.datasets import make_sparse_spd_matrix
from sklearn.covariance import GraphLassoCV, ledoit_wolf
import matplotlib.pyplot as plt

##############################################################################
# Generate the data
n_samples = 60
n_features = 20

prng = np.random.RandomState(1)
prec = make_sparse_spd_matrix(n_features, alpha=.98,
                              smallest_coef=.4,
                              largest_coef=.7,
                              random_state=prng)
cov = linalg.inv(prec)
d = np.sqrt(np.diag(cov))
cov /= d
cov /= d[:, np.newaxis]
prec *= d
prec *= d[:, np.newaxis]
X = prng.multivariate_normal(np.zeros(n_features), cov, size=n_samples)
X -= X.mean(axis=0)
X /= X.std(axis=0)

##############################################################################
# Estimate the covariance
emp_cov = np.dot(X.T, X) / n_samples

model = GraphLassoCV()
model.fit(X)
cov_ = model.covariance_
prec_ = model.precision_

lw_cov_, _ = ledoit_wolf(X)
lw_prec_ = linalg.inv(lw_cov_)

##############################################################################
# Plot the results
plt.figure(figsize=(10, 6))
plt.subplots_adjust(left=0.02, right=0.98)

# plot the covariances
covs = [('Empirical', emp_cov), ('Ledoit-Wolf', lw_cov_),
        ('GraphLasso', cov_), ('True', cov)]
vmax = cov_.max()
for i, (name, this_cov) in enumerate(covs):
    plt.subplot(2, 4, i + 1)
    plt.imshow(this_cov, interpolation='nearest', vmin=-vmax, vmax=vmax,
               cmap=plt.cm.RdBu_r)
    plt.xticks(())
    plt.yticks(())
    plt.title('%s covariance' % name)

# plot the precisions
precs = [('Empirical', linalg.inv(emp_cov)), ('Ledoit-Wolf', lw_prec_),
         ('GraphLasso', prec_), ('True', prec)]
vmax = .9 * prec_.max()
for i, (name, this_prec) in enumerate(precs):
    ax = plt.subplot(2, 4, i + 5)
    plt.imshow(np.ma.masked_equal(this_prec, 0),
               interpolation='nearest', vmin=-vmax, vmax=vmax,
               cmap=plt.cm.RdBu_r)
    plt.xticks(())
    plt.yticks(())
    plt.title('%s precision' % name)
    ax.set_axis_bgcolor('.7')

# plot the model selection metric
plt.figure(figsize=(4, 3))
plt.axes([.2, .15, .75, .7])
plt.plot(model.cv_alphas_, np.mean(model.grid_scores, axis=1), 'o-')
plt.axvline(model.alpha_, color='.5')
plt.title('Model selection')
plt.ylabel('Cross-validation score')
plt.xlabel('alpha')

plt.show()


"""
Tests specific to the collections module.
"""
from __future__ import (absolute_import, division, print_function,
                        unicode_literals)

import io

import numpy as np
from numpy.testing import (
    assert_array_equal, assert_array_almost_equal, assert_equal)
import pytest

import matplotlib.pyplot as plt
import matplotlib.collections as mcollections
import matplotlib.transforms as mtransforms
from matplotlib.collections import Collection, EventCollection
from matplotlib.testing.decorators import image_comparison

def generate_EventCollection_plot():
    '''
    generate the initial collection and plot it
    '''
    positions = np.array([0., 1., 2., 3., 5., 8., 13., 21.])
    extra_positions = np.array([34., 55., 89.])
    orientation = 'horizontal'
    lineoffset = 1
    linelength = .5
    linewidth = 2
    color = [1, 0, 0, 1]
    linestyle = 'solid'
    antialiased = True

    coll = EventCollection(positions,
                           orientation=orientation,
                           lineoffset=lineoffset,
                           linelength=linelength,
                           linewidth=linewidth,
                           color=color,
                           linestyle=linestyle,
                           antialiased=antialiased
                           )

    fig = plt.figure()
    splt = fig.add_subplot(1, 1, 1)
    splt.add_collection(coll)
    splt.set_title('EventCollection: default')
    props = {'positions': positions,
             'extra_positions': extra_positions,
             'orientation': orientation,
             'lineoffset': lineoffset,
             'linelength': linelength,
             'linewidth': linewidth,
             'color': color,
             'linestyle': linestyle,
             'antialiased': antialiased
             }
    splt.set_xlim(-1, 22)
    splt.set_ylim(0, 2)
    return splt, coll, props

@image_comparison(baseline_images=['EventCollection_plot__default'])
def test__EventCollection__get_segments():
    '''
    check to make sure the default segments have the correct coordinates
    '''
    _, coll, props = generate_EventCollection_plot()
    check_segments(coll,
                   props['positions'],
                   props['linelength'],
                   props['lineoffset'],
                   props['orientation'])

def test__EventCollection__get_positions():
    '''
    check to make sure the default positions match the input positions
    '''
    _, coll, props = generate_EventCollection_plot()
    np.testing.assert_array_equal(props['positions'], coll.get_positions())

def test__EventCollection__get_orientation():
    '''
    check to make sure the default orientation matches the input
    orientation
    '''
    _, coll, props = generate_EventCollection_plot()
    assert_equal(props['orientation'], coll.get_orientation())

def test__EventCollection__is_horizontal():
    '''
    check to make sure the default orientation matches the input
    orientation
    '''
    _, coll, _ = generate_EventCollection_plot()
    assert_equal(True, coll.is_horizontal())

def test__EventCollection__get_linelength():
    '''
    check to make sure the default linelength matches the input linelength
    '''
    _, coll, props = generate_EventCollection_plot()
    assert_equal(props['linelength'], coll.get_linelength())

def test__EventCollection__get_lineoffset():
    '''
    check to make sure the default lineoffset matches the input lineoffset
    '''
    _, coll, props = generate_EventCollection_plot()
    assert_equal(props['lineoffset'], coll.get_lineoffset())

def test__EventCollection__get_linestyle():
    '''
    check to make sure the default linestyle matches the input linestyle
    '''
    _, coll, _ = generate_EventCollection_plot()
    assert_equal(coll.get_linestyle(), [(None, None)])

def test__EventCollection__get_color():
    '''
    check to make sure the default color matches the input color
    '''
    _, coll, props = generate_EventCollection_plot()
    np.testing.assert_array_equal(props['color'], coll.get_color())
    check_allprop_array(coll.get_colors(), props['color'])

@image_comparison(baseline_images=['EventCollection_plot__set_positions'])
def test__EventCollection__set_positions():
    '''
    check to make sure set_positions works properly
    '''
    splt, coll, props = generate_EventCollection_plot()
    new_positions = np.hstack([props['positions'], props['extra_positions']])
    coll.set_positions(new_positions)
    np.testing.assert_array_equal(new_positions, coll.get_positions())
    check_segments(coll, new_positions,
                   props['linelength'],
                   props['lineoffset'],
                   props['orientation'])
    splt.set_title('EventCollection: set_positions')
    splt.set_xlim(-1, 90)

@image_comparison(baseline_images=['EventCollection_plot__add_positions'])
def test__EventCollection__add_positions():
    '''
    check to make sure add_positions works properly
    '''
    splt, coll, props = generate_EventCollection_plot()
    new_positions = np.hstack([props['positions'],
                               props['extra_positions'][0]])
    coll.add_positions(props['extra_positions'][0])
    np.testing.assert_array_equal(new_positions, coll.get_positions())
    check_segments(coll,
                   new_positions,
                   props['linelength'],
                   props['lineoffset'],
                   props['orientation'])
    splt.set_title('EventCollection: add_positions')
    splt.set_xlim(-1, 35)

@image_comparison(baseline_images=['EventCollection_plot__append_positions'])
def test__EventCollection__append_positions():
    '''
    check to make sure append_positions works properly
    '''
    splt, coll, props = generate_EventCollection_plot()
    new_positions = np.hstack([props['positions'],
                               props['extra_positions'][2]])
    coll.append_positions(props['extra_positions'][2])
    np.testing.assert_array_equal(new_positions, coll.get_positions())
    check_segments(coll,
                   new_positions,
                   props['linelength'],
                   props['lineoffset'],
                   props['orientation'])
    splt.set_title('EventCollection: append_positions')
    splt.set_xlim(-1, 90)

@image_comparison(baseline_images=['EventCollection_plot__extend_positions'])
def test__EventCollection__extend_positions():
    '''
    check to make sure extend_positions works properly
    '''
    splt, coll, props = generate_EventCollection_plot()
    new_positions = np.hstack([props['positions'],
                               props['extra_positions'][1:]])
    coll.extend_positions(props['extra_positions'][1:])
    np.testing.assert_array_equal(new_positions, coll.get_positions())
    check_segments(coll,
                   new_positions,
                   props['linelength'],
                   props['lineoffset'],
                   props['orientation'])
    splt.set_title('EventCollection: extend_positions')
    splt.set_xlim(-1, 90)

@image_comparison(baseline_images=['EventCollection_plot__switch_orientation'])
def test__EventCollection__switch_orientation():
    '''
    check to make sure switch_orientation works properly
    '''
    splt, coll, props = generate_EventCollection_plot()
    new_orientation = 'vertical'
    coll.switch_orientation()
    assert_equal(new_orientation, coll.get_orientation())
    assert_equal(False, coll.is_horizontal())
    new_positions = coll.get_positions()
    check_segments(coll,
                   new_positions,
                   props['linelength'],
                   props['lineoffset'], new_orientation)
    splt.set_title('EventCollection: switch_orientation')
    splt.set_ylim(-1, 22)
    splt.set_xlim(0, 2)

@image_comparison(
    baseline_images=['EventCollection_plot__switch_orientation__2x'])
def test__EventCollection__switch_orientation_2x():
    '''
    check to make sure calling switch_orientation twice sets the
    orientation back to the default
    '''
    splt, coll, props = generate_EventCollection_plot()
    coll.switch_orientation()
    coll.switch_orientation()
    new_positions = coll.get_positions()
    assert_equal(props['orientation'], coll.get_orientation())
    assert_equal(True, coll.is_horizontal())
    np.testing.assert_array_equal(props['positions'], new_positions)
    check_segments(coll,
                   new_positions,
                   props['linelength'],
                   props['lineoffset'],
                   props['orientation'])
    splt.set_title('EventCollection: switch_orientation 2x')

@image_comparison(baseline_images=['EventCollection_plot__set_orientation'])
def test__EventCollection__set_orientation():
    '''
    check to make sure set_orientation works properly
    '''
    splt, coll, props = generate_EventCollection_plot()
    new_orientation = 'vertical'
    coll.set_orientation(new_orientation)
    assert_equal(new_orientation, coll.get_orientation())
    assert_equal(False, coll.is_horizontal())
    check_segments(coll,
                   props['positions'],
                   props['linelength'],
                   props['lineoffset'],
                   new_orientation)
    splt.set_title('EventCollection: set_orientation')
    splt.set_ylim(-1, 22)
    splt.set_xlim(0, 2)

@image_comparison(baseline_images=['EventCollection_plot__set_linelength'])
def test__EventCollection__set_linelength():
    '''
    check to make sure set_linelength works properly
    '''
    splt, coll, props = generate_EventCollection_plot()
    new_linelength = 15
    coll.set_linelength(new_linelength)
    assert_equal(new_linelength, coll.get_linelength())
    check_segments(coll,
                   props['positions'],
                   new_linelength,
                   props['lineoffset'],
                   props['orientation'])
    splt.set_title('EventCollection: set_linelength')
    splt.set_ylim(-20, 20)

@image_comparison(baseline_images=['EventCollection_plot__set_lineoffset'])
def test__EventCollection__set_lineoffset():
    '''
    check to make sure set_lineoffset works properly
    '''
    splt, coll, props = generate_EventCollection_plot()
    new_lineoffset = -5.
    coll.set_lineoffset(new_lineoffset)
    assert_equal(new_lineoffset, coll.get_lineoffset())
    check_segments(coll,
                   props['positions'],
                   props['linelength'],
                   new_lineoffset,
                   props['orientation'])
    splt.set_title('EventCollection: set_lineoffset')
    splt.set_ylim(-6, -4)

@image_comparison(baseline_images=['EventCollection_plot__set_linestyle'])
def test__EventCollection__set_linestyle():
    '''
    check to make sure set_linestyle works properly
    '''
    splt, coll, _ = generate_EventCollection_plot()
    new_linestyle = 'dashed'
    coll.set_linestyle(new_linestyle)
    assert_equal(coll.get_linestyle(), [(0, (6.0, 6.0))])
    splt.set_title('EventCollection: set_linestyle')

@image_comparison(baseline_images=['EventCollection_plot__set_ls_dash'],
                  remove_text=True)
def test__EventCollection__set_linestyle_single_dash():
    '''
    check to make sure set_linestyle accepts a single dash pattern
    '''
    splt, coll, _ = generate_EventCollection_plot()
    new_linestyle = (0, (6., 6.))
    coll.set_linestyle(new_linestyle)
    assert_equal(coll.get_linestyle(), [(0, (6.0, 6.0))])
    splt.set_title('EventCollection: set_linestyle')

@image_comparison(baseline_images=['EventCollection_plot__set_linewidth'])
def test__EventCollection__set_linewidth():
    '''
    check to make sure set_linestyle works properly
    '''
    splt, coll, _ = generate_EventCollection_plot()
    new_linewidth = 5
    coll.set_linewidth(new_linewidth)
    assert_equal(coll.get_linewidth(), new_linewidth)
    splt.set_title('EventCollection: set_linewidth')

@image_comparison(baseline_images=['EventCollection_plot__set_color'])
def test__EventCollection__set_color():
    '''
    check to make sure set_color works properly
    '''
    splt, coll, _ = generate_EventCollection_plot()
    new_color = np.array([0, 1, 1, 1])
    coll.set_color(new_color)
    np.testing.assert_array_equal(new_color, coll.get_color())
    check_allprop_array(coll.get_colors(), new_color)
    splt.set_title('EventCollection: set_color')

def check_segments(coll, positions, linelength, lineoffset, orientation):
    '''
    check to make sure all values in the segment are correct, given a
    particular set of inputs

    note: this is not a test, it is used by tests
    '''
    segments = coll.get_segments()
    if (orientation.lower() == 'horizontal'
            or orientation.lower() == 'none' or orientation is None):
        # if horizontal, the position in is in the y-axis
        pos1 = 1
        pos2 = 0
    elif orientation.lower() == 'vertical':
        # if vertical, the position in is in the x-axis
        pos1 = 0
        pos2 = 1
    else:
        raise ValueError("orientation must be 'horizontal' or 'vertical'")

    # test to make sure each segment is correct
    for i, segment in enumerate(segments):
        assert_equal(segment[0, pos1], lineoffset + linelength / 2.)
        assert_equal(segment[1, pos1], lineoffset - linelength / 2.)
        assert_equal(segment[0, pos2], positions[i])
        assert_equal(segment[1, pos2], positions[i])

def check_allprop_array(values, target):
    '''
    check to make sure all values match the given target if arrays

    note: this is not a test, it is used by tests
    '''
    for value in values:
        np.testing.assert_array_equal(value, target)

def test_null_collection_datalim():
    col = mcollections.PathCollection([])
    col_data_lim = col.get_datalim(mtransforms.IdentityTransform())
    assert_array_equal(col_data_lim.get_points(),
                       mtransforms.Bbox.null().get_points())

def test_add_collection():
    # Test if data limits are unchanged by adding an empty collection.
    # Github issue #1490, pull #1497.
    plt.figure()
    ax = plt.axes()
    coll = ax.scatter([0, 1], [0, 1])
    ax.add_collection(coll)
    bounds = ax.dataLim.bounds
    coll = ax.scatter([], [])
    assert_equal(ax.dataLim.bounds, bounds)

def test_quiver_limits():
    ax = plt.axes()
    x, y = np.arange(8), np.arange(10)
    u = v = np.linspace(0, 10, 80).reshape(10, 8)
    q = plt.quiver(x, y, u, v)
    assert_equal(q.get_datalim(ax.transData).bounds, (0., 0., 7., 9.))

    plt.figure()
    ax = plt.axes()
    x = np.linspace(-5, 10, 20)
    y = np.linspace(-2, 4, 10)
    y, x = np.meshgrid(y, x)
    trans = mtransforms.Affine2D().translate(25, 32) + ax.transData
    plt.quiver(x, y, np.sin(x), np.cos(y), transform=trans)
    assert_equal(ax.dataLim.bounds, (20.0, 30.0, 15.0, 6.0))

def test_barb_limits():
    ax = plt.axes()
    x = np.linspace(-5, 10, 20)
    y = np.linspace(-2, 4, 10)
    y, x = np.meshgrid(y, x)
    trans = mtransforms.Affine2D().translate(25, 32) + ax.transData
    plt.barbs(x, y, np.sin(x), np.cos(y), transform=trans)
    # The calculated bounds are approximately the bounds of the original data,
    # this is because the entire path is taken into account when updating the
    # datalim.
    assert_array_almost_equal(ax.dataLim.bounds, (20, 30, 15, 6),
                              decimal=1)

@image_comparison(baseline_images=['EllipseCollection_test_image'],
                  extensions=['png'],
                  remove_text=True)
def test_EllipseCollection():
    # Test basic functionality
    fig, ax = plt.subplots()
    x = np.arange(4)
    y = np.arange(3)
    X, Y = np.meshgrid(x, y)
    XY = np.vstack((X.ravel(), Y.ravel())).T

    ww = X/float(x[-1])
    hh = Y/float(y[-1])
    aa = np.ones_like(ww) * 20  # first axis is 20 degrees CCW from x axis

    ec = mcollections.EllipseCollection(ww, hh, aa,
                                        units='x',
                                        offsets=XY,
                                        transOffset=ax.transData,
                                        facecolors='none')
    ax.add_collection(ec)
    ax.autoscale_view()

@image_comparison(baseline_images=['polycollection_close'],
                  extensions=['png'], remove_text=True)
def test_polycollection_close():
    from mpl_toolkits.mplot3d import Axes3D

    vertsQuad = [
        [[0., 0.], [0., 1.], [1., 1.], [1., 0.]],
        [[0., 1.], [2., 3.], [2., 2.], [1., 1.]],
        [[2., 2.], [2., 3.], [4., 1.], [3., 1.]],
        [[3., 0.], [3., 1.], [4., 1.], [4., 0.]]]

    fig = plt.figure()
    ax = Axes3D(fig)

    colors = ['r', 'g', 'b', 'y', 'k']
    zpos = list(range(5))

    poly = mcollections.PolyCollection(
        vertsQuad * len(zpos), linewidth=0.25)
    poly.set_alpha(0.7)

    # need to have a z-value for *each* polygon = element!
    zs = []
    cs = []
    for z, c in zip(zpos, colors):
        zs.extend([z] * len(vertsQuad))
        cs.extend([c] * len(vertsQuad))

    poly.set_color(cs)

    ax.add_collection3d(poly, zs=zs, zdir='y')

    # axis limit settings:
    ax.set_xlim3d(0, 4)
    ax.set_zlim3d(0, 3)
    ax.set_ylim3d(0, 4)

@image_comparison(baseline_images=['regularpolycollection_rotate'],
                  extensions=['png'], remove_text=True)
def test_regularpolycollection_rotate():
    xx, yy = np.mgrid[:10, :10]
    xy_points = np.transpose([xx.flatten(), yy.flatten()])
    rotations = np.linspace(0, 2*np.pi, len(xy_points))

    fig, ax = plt.subplots()
    for xy, alpha in zip(xy_points, rotations):
        col = mcollections.RegularPolyCollection(
            4, sizes=(100,), rotation=alpha,
            offsets=[xy], transOffset=ax.transData)
        ax.add_collection(col, autolim=True)
    ax.autoscale_view()

@image_comparison(baseline_images=['regularpolycollection_scale'],
                  extensions=['png'], remove_text=True)
def test_regularpolycollection_scale():
    # See issue #3860

    class SquareCollection(mcollections.RegularPolyCollection):
        def __init__(self, **kwargs):
            super(SquareCollection, self).__init__(
                4, rotation=np.pi/4., **kwargs)

        def get_transform(self):
            """Return transform scaling circle areas to data space."""
            ax = self.axes

            pts2pixels = 72.0 / ax.figure.dpi

            scale_x = pts2pixels * ax.bbox.width / ax.viewLim.width
            scale_y = pts2pixels * ax.bbox.height / ax.viewLim.height
            return mtransforms.Affine2D().scale(scale_x, scale_y)

    fig, ax = plt.subplots()

    xy = [(0, 0)]
    # Unit square has a half-diagonal of `1 / sqrt(2)`, so `pi * r**2`
    # equals...
    circle_areas = [np.pi / 2]
    squares = SquareCollection(sizes=circle_areas, offsets=xy,
                               transOffset=ax.transData)
    ax.add_collection(squares, autolim=True)
    ax.axis([-1, 1, -1, 1])

def test_picking():
    fig, ax = plt.subplots()
    col = ax.scatter([0], [0], [1000], picker=True)
    fig.savefig(io.BytesIO(), dpi=fig.dpi)

    class MouseEvent(object):
        pass
    event = MouseEvent()
    event.x = 325
    event.y = 240

    found, indices = col.contains(event)
    assert found
    assert_array_equal(indices['ind'], [0])

def test_linestyle_single_dashes():
    plt.scatter([0, 1, 2], [0, 1, 2], linestyle=(0., [2., 2.]))
    plt.draw()

@image_comparison(baseline_images=['size_in_xy'], remove_text=True,
                  extensions=['png'])
def test_size_in_xy():
    fig, ax = plt.subplots()

    widths, heights, angles = (10, 10), 10, 0
    widths = 10, 10
    coords = [(10, 10), (15, 15)]
    e = mcollections.EllipseCollection(
        widths, heights, angles,
        units='xy',
        offsets=coords,
        transOffset=ax.transData)

    ax.add_collection(e)

    ax.set_xlim(0, 30)
    ax.set_ylim(0, 30)

def test_pandas_indexing():
    pd = pytest.importorskip('pandas')

    # Should not fail break when faced with a
    # non-zero indexed series
    index = [11, 12, 13]
    ec = fc = pd.Series(['red', 'blue', 'green'], index=index)
    lw = pd.Series([1, 2, 3], index=index)
    ls = pd.Series(['solid', 'dashed', 'dashdot'], index=index)
    aa = pd.Series([True, False, True], index=index)

    Collection(edgecolors=ec)
    Collection(facecolors=fc)
    Collection(linewidths=lw)
    Collection(linestyles=ls)
    Collection(antialiaseds=aa)

@pytest.mark.style('default')
def test_lslw_bcast():
    col = mcollections.PathCollection([])
    col.set_linestyles(['-', '-'])
    col.set_linewidths([1, 2, 3])

    assert_equal(col.get_linestyles(), [(None, None)] * 6)
    assert_equal(col.get_linewidths(), [1, 2, 3] * 2)

    col.set_linestyles(['-', '-', '-'])
    assert_equal(col.get_linestyles(), [(None, None)] * 3)
    assert_equal(col.get_linewidths(), [1, 2, 3])

@image_comparison(baseline_images=['scatter_post_alpha'],
                  extensions=['png'], remove_text=True,
                  style='default')
def test_scatter_post_alpha():
    fig, ax = plt.subplots()
    sc = ax.scatter(range(5), range(5), c=range(5))
    # this needs to be here to update internal state
    fig.canvas.draw()
    sc.set_alpha(.1)


import matplotlib
matplotlib.use('Agg')
import matplotlib.pylab as plt
import matplotlib.colors as mat_col
from matplotlib.colors import LinearSegmentedColormap
import scipy
import scipy.cluster.hierarchy as sch
from scipy.cluster.hierarchy import set_link_color_palette
import numpy as np
import pandas as pd
import glob
from scipy.stats import pearsonr
from scipy.stats import spearmanr
from scipy.spatial import distance

#Dig=pd.read_csv("all_regions_sum_nPix_perk_red_channel_PaperData_thres50_newnames.csv")
#Dig=pd.read_csv("all_regions_sum_nPix_perk_green_channel_PaperData_thres50_newnames.csv")
#Dig=pd.read_csv("all_regions_sum_perk_red_channel_PaperData_thres50_newnames.csv")
#Dir=pd.read_csv("all_regions_sum_perk_red_channel_PaperData_newnames.csv")
#Db=pd.read_csv("MAYbehaviorfullset_transposed.csv")
Db=pd.read_csv("AUG16_12_dectest.csv")
#Db=pd.read_csv("AUGMAY18testingfinalfullgoodonesoct30nonoise_transposed.csv")

#Dig = Dig.applymap(np.log)
#Digl = Dig # use if skipping log10
#Digl = Dig.applymap(np.log10)

#print Dig
#Digl = Digl.replace([np.inf, -np.inf], 0)
#Digl = Digl.replace([np.inf, -np.inf], np.nan)

# use if not doing log10
#Digl = Digl.replace([0], np.nan)

#Dig = Dig.replace([0], np.nan)
#DignoNA = Dig.dropna()

#Db = Db.apply(lambda x: [y if 0 < y < 0.05 else np.nan for y in x])
#Db = Db.apply(lambda x: [y if -0.05 < y < 0 else np.nan for y in x])

#print Db["adamtsl3"]
#for binarizing
# DEC 2018, THIS BINARIZING WORKS, BUT NOT DOIN GIT
# only binarizing the "non-significant" data
Db = Db.apply(lambda x: [y if -0.05 < y < 0.05 else 1 for y in x])
# convert all non-significant values to large number
##Db = Db.apply(lambda x: [y if -0.05 < y < 0.05 else 5 for y in x])
#print Db["adamtsl3"]
# keep all positive values, everything negative (between 0 and -0.05) becomes -1
##Db = Db.apply(lambda x: [y if y > 0 else -1 for y in x])
#print Db["adamtsl3"]
##Db = Db.apply(lambda x: [y if y < 2 else 0 for y in x])
#print Db["adamtsl3"]
# everything that is negative or 0 stays the same, everything else (between 0 and 0.05) becomes 1
##Db = Db.apply(lambda x: [y if y <= 0 else 1 for y in x])
#print Db["adamtsl3"]

#Db = Db.apply(lambda x: [y if y == np.nan else 1 for y in x])
#Db = Db.apply(lambda x: [y if y != np.nan else 0 for y in x])
# TRYING LOG ON P-VALUES, NOT SURE IF GOOD IDEA
#Db = Db.applymap(np.log10)
###Db = Db.apply(lambda x: [y if -0.1 < y < 0.1 else np.nan for y in x])
#print Db
#exit()

corrlist = []
dfdict = {}
dfdictdist = {}
collist = []
for column1 in Db:
	for column2 in Db:
		corr = Db[column1].corr(Db[column2], min_periods=6)
	#	dist = np.square(Db[column1] - Db[column2])
	#	print dist
		dist = distance.euclidean(Db[column1], Db[column2])
#		print dist
		#corr = Db[column1].corr(Dig[column2], method='spearman', min_periods=7)
	#	if corr > 0.6 or corr < -0.6:
			#corrlist.append( (corr, column1, column2))
			#newdf = pd.concat([Dig[column2], Digl[column2], Db[column1]], axis=1)
		newdf = pd.concat([Db[column2], Db[column1]], axis=1)
	#		newdf = newdf.dropna()
		corrlist.append( (corr, newdf, column1, column2, dist))
		if column1 in dfdict.keys():
			dfdict[column1].append(corr)
			dfdictdist[column1].append(dist)
		else:
			dfdict[column1] = []
			dfdictdist[column1] = []
			dfdict[column1].append(corr)
			dfdictdist[column1].append(dist)
		if column2 not in collist:
			collist.append(column2)
			#corrlist.append( (corr, column1, column2, newdf))
			#newdf = Dig[column2].copy()
			#newdf2 = newdf.concat(Db[column1])
			#newdf[column1] = Db[column1]
			#print newdf.dropna()
			#exit()
	#	break
	#break

#print dfdict
#print dfdictdist
#print collist

dfcor = pd.DataFrame.from_dict(dfdict, orient='index')
dfcor.columns = collist
dfdist = pd.DataFrame.from_dict(dfdictdist, orient='index')
dfdist.columns = collist

dfcor = dfcor.sort_index()
dfdist = dfdist.sort_index()

dfcor.to_csv("dec_correlation_sort1.csv")
dfdist.to_csv("dec_distance_sort1.csv")

#print dfcor

#corrlist.sort(key=lambda tup: tup[0])

#old way of just printing before generate the DF
##for i in range(0, len(corrlist)):
##	print corrlist[i][0], corrlist[i][4], corrlist[i][2], corrlist[i][3]
	#print corrlist[i][1]
	#print corrlist[i][2]

#Db=pd.read_csv("MAY2018fullheatmapsetfinal_0.csv")
#Db = Db.transpose()
#Dig = Dig.values
#Dir = Dir.values
#Db = Db.values

#print "test1"
#with pd.option_context('display.max_rows', None, 'display.max_columns', None):
#	print Dig
#print "test2"
#with pd.option_context('display.max_rows', None, 'display.max_columns', None):
#	print Db
#Digb = Dig[:,1:]
#Dirb = Dir[:,1:]
#Digb = np.delete(Dig, 0, axis=1)
#Dbb = Db[:,1:]
#Dbb = np.delete(Db, 0, axis=1)

#Digb = np.log(Digb)

#Digb = Dig.values
#Dbb = Db.values

#print "test1"
#print Dbb
#print "test2"
#print Digb

#print np.shape(Dbb)
#print np.shape(Digb)

#for row in range(Digb.shape[0]):
 #print str(pearsonr(Dbb[row,:], Digb[row,:]))
 #print str(pearsonr(Dbb[:,row], Digb[:,row]))

#spearlist = []
#print "green correlation"
#for column1 in Digb.T:
#	for column2 in Dbb.T:
#		spearlist.append(str(spearmanr(column1, column2, nan_policy='omit')))

#spearlist.sort()
#for s in spearlist:
#	print s

#print "red correlation"
#for column3 in Dirb.T:
#	for column4 in Dbb.T:
#		print str(pearsonr(column3, column4))

#for column1 in Dig:
#	for column2 in Db:
#		print column1.corr

#print "green correlation"

#with pd.option_context('display.max_rows', None, 'display.max_columns', None):
	#print Dig.corrwith(Db.set_axis(Dig.columns, axis='columns', inplace=False))
	#print Dig.corrwith(Db)
#print "red correlation"
#Dir.corrwith(Db)


# -*- coding: utf-8 -*-
# Copyright 2007-2011 The Hyperspy developers
#
# This file is part of  Hyperspy.
#
#  Hyperspy is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
#  Hyperspy is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with  Hyperspy.  If not, see <http://www.gnu.org/licenses/>.

import matplotlib.pyplot as plt

from hyperspy.drawing.marker import MarkerBase

class HorizontalLineSegment(MarkerBase):

    """Horizontal line segment marker that can be added to the signal figure

    Parameters
    ---------
    x1: array or float
        The position of the start of the line segment in x.
        If float, the marker is fixed.
        If array, the marker will be updated when navigating. The array should
        have the same dimensions in the nagivation axes.
    x2: array or float
        The position of the end of the line segment in x.
        see x1 arguments
    y: array or float
        The position of line segment in y.
        see x1 arguments
    kwargs:
        Kewywords argument of axvline valid properties (i.e. recognized by
        mpl.plot).

    Example
    -------
    >>> import numpy as np
    >>> im = signals.Image(np.zeros((100, 100)))
    >>> m = utils.plot.markers.horizontal_line_segment(
    >>>     x1=20, x2=70, y=70, linewidth=4, color='red', linestyle='dotted')
    >>> im.add_marker(m)

    """

    def __init__(self, x1, x2, y, **kwargs):
        MarkerBase.__init__(self)
        lp = {}
        lp['color'] = 'black'
        lp['linewidth'] = 1
        self.marker_properties = lp
        self.set_data(x1=x1, x2=x2, y1=y)
        self.set_marker_properties(**kwargs)

    def update(self):
        if self.auto_update is False:
            return
        self._update_segment()

    def plot(self):
        if self.ax is None:
            raise AttributeError(
                "To use this method the marker needs to be first add to a " +
                "figure using `s._plot.signal_plot.add_marker(m)` or " +
                "`s._plot.navigator_plot.add_marker(m)`")
        self.marker = self.ax.vlines(0, 0, 1, **self.marker_properties)
        self._update_segment()
        self.marker.set_animated(True)
        try:
            self.ax.hspy_fig._draw_animated()
        except:
            pass

    def _update_segment(self):
        segments = self.marker.get_segments()
        segments[0][0, 1] = self.get_data_position('y1')
        segments[0][1, 1] = segments[0][0, 1]
        if self.get_data_position('x1') is None:
            segments[0][0, 0] = plt.getp(self.marker.axes, 'xlim')[0]
        else:
            segments[0][0, 0] = self.get_data_position('x1')
        if self.get_data_position('x2') is None:
            segments[0][1, 0] = plt.getp(self.marker.axes, 'xlim')[1]
        else:
            segments[0][1, 0] = self.get_data_position('x2')
        self.marker.set_segments(segments)


__author__ = 'giacomov'

# !/usr/bin/env python

# add |^| to the top line to run the script without needing 'python' to run it at cmd

# importing modules1
import numpy as np

# cant use 'show' inside the farm
import matplotlib
matplotlib.use("Agg")

import matplotlib.pyplot as plt
from matplotlib import gridspec

import os
import argparse

import decayLikelihood

import warnings

####################################################################

mycmd = argparse.ArgumentParser()  # this is a class
mycmd.add_argument('triggername', help="The name of the GRB in YYMMDDXXX format (ex. bn080916009)")
mycmd.add_argument('redshift', help="Redshift for object.")
mycmd.add_argument('function', help="Function to model. (ex. crystalball2, band)")

mycmd.add_argument('directory', help="Directory containing the file produced by gtburst")

if __name__ == "__main__":

    args = mycmd.parse_args()

    os.chdir(args.directory)

    ##############################################################################

    textfile = os.path.join(args.directory, '%s_res.txt' % (args.triggername))
    tbin = np.recfromtxt(textfile, names=True)

    textfile = os.path.join(args.directory, '%s_MCsamples_%s.txt' % (args.triggername, args.function))
    samples = np.recfromtxt(textfile, names=True)

    # function for returning 1 and 2 sigma errors from sample median
    def getErr(sampleArr):
        # compute sample percentiles for 1 and 2 sigma
        m, c, p = np.percentile(sampleArr, [16, 50, 84])
        # print("%.3f -%.3f +%.3f" %(c,m-c,p-c)) median, minus, plus
        m2, c2, p2 = np.percentile(sampleArr, [3, 50, 97])
        return m, c, p, m2, c2, p2

    # prepare for plotting and LOOP

    t = np.logspace(0, 4, 100)
    t = np.append(t, np.linspace(0, 1, 10))
    t.sort()
    t = np.unique(t)
    print('NUMBER OF times to iterate: %s' % (len(t)))

    x = decayLikelihood.DecayLikelihood()

    if args.function == 'crystalball2':

        crystal = decayLikelihood.CrystalBall2()  # declaring instance of DecayLikelihood using POWER LAW FIT
        x.setDecayFunction(crystal)

        # CrystalBall DiffFlux####################################################

        Peak = np.zeros(samples.shape[0])
        ePeak = np.zeros(samples.shape[0])
        tPeak = np.zeros(samples.shape[0])
        tePeak = np.zeros(samples.shape[0])

        print('ENTERING samples LOOP')

        # mu,sigma,decayIndex, and N
        for i, parameters in enumerate(samples):
            x.decayFunction.setParameters(*parameters)

            # NORMALIZATION IS THE FLUX AT THE PEAK
            pB = parameters[3]  # decay time is independent of scale # (y*.001) # scale =0.001, for all xml files
            fBe = pB / np.e

            # t = (fBe/N)**(-1/a) defined to be 1
            mu = parameters[0]
            tP = mu

            with warnings.catch_warnings():
                warnings.filterwarnings('error')
                try:
                    teP = mu + (fBe / parameters[3]) ** (
                    -1 / parameters[2])  # sometimes 'RuntimeWarning: overflow encountered in double_scalars'

                except Warning:
                    print('RuntimeWarning Raised! mu,sigma,decayIndex,and N:', parameters)

            teP = parameters[0] + (fBe / parameters[3]) ** (-1 / parameters[2])
            Peak[i] = pB
            ePeak[i] = fBe
            # redshift correcting t/(1+z)
            tPeak[i] = tP / (1 + float(args.redshift))  ################################
            tePeak[i] = teP / (1 + float(args.redshift))  ################################

    elif args.function == 'band':

        band = decayLikelihood.DecayBand()  # declaring instance of DecayLikelihood using POWER LAW FIT
        x.setDecayFunction(band)

        Peak = np.zeros(samples.shape[0])
        ePeak = np.zeros(samples.shape[0])  # fractional brightness used in calcuating char-time, but not needed otherwise
        tPeak = np.zeros(samples.shape[0])
        tePeak = np.zeros(samples.shape[0])  # characteristic time

        T05 = np.zeros(samples.shape[0])
        T90 = np.zeros(samples.shape[0])
        T95 = np.zeros(samples.shape[0])

        T25 = np.zeros(samples.shape[0])
        T50 = np.zeros(samples.shape[0])
        T75 = np.zeros(samples.shape[0])

        print('ENTERING samples LOOP')

        # mu,sigma,decayIndex, and N
        for i, parameters in enumerate(samples):
            x.decayFunction.setParameters(*parameters)

            tc = band.getCharacteristicTime()  # get the characteristic time.
            # T50/T90 TAKING TOO LONG (1/4)
            # t90, t05, t95 = band.getTsomething( 90 ) # if the argument is 90, returns the T90 as well as the T05 and the T95. If the argument is 50, returns the T50 as well as the T25 and T75, and so on.
            # t50, t25, t75 = band.getTsomething( 50 )

            tp, fp = band.getPeakTimeAndFlux()  # returns the time of the peak, as well as the peak flux

            tePeak[i] = tc / (1 + float(args.redshift))  ################################
            tPeak[i] = tp / (1 + float(args.redshift))
            Peak[i] = fp
            # T50/T90 TAKING TOO LONG (2/4)
            # T05[i] = t05/(1+float(args.redshift))
            # T90[i] = t90/(1+float(args.redshift))
            # T95[i] = t95/(1+float(args.redshift))
            # T50/T90 TAKING TOO LONG (3/4)
            # T25[i] = t25/(1+float(args.redshift))
            # T50[i] = t50/(1+float(args.redshift))
            # T75[i] = t75/(1+float(args.redshift))

    # Defining sigma bands
    print('ENTERING Percentile LOOP')
    upper = np.zeros(t.shape[0])
    lower = np.zeros(t.shape[0])
    upper2 = np.zeros(t.shape[0])
    lower2 = np.zeros(t.shape[0])
    meas = np.zeros(t.shape[0])
    fluxMatrix = np.zeros([samples.shape[0], t.shape[0]])

    for i, s in enumerate(samples):
        x.decayFunction.setParameters(*s)
        fluxes = map(x.decayFunction.getDifferentialFlux, t)
        fluxMatrix[i, :] = np.array(fluxes)

    for i, tt in enumerate(t):
        allFluxes = fluxMatrix[:, i]
        m, p = np.percentile(allFluxes, [16, 84])
        lower[i] = m
        upper[i] = p
        m2, p2 = np.percentile(allFluxes, [2.5, 97.5])
        lower2[i] = m2
        upper2[i] = p2

    wdir = '%s' % (args.directory)
    # save TXT files instead of .npy
    placeFile = os.path.join(wdir, "%s_tBrightness_%s" % (args.triggername, args.function))
    with open(placeFile, 'w+') as f:
        f.write("Peak tPeak ePeak tePeak\n")
        for i, s in enumerate(Peak):
            f.write("%s %s %s %s\n" % (Peak[i], tPeak[i], ePeak[i], tePeak[i]))
    # CALCULATING T50/T90 TAKES TOO LONG
    # T50/T90 TAKING TOO LONG (4/4)
    # if args.function == 'band':
    #  #compute percentiles for 1 sigma
    #  m90,c90,p90 = np.percentile(T90,[16,50,84])
    #  m50,c50,p50 = np.percentile(T50,[16,50,84])
    #  #compute percentiles for 1 and 2 sigma
    #  #90m,90c,90p,90m2,90c2,90p2 = getErr(T90)
    #  #50m,50c,50p,50m2,50c2,50p2 = getErr(T50)
    #  #print("%.3f -%.3f +%.3f" %(c,m-c,p-c)) median, minus, plus
    #
    #  placeFile=os.path.join(wdir,"%s_t90_t50_%s" % (args.triggername, args.function) )
    #  with open(placeFile,'w+') as f:
    #    f.write("t90 90minus 90plus t50 50minus 50plus\n")
    #    for i,s in enumerate(T90):
    #      f.write("%s %s %s %s %s %s\n" % (m90,m90-c90,p90-c90,c50,m50-c50,p50-c50)) #c,m-c,p-c
    #
    #  placeFile=os.path.join(wdir,"%s_samplesT90_%s" % (args.triggername, args.function) )
    #  with open(placeFile,'w+') as f:
    #    f.write("t90 t05 t95\n")
    #    for i,s in enumerate(T90):
    #      f.write("%s %s %s\n" % (T90[i],T05[i],T95[i]))

    #  placeFile=os.path.join(wdir,"%s_samplesT50_%s" % (args.triggername, args.function) )
    #  with open(placeFile,'w+') as f:
    #    f.write("t50 t25 t25\n")
    #    for i,s in enumerate(T50):
    #      f.write("%s %s %s\n" % (T50[i],T25[i],T75[i]))

    # compute char-time percentiles for 1 and 2 sigma
    m, c, p, m2, c2, p2 = getErr(tePeak)

    # saves txt file
    wkdir = '%s' % (args.directory)

    fileDir = os.path.join(wkdir, '%s_timeRes_%s' % (args.triggername, args.function))
    with open(fileDir, 'w+') as f:
        f.write('%s %s %s\n' % ('median', 'minus', 'plus'))
        f.write('%s %s %s\n' % (c, m - c, p - c))

    # PLOTTING BINS AND SIGMA BAND
    print("PLOTTING...")
    fig = plt.figure()

    # median is your "x"
    # Y is your "y"
    # DY is the array containing the errors
    # DY==0 filters only the zero error

    data = tbin
    # redshift correction /(1+args.redshif)
    median = (data["tstart"] + data["tstop"]) / 2 / (1 + float(args.redshift))
    start = data['tstart'] / (1 + float(args.redshift))  ##
    stop = data['tstop'] / (1 + float(args.redshift))  ##

    y = data["photonFlux"]
    Dy = data["photonFluxError"]

    try:
        y = np.core.defchararray.replace(y, "<", "", count=None)  # runs through array and removes strings
    except:
        print('No Upper-Limits Found in %s.' % (args.triggername))

    try:
        Dy = np.core.defchararray.replace(Dy, "n.a.", "0",
                                          count=None)  ## 0 error is nonphysical, and will be checked for in plotting
    except:
        print('No 0-Error Found in %s.' % (args.triggername))

    bar = 0.5
    color = "blue"

    Y = np.empty(0, dtype=float)  # makes empty 1-D array for float values

    for i in y:
        Y = np.append(Y, float(i))

    DY = np.empty(0, dtype=float)

    for i in Dy:
        DY = np.append(DY, float(i))
    plt.clf()
    if (DY > 0).sum() > 0:  # if sum() gives a non-zero value then there are error values
        plt.errorbar(median[DY > 0], Y[DY > 0],
                     xerr=[median[DY > 0] - start[DY > 0], stop[DY > 0] - median[DY > 0]],
                     yerr=DY[DY > 0], ls='None', marker='o', mfc=color, mec=color, ecolor=color, lw=2, label=None)

    if (DY == 0).sum() > 0:
        plt.errorbar(median[DY == 0], Y[DY == 0],
                     xerr=[median[DY == 0] - start[DY == 0], stop[DY == 0] - median[DY == 0]],
                     yerr=[bar * Y[DY == 0], 0.0 * Y[DY == 0]], lolims=True, ls='None', marker='', mfc=color, mec=color,
                     ecolor=color, lw=2, label=None)

    plt.suptitle('%s photonFlux per Time' % (args.triggername))
    plt.xlabel('Rest Frame Time(s)')
    plt.ylabel('Photon Flux')
    plt.xscale('symlog')
    plt.yscale('log')
    plt.grid(True)

    if args.function == 'crystalball2':
        SCALE = 0.001
    elif args.function == 'band':
        SCALE = 1.0  # 0.1 # shouldn't need a scale anymore for Band function

    ylo = 1e-7  # min(lower2*SCALE)*1e-1 # CANT GET THIS TO WORK YET DYNAMICALLY
    yup = max(upper2 * SCALE) * 10
    plt.ylim([ylo, yup])

    # correcting for redshift t/(1+args.redshift)
    plt.fill_between(t / (1 + float(args.redshift)), lower * SCALE, upper * SCALE, alpha=0.5, color='blue')
    plt.fill_between(t / (1 + float(args.redshift)), lower2 * SCALE, upper2 * SCALE, alpha=0.3, color='green')
    # y = map(x.decayFunction.getDifferentialFlux, t) # maps infinitesimal values of flux at time t to y

    # raw_input("Press ENTER")
    # PowerLaw
    # plt.plot(t,,'o')

    # saves plots
    wdir = '%s' % (args.directory)

    imsave = os.path.join(wdir, '%s_objFit_%s' % (args.triggername, args.function))

    plt.savefig(imsave + '.png')

    # histograms of 1/e and save
    print("Making histograms")
    fig = plt.figure(figsize=(10, 6))
    gs = gridspec.GridSpec(1, 2, width_ratios=[1, 1])

    bins = np.linspace(min(tePeak), np.max(tePeak), 100)

    ax0 = plt.subplot(gs[0])
    ax0.hist(tePeak, bins, normed=True)
    plt.title('1/e (min to medx2)')
    plt.xlabel('1/e time (s)')
    plt.xlim([min(tePeak), np.median(tePeak) * 2])

    ax1 = plt.subplot(gs[1])

    ax1.hist(tePeak, bins, normed=True)
    plt.title('1/e (min to max)')
    plt.xlabel('time (s)')

    plt.tight_layout()

    imsave = os.path.join(wdir, '%s_hist_%s' % (args.triggername, args.function))

    plt.savefig(imsave + '.png')
    print("Finished Potting/Saving!")


#!/usr/bin/python2

'''
This script parses JPEG images of text documents to isolate and save images
of individual characters.  The size of these output images in pixels is
specified by the parameters desired_height and desired_width.

The JPEG images are converted to grey scale using a parameter called
luminance_threshold to distinguish between light and dark pixels.  Lines of
text are found by searching for rows that contain dark pixels, and
characters are found by searching for columns that contain dark pixels. Once
a character is found it is padded with blank rows and columns to obtain the
desired size.  The images are saved using the filenames given in the XML file.
'''

# Set desired output image height and width in pixels
desired_height = 32
desired_width = 32

DEBUG = False

import matplotlib.pyplot as plot
import numpy as np
import operator
import sys
import re
import os

from PIL import Image
from xml.dom import minidom

jpg_list = [ 'characters-0.jpg', 'characters-1.jpg', 'characters-2.jpg',
  'characters-3.jpg', 'characters-4.jpg', 'characters-5.jpg',
  'characters-6.jpg', 'characters-7.jpg', 'characters-8.jpg',
  'characters-9.jpg', 'characters-10.jpg', 'characters-11.jpg',
  'characters-12.jpg', 'characters-13.jpg', 'characters-14.jpg',
  'characters-15.jpg', 'characters-16.jpg', 'characters-17.jpg',
  'characters-18.jpg', 'characters-19.jpg' ]

#jpg_list = [ 'debug_doc.jpg' ]

# Parse XML file for filenames to use when saving each character image
xmldoc = minidom.parse('characters.xml')
#xmldoc = minidom.parse('debug_doc.xml')
filelist = xmldoc.getElementsByTagName('image')
print len(filelist)
#for i in range(145):
  #print filelist[62*i].attributes['file'].value

# this counter gets used to select file names from an xml file
output_files_saved = 0

for jpg in jpg_list:
  print jpg
  im = Image.open(jpg)
  width, length = im.size
  if DEBUG:
    print "image size: ", im.size
    print "image mode: ", im.mode
    print im.size[1],im.size[0]

  # read pixel data from image into a numpy array
  if im.mode == 'L':
    pixels = np.array(list(im.getdata())).reshape(im.size[1],im.size[0])
  elif im.mode == 'RGB':
    pixels = np.array(list(im.convert('L').getdata())).reshape(im.size[1],
      im.size[0])

  #im.show()

  ##############################################################################
  # Removed all logic for determining the value to use to distinguish between
  # light and dark pixels because this is a non-trivial challenge of its own and
  # I want to get to generating a data set for OCR which I can do much faster by
  # choosing the threshold manually.
  ##############################################################################

  luminance_threshold = 100

  ##############################################################################
  # parse document for lines of text
  ##############################################################################

  row = 0
  while row < length:
    # Find the first row of pixels in next line of text by ignoring blank rows
    # of pixels which will have a non-zero product since white pixels have a
    # luminance value of 255
    #row_data = pixels[row * width : row * width + width]
    while (row < length and pixels[row,:].min() > luminance_threshold):
      row += 1
    first_row = row
    if DEBUG:
      print "the first row of pixels in the line of text is ", first_row
    # Find the last row of pixels in this line of text by counting rows with
    # dark pixels. These rows have a product of zero since the luminance value
    # of all dark pixels was set to zero
    while (row < length and pixels[row:row + 2,:].min() < luminance_threshold):
      row += 1
    last_row = row
    #if row < length:
      #last_row = row + 2  # this is a hack for Cochin font Q
    #row += 5  # this is a hack for Cochin font Q
    if DEBUG:
      print "the last row of pixels in the line of text is ", last_row

  ##############################################################################
  # parse line of text for characters
  ##############################################################################

    if first_row < last_row:
      col = 0
      while col < width:
        # find first column of pixels in the next character by ignoring blank
        # cols of pixels
        while col < width and pixels[first_row:last_row,col].min() > luminance_threshold:
          col += 1
        first_col = col
        # find last column of pixels in the next character by counting columns
        # with dark pixels
        while col < width and \
          pixels[first_row:last_row,col:col + 5].min() < luminance_threshold:
          col += 1
        last_col = col

  ##############################################################################
  # remove blank rows from the top and bottom of characters
  ##############################################################################

        if first_col < last_col:
          # remove blank rows from the top of the character
          r = first_row;
          while pixels[r,first_col:last_col].min() > luminance_threshold:
            r = r + 1;
          char_first_row = r;
          # remove blank rows from the bottom of the character
          r = last_row;
          while pixels[r,first_col:last_col].min() > luminance_threshold:
            r = r - 1;
          char_last_row = r + 1;
          if DEBUG:
            # isolate an image of this character
            character = im.crop([first_col, char_first_row, last_col,
              char_last_row])
            print "Character size after whitespace removal", character.size
            print first_col, first_row, last_col, last_row
            #character.show()
          # pad character width out to desired_width
          char_width = last_col - first_col
          if char_width > desired_width:
            print "Character is wider than ", desired_width
          else:
            # add the same number of blank columns to the left and right
            first_col = first_col - (desired_width - char_width) / 2
            last_col = last_col + (desired_width - char_width) / 2
            # if the difference was odd we'll be short one column
            char_width = last_col - first_col
            if char_width < desired_width:
              last_col = last_col + 1
          # pad character height out to desired_height
          char_height = char_last_row - char_first_row
          if char_height > desired_height:
            print "Character is taller than ", desired_height
          else:
            # add the same number of blank rows to the left and right
            char_first_row = char_first_row - (desired_height - char_height) / 2
            char_last_row = char_last_row + (desired_height - char_height) / 2
            # if the difference was odd we'll be short one row
            char_height = char_last_row - char_first_row
            if char_height < desired_height:
              char_last_row = char_last_row + 1
          character = im.crop([first_col, char_first_row, last_col,
            char_last_row])
          if DEBUG:
            print "Character size after padding", character.size
            print first_col, char_first_row, last_col, char_last_row
            #character.show()
            #garbage = raw_input()
          # save image to filename specified in ground truth file
          filename = filelist[output_files_saved].attributes['file'].value
          directory = filename.split('/')[0]
          if not os.path.exists(directory):
            os.makedirs(directory)
          character.save(filename, "JPEG", quality=80)
          output_files_saved = output_files_saved + 1

print output_files_saved


# Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Graph processing script."""

import os

from absl import app
from absl import flags
from absl import logging
import networkx as nx
import pandas as pd

from utils import graph_filter_with_degree
from utils import load_graph_from_edgelist_csv

FLAGS = flags.FLAGS
flags.DEFINE_string(
    'g_file',
    '../proj_Data/cat_data/test3/sr_timespan_post_graph-00000-of-00001.csv',
    'raw graph edgelist csv file')
flags.DEFINE_integer('low', 40, 'low degree threshold')
flags.DEFINE_integer('high', 80, 'high degree threshold')
flags.DEFINE_string('data_file', '', 'raw data path')
flags.DEFINE_string('filename', '', 'graph filename')
flags.DEFINE_string('save_path', '', 'graph save path')

def main(_):
  df = pd.read_csv(FLAGS.data_file)
  author_set = set(df['author'].unique())

  graph = load_graph_from_edgelist_csv(FLAGS.g_file)
  logging.info('Original Graph size: %d nodes, %d edges',
               graph.number_of_nodes(), graph.number_of_edges())
  graph = graph_filter_with_degree(graph, FLAGS.low, FLAGS.high, author_set)
  logging.info('Filtered Graph size: %d nodes, %d edges',
               graph.number_of_nodes(), graph.number_of_edges())
  nx.write_gpickle(graph, os.path.join(
      FLAGS.save_path, FLAGS.filename + '%s_%s.gpickle' %
      (FLAGS.low, FLAGS.high)))
  logging.info('Saved graph.')

if __name__ == '__main__':
  app.run(main)