#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

from distutils.version import LooseVersion

import numpy as np
import pandas as pd

try:
    from pandas._testing import makeMissingDataframe
except ImportError:
    from pandas.util.testing import makeMissingDataframe

from pyspark import pandas as ps
from pyspark.pandas.config import option_context
from pyspark.testing.pandasutils import PandasOnSparkTestCase, SPARK_CONF_ARROW_ENABLED
from pyspark.testing.sqlutils import SQLTestUtils

class StatsTest(PandasOnSparkTestCase, SQLTestUtils):
    def _test_stat_functions(self, pdf_or_pser, psdf_or_psser):
        functions = ["max", "min", "mean", "sum", "count"]
        for funcname in functions:
            self.assert_eq(getattr(psdf_or_psser, funcname)(), getattr(pdf_or_pser, funcname)())

        functions = ["std", "var", "product", "sem"]
        for funcname in functions:
            self.assert_eq(
                getattr(psdf_or_psser, funcname)(),
                getattr(pdf_or_pser, funcname)(),
                check_exact=False,
            )

        functions = ["std", "var", "sem"]
        for funcname in functions:
            self.assert_eq(
                getattr(psdf_or_psser, funcname)(ddof=0),
                getattr(pdf_or_pser, funcname)(ddof=0),
                check_exact=False,
            )

        # NOTE: To test skew, kurt, and median, just make sure they run.
        #       The numbers are different in spark and pandas.
        functions = ["skew", "kurt", "median"]
        for funcname in functions:
            getattr(psdf_or_psser, funcname)()

    def test_stat_functions(self):
        pdf = pd.DataFrame({"A": [1, 2, 3, 4], "B": [1, 2, 3, 4], "C": [1, np.nan, 3, np.nan]})
        psdf = ps.from_pandas(pdf)
        self._test_stat_functions(pdf.A, psdf.A)
        self._test_stat_functions(pdf, psdf)

        # empty
        self._test_stat_functions(pdf.A.loc[[]], psdf.A.loc[[]])
        self._test_stat_functions(pdf.loc[[]], psdf.loc[[]])

    def test_stat_functions_multiindex_column(self):
        arrays = [np.array(["A", "A", "B", "B"]), np.array(["one", "two", "one", "two"])]
        pdf = pd.DataFrame(np.random.randn(3, 4), index=["A", "B", "C"], columns=arrays)
        psdf = ps.from_pandas(pdf)
        self._test_stat_functions(pdf.A, psdf.A)
        self._test_stat_functions(pdf, psdf)

    def test_stat_functions_with_no_numeric_columns(self):
        pdf = pd.DataFrame(
            {
                "A": ["a", None, "c", "d", None, "f", "g"],
                "B": ["A", "B", "C", None, "E", "F", None],
            }
        )
        psdf = ps.from_pandas(pdf)

        self._test_stat_functions(pdf, psdf)

    def test_sum(self):
        pdf = pd.DataFrame({"a": [1, 2, 3, np.nan], "b": [0.1, np.nan, 0.3, np.nan]})
        psdf = ps.from_pandas(pdf)

        self.assert_eq(psdf.sum(), pdf.sum())
        self.assert_eq(psdf.sum(axis=1), pdf.sum(axis=1))
        self.assert_eq(psdf.sum(min_count=3), pdf.sum(min_count=3))
        self.assert_eq(psdf.sum(axis=1, min_count=1), pdf.sum(axis=1, min_count=1))
        self.assert_eq(psdf.loc[[]].sum(), pdf.loc[[]].sum())
        self.assert_eq(psdf.loc[[]].sum(min_count=1), pdf.loc[[]].sum(min_count=1))

        self.assert_eq(psdf["a"].sum(), pdf["a"].sum())
        self.assert_eq(psdf["a"].sum(min_count=3), pdf["a"].sum(min_count=3))
        self.assert_eq(psdf["b"].sum(min_count=3), pdf["b"].sum(min_count=3))
        self.assert_eq(psdf["a"].loc[[]].sum(), pdf["a"].loc[[]].sum())
        self.assert_eq(psdf["a"].loc[[]].sum(min_count=1), pdf["a"].loc[[]].sum(min_count=1))

    def test_product(self):
        pdf = pd.DataFrame(
            {"a": [1, -2, -3, np.nan], "b": [0.1, np.nan, -0.3, np.nan], "c": [10, 20, 0, -10]}
        )
        psdf = ps.from_pandas(pdf)

        self.assert_eq(psdf.product(), pdf.product(), check_exact=False)
        self.assert_eq(psdf.product(axis=1), pdf.product(axis=1))
        self.assert_eq(psdf.product(min_count=3), pdf.product(min_count=3), check_exact=False)
        self.assert_eq(psdf.product(axis=1, min_count=1), pdf.product(axis=1, min_count=1))
        self.assert_eq(psdf.loc[[]].product(), pdf.loc[[]].product())
        self.assert_eq(psdf.loc[[]].product(min_count=1), pdf.loc[[]].product(min_count=1))

        self.assert_eq(psdf["a"].product(), pdf["a"].product(), check_exact=False)
        self.assert_eq(
            psdf["a"].product(min_count=3), pdf["a"].product(min_count=3), check_exact=False
        )
        self.assert_eq(psdf["b"].product(min_count=3), pdf["b"].product(min_count=3))
        self.assert_eq(psdf["c"].product(min_count=3), pdf["c"].product(min_count=3))
        self.assert_eq(psdf["a"].loc[[]].product(), pdf["a"].loc[[]].product())
        self.assert_eq(
            psdf["a"].loc[[]].product(min_count=1), pdf["a"].loc[[]].product(min_count=1)
        )

    def test_abs(self):
        pdf = pd.DataFrame(
            {
                "A": [1, -2, np.nan, -4, 5],
                "B": [1.0, -2, np.nan, -4, 5],
                "C": [-6.0, -7, -8, np.nan, 10],
                "D": ["a", "b", "c", "d", np.nan],
                "E": [True, np.nan, False, True, True],
            }
        )
        psdf = ps.from_pandas(pdf)
        self.assert_eq(psdf.A.abs(), pdf.A.abs())
        self.assert_eq(psdf.B.abs(), pdf.B.abs())
        self.assert_eq(psdf.E.abs(), pdf.E.abs())
        # pandas' bug?
        # self.assert_eq(psdf[["B", "C", "E"]].abs(), pdf[["B", "C", "E"]].abs())
        self.assert_eq(psdf[["B", "C"]].abs(), pdf[["B", "C"]].abs())
        self.assert_eq(psdf[["E"]].abs(), pdf[["E"]].abs())

        with self.assertRaisesRegex(
            TypeError, "bad operand type for abs\\(\\): object \\(string\\)"
        ):
            psdf.abs()
        with self.assertRaisesRegex(
            TypeError, "bad operand type for abs\\(\\): object \\(string\\)"
        ):
            psdf.D.abs()

    def test_axis_on_dataframe(self):
        # The number of each count is intentionally big
        # because when data is small, it executes a shortcut.
        # Less than 'compute.shortcut_limit' will execute a shortcut
        # by using collected pandas dataframe directly.
        # now we set the 'compute.shortcut_limit' as 1000 explicitly
        with option_context("compute.shortcut_limit", 1000):
            pdf = pd.DataFrame(
                {
                    "A": [1, -2, 3, -4, 5] * 300,
                    "B": [1.0, -2, 3, -4, 5] * 300,
                    "C": [-6.0, -7, -8, -9, 10] * 300,
                    "D": [True, False, True, False, False] * 300,
                },
                index=range(10, 15001, 10),
            )
            psdf = ps.from_pandas(pdf)
            self.assert_eq(psdf.count(axis=1), pdf.count(axis=1))
            self.assert_eq(psdf.var(axis=1), pdf.var(axis=1))
            self.assert_eq(psdf.var(axis=1, ddof=0), pdf.var(axis=1, ddof=0))
            self.assert_eq(psdf.std(axis=1), pdf.std(axis=1))
            self.assert_eq(psdf.std(axis=1, ddof=0), pdf.std(axis=1, ddof=0))
            self.assert_eq(psdf.max(axis=1), pdf.max(axis=1))
            self.assert_eq(psdf.min(axis=1), pdf.min(axis=1))
            self.assert_eq(psdf.sum(axis=1), pdf.sum(axis=1))
            self.assert_eq(psdf.product(axis=1), pdf.product(axis=1))
            self.assert_eq(psdf.kurtosis(axis=1), pdf.kurtosis(axis=1))
            self.assert_eq(psdf.skew(axis=1), pdf.skew(axis=1))
            self.assert_eq(psdf.mean(axis=1), pdf.mean(axis=1))
            self.assert_eq(psdf.sem(axis=1), pdf.sem(axis=1))
            self.assert_eq(psdf.sem(axis=1, ddof=0), pdf.sem(axis=1, ddof=0))

            self.assert_eq(
                psdf.count(axis=1, numeric_only=True), pdf.count(axis=1, numeric_only=True)
            )
            self.assert_eq(psdf.var(axis=1, numeric_only=True), pdf.var(axis=1, numeric_only=True))
            self.assert_eq(
                psdf.var(axis=1, ddof=0, numeric_only=True),
                pdf.var(axis=1, ddof=0, numeric_only=True),
            )
            self.assert_eq(psdf.std(axis=1, numeric_only=True), pdf.std(axis=1, numeric_only=True))
            self.assert_eq(
                psdf.std(axis=1, ddof=0, numeric_only=True),
                pdf.std(axis=1, ddof=0, numeric_only=True),
            )
            self.assert_eq(
                psdf.max(axis=1, numeric_only=True),
                pdf.max(axis=1, numeric_only=True).astype(float),
            )
            self.assert_eq(
                psdf.min(axis=1, numeric_only=True),
                pdf.min(axis=1, numeric_only=True).astype(float),
            )
            self.assert_eq(
                psdf.sum(axis=1, numeric_only=True),
                pdf.sum(axis=1, numeric_only=True).astype(float),
            )
            self.assert_eq(
                psdf.product(axis=1, numeric_only=True),
                pdf.product(axis=1, numeric_only=True).astype(float),
            )
            self.assert_eq(
                psdf.kurtosis(axis=1, numeric_only=True), pdf.kurtosis(axis=1, numeric_only=True)
            )
            self.assert_eq(
                psdf.skew(axis=1, numeric_only=True), pdf.skew(axis=1, numeric_only=True)
            )
            self.assert_eq(
                psdf.mean(axis=1, numeric_only=True), pdf.mean(axis=1, numeric_only=True)
            )
            self.assert_eq(psdf.sem(axis=1, numeric_only=True), pdf.sem(axis=1, numeric_only=True))
            self.assert_eq(
                psdf.sem(axis=1, ddof=0, numeric_only=True),
                pdf.sem(axis=1, ddof=0, numeric_only=True),
            )

    def test_corr(self):
        # Disable arrow execution since corr() is using UDT internally which is not supported.
        with self.sql_conf({SPARK_CONF_ARROW_ENABLED: False}):
            # DataFrame
            # we do not handle NaNs for now
            pdf = makeMissingDataframe(0.3, 42).fillna(0)
            psdf = ps.from_pandas(pdf)

            self.assert_eq(psdf.corr(), pdf.corr(), check_exact=False)

            # Series
            pser_a = pdf.A
            pser_b = pdf.B
            psser_a = psdf.A
            psser_b = psdf.B

            self.assertAlmostEqual(psser_a.corr(psser_b), pser_a.corr(pser_b))
            self.assertRaises(TypeError, lambda: psser_a.corr(psdf))

            # multi-index columns
            columns = pd.MultiIndex.from_tuples([("X", "A"), ("X", "B"), ("Y", "C"), ("Z", "D")])
            pdf.columns = columns
            psdf.columns = columns

            self.assert_eq(psdf.corr(), pdf.corr(), check_exact=False)

            # Series
            pser_xa = pdf[("X", "A")]
            pser_xb = pdf[("X", "B")]
            psser_xa = psdf[("X", "A")]
            psser_xb = psdf[("X", "B")]

            self.assert_eq(psser_xa.corr(psser_xb), pser_xa.corr(pser_xb), almost=True)

    def test_cov_corr_meta(self):
        # Disable arrow execution since corr() is using UDT internally which is not supported.
        with self.sql_conf({SPARK_CONF_ARROW_ENABLED: False}):
            pdf = pd.DataFrame(
                {
                    "a": np.array([1, 2, 3], dtype="i1"),
                    "b": np.array([1, 2, 3], dtype="i2"),
                    "c": np.array([1, 2, 3], dtype="i4"),
                    "d": np.array([1, 2, 3]),
                    "e": np.array([1.0, 2.0, 3.0], dtype="f4"),
                    "f": np.array([1.0, 2.0, 3.0]),
                    "g": np.array([True, False, True]),
                    "h": np.array(list("abc")),
                },
                index=pd.Index([1, 2, 3], name="myindex"),
            )
            psdf = ps.from_pandas(pdf)
            self.assert_eq(psdf.corr(), pdf.corr())

    def test_stats_on_boolean_dataframe(self):
        pdf = pd.DataFrame({"A": [True, False, True], "B": [False, False, True]})
        psdf = ps.from_pandas(pdf)

        self.assert_eq(psdf.min(), pdf.min())
        self.assert_eq(psdf.max(), pdf.max())
        self.assert_eq(psdf.count(), pdf.count())

        self.assert_eq(psdf.sum(), pdf.sum())
        self.assert_eq(psdf.product(), pdf.product())
        self.assert_eq(psdf.mean(), pdf.mean())

        self.assert_eq(psdf.var(), pdf.var(), check_exact=False)
        self.assert_eq(psdf.var(ddof=0), pdf.var(ddof=0), check_exact=False)
        self.assert_eq(psdf.std(), pdf.std(), check_exact=False)
        self.assert_eq(psdf.std(ddof=0), pdf.std(ddof=0), check_exact=False)
        self.assert_eq(psdf.sem(), pdf.sem(), check_exact=False)
        self.assert_eq(psdf.sem(ddof=0), pdf.sem(ddof=0), check_exact=False)

    def test_stats_on_boolean_series(self):
        pser = pd.Series([True, False, True])
        psser = ps.from_pandas(pser)

        self.assert_eq(psser.min(), pser.min())
        self.assert_eq(psser.max(), pser.max())
        self.assert_eq(psser.count(), pser.count())

        self.assert_eq(psser.sum(), pser.sum())
        self.assert_eq(psser.product(), pser.product())
        self.assert_eq(psser.mean(), pser.mean())

        self.assert_eq(psser.var(), pser.var(), almost=True)
        self.assert_eq(psser.var(ddof=0), pser.var(ddof=0), almost=True)
        self.assert_eq(psser.std(), pser.std(), almost=True)
        self.assert_eq(psser.std(ddof=0), pser.std(ddof=0), almost=True)
        self.assert_eq(psser.sem(), pser.sem(), almost=True)
        self.assert_eq(psser.sem(ddof=0), pser.sem(ddof=0), almost=True)

    def test_stats_on_non_numeric_columns_should_be_discarded_if_numeric_only_is_true(self):
        pdf = pd.DataFrame({"i": [0, 1, 2], "b": [False, False, True], "s": ["x", "y", "z"]})
        psdf = ps.from_pandas(pdf)

        self.assert_eq(
            psdf[["i", "s"]].max(numeric_only=True), pdf[["i", "s"]].max(numeric_only=True)
        )
        self.assert_eq(
            psdf[["b", "s"]].max(numeric_only=True), pdf[["b", "s"]].max(numeric_only=True)
        )
        self.assert_eq(
            psdf[["i", "s"]].min(numeric_only=True), pdf[["i", "s"]].min(numeric_only=True)
        )
        self.assert_eq(
            psdf[["b", "s"]].min(numeric_only=True), pdf[["b", "s"]].min(numeric_only=True)
        )
        self.assert_eq(psdf.count(numeric_only=True), pdf.count(numeric_only=True))

        if LooseVersion(pd.__version__) >= LooseVersion("1.0.0"):
            self.assert_eq(psdf.sum(numeric_only=True), pdf.sum(numeric_only=True))
            self.assert_eq(psdf.product(numeric_only=True), pdf.product(numeric_only=True))
        else:
            self.assert_eq(psdf.sum(numeric_only=True), pdf.sum(numeric_only=True).astype(int))
            self.assert_eq(
                psdf.product(numeric_only=True), pdf.product(numeric_only=True).astype(int)
            )

        self.assert_eq(psdf.mean(numeric_only=True), pdf.mean(numeric_only=True))

        self.assert_eq(psdf.var(numeric_only=True), pdf.var(numeric_only=True), check_exact=False)
        self.assert_eq(
            psdf.var(ddof=0, numeric_only=True),
            pdf.var(ddof=0, numeric_only=True),
            check_exact=False,
        )
        self.assert_eq(psdf.std(numeric_only=True), pdf.std(numeric_only=True), check_exact=False)
        self.assert_eq(
            psdf.std(ddof=0, numeric_only=True),
            pdf.std(ddof=0, numeric_only=True),
            check_exact=False,
        )
        self.assert_eq(psdf.sem(numeric_only=True), pdf.sem(numeric_only=True), check_exact=False)
        self.assert_eq(
            psdf.sem(ddof=0, numeric_only=True),
            pdf.sem(ddof=0, numeric_only=True),
            check_exact=False,
        )

        self.assert_eq(len(psdf.median(numeric_only=True)), len(pdf.median(numeric_only=True)))
        self.assert_eq(len(psdf.kurtosis(numeric_only=True)), len(pdf.kurtosis(numeric_only=True)))
        self.assert_eq(len(psdf.skew(numeric_only=True)), len(pdf.skew(numeric_only=True)))

        # Boolean was excluded because of a behavior change in NumPy
        # https://github.com/numpy/numpy/pull/16273#discussion_r641264085 which pandas inherits
        # but this behavior is inconsistent in pandas context.
        # Boolean column in quantile tests are excluded for now.
        # TODO(SPARK-35555): track and match the behavior of quantile to pandas'
        pdf = pd.DataFrame({"i": [0, 1, 2], "s": ["x", "y", "z"]})
        psdf = ps.from_pandas(pdf)
        self.assert_eq(
            len(psdf.quantile(q=0.5, numeric_only=True)),
            len(pdf.quantile(q=0.5, numeric_only=True)),
        )
        self.assert_eq(
            len(psdf.quantile(q=[0.25, 0.5, 0.75], numeric_only=True)),
            len(pdf.quantile(q=[0.25, 0.5, 0.75], numeric_only=True)),
        )

    def test_numeric_only_unsupported(self):
        pdf = pd.DataFrame({"i": [0, 1, 2], "b": [False, False, True], "s": ["x", "y", "z"]})
        psdf = ps.from_pandas(pdf)

        if LooseVersion(pd.__version__) >= LooseVersion("1.0.0"):
            self.assert_eq(psdf.sum(numeric_only=True), pdf.sum(numeric_only=True))
            self.assert_eq(
                psdf[["i", "b"]].sum(numeric_only=False), pdf[["i", "b"]].sum(numeric_only=False)
            )
        else:
            self.assert_eq(psdf.sum(numeric_only=True), pdf.sum(numeric_only=True).astype(int))
            self.assert_eq(
                psdf[["i", "b"]].sum(numeric_only=False),
                pdf[["i", "b"]].sum(numeric_only=False).astype(int),
            )

        with self.assertRaisesRegex(TypeError, "Could not convert object \\(string\\) to numeric"):
            psdf.sum(numeric_only=False)

        with self.assertRaisesRegex(TypeError, "Could not convert object \\(string\\) to numeric"):
            psdf.s.sum()

if __name__ == "__main__":
    import unittest
    from pyspark.pandas.tests.test_stats import *  # noqa: F401

    try:
        import xmlrunner  # type: ignore[import]

        testRunner = xmlrunner.XMLTestRunner(output="target/test-reports", verbosity=2)
    except ImportError:
        testRunner = None
    unittest.main(testRunner=testRunner, verbosity=2)


#! /usr/bin/env python

from datetime import datetime, timedelta
import hb_config
import json
import pandas as pd
import requests
from requests_oauthlib import OAuth1
from urllib import parse

rt_header = """== Popular articles {date7} to {date1} ==
Last updated on ~~~~~

{{| class="wikitable sortable"
!Rank
!Article
!Total weekly views
!Days in top 1k this week
"""

footer = """|}

<!--IMPORTANT add all categories to the top section of the page, not here. Otherwise, they will get removed when the bot runs tomorrow! -->

"""

rt_row = """|-
|{rank}
|[[w:{title}|{title}]]
|{week_total}
|{days_in_topk}
"""

def get_yesterdates(lookback=7):
    """
    Accepts a lookback parameter of how many days ago to gather data for (not including the current day per UTC time)
    Defaults to seven days lookback (val must be at least 1)
    Returns a list of dictionaries with the previous n dates (exclusive), in reverse chronological order
    """

    date_range = []

    for d in range(1, lookback + 1):
        date_parts = {'year': datetime.strftime(datetime.now() - timedelta(d), '%Y'),
           'month' : datetime.strftime(datetime.now() - timedelta(d), '%m'),
           'day': datetime.strftime(datetime.now() - timedelta(d), '%d'),
            }

        date_parts['display_date'] = date_parts['year'] + "-" + date_parts['month'] + "-" + date_parts['day']

        date_parts['api_date'] = date_parts['year'] + date_parts['month'] + date_parts['day'] + "00"

        date_range.append(date_parts)

    return date_range

def get_all_topk_articles(day_range):
    """
    Accepts a list of dicts with year, month, and day values
    Returns a dictionary (article titles as keys)
        with all articles that were in the topk list during those dates
        and the pageview counts for each of the dates the article appeared in the topk

    Example query: https://wikimedia.org/api/rest_v1/metrics/pageviews/top/en.wikipedia.org/all-access/2020/03/31
    """

    q_template= "https://wikimedia.org/api/rest_v1/metrics/pageviews/top/en.wikipedia.org/all-access/{year}/{month}/{day}"

    all_articles = {}

    for day_val in day_range:
        q_string = q_template.format(**day_val)
        r = requests.get(
            url = q_string,
            headers = {'User-Agent': "hostbot (https://wikitech.wikimedia.org/wiki/Tool:HostBot, jonnymorgan.esq@gmail.com)"},
                )
    #     print(r.headers)
    #     print(r.text)
    #     print(r.url)
        response = r.json()
    #     print(response)

#         response = requests.get(q_string).json()
        top_articles_list = response['items'][0]['articles']

        for ar in top_articles_list:
            if ar['article'] in all_articles.keys():
                all_articles[ar['article']].update({day_val['api_date'] : ar['views']})

            else:
                all_articles.update({ar['article'] : {day_val['api_date'] : ar['views']}})

    return all_articles

def ar_days_in_topk(day_range, ar_dict):
    """
    Accepts a day range dictionary
        And a nested dict with articles as keys
        And as values varying numbers of k,v pairs
    Returns the article dictionary with a new k,v pair value
        that counts the number of existing k,v pairs in that article dict
    """

    for k,v in ar_dict.items():
        v['topk_days'] = len(ar_dict[k])

    return ar_dict

def get_daily_non_topk_counts(day_range, ar_dict):
    """
    Accepts a list of dicts with year, month, and day values
        And a dict with article titles as keys dicts with different numbers of k,v pairs as values
    Returns a dict that contains for each article the difference between the length of the day range
        And the number of keys in each dict
    Example query:         https://wikimedia.org/api/rest_v1/metrics/pageviews/per-article/en.wikipedia.org/all-access/user/2009_swine_flu_pandemic/daily/2020032500/2020033100
    """

    q_template= "https://wikimedia.org/api/rest_v1/metrics/pageviews/per-article/en.wikipedia.org/all-access/user/{article}/daily/{day7}/{day1}"

    for k,v in ar_dict.items():
        if len(v) < 8: #if this article didn't spend all week among the top 1000

            safe_title = parse.quote(k, safe='') #in case there are weird chars in title
            q_string = q_template.format(article = safe_title, day7 = day_range[6]['api_date'], day1 = day_range[0]['api_date'])
#             print(q_string)
            r = requests.get(
                url = q_string,
                headers = {'User-Agent': "hostbot (https://wikitech.wikimedia.org/wiki/Tool:HostBot, jonnymorgan.esq@gmail.com)"},
                    )
        #     print(r.headers)
        #     print(r.text)
        #     print(r.url)
            response = r.json()
        #     print(response)
#             response = requests.get(q_string).json()

            ar_views = response['items']
    #         print(ar_views)

            for d in ar_views:
                if d['timestamp'] not in v.keys():
                    v.update({d['timestamp'] : d['views']})
        else:
            pass

    return ar_dict

def fill_null_date_vals(day_range, ar_dict):
    """
    Accepts a list of dicts with year, month, and day values
        And a dict with article titles as keys and gaps in the date keys
    Returns the article dictionary with each sub-dict fully populated
        With pageview values for all dates in range, even if val is 0
    """
    #https://www.geeksforgeeks.org/dictionary-methods-in-python-set-2-update-has_key-fromkeys/
    for day_val in week_of_days:
        for v in ar_dict.values():
            if len(v) < 8: #if we still don't have any pageviews for some days
                v.setdefault(day_val['api_date'], 0) #adds a key with val of 0 if no key present
            else:
                pass

    return ar_dict

def format_row(rank, title, week_total, days_in_topk, row_template):

    table_row = {'rank': rank,
           'title' : title.replace("_"," "),
            'week_total' : week_total,
            'days_in_topk' : days_in_topk
                }

    row = row_template.format(**table_row)
#     print(row)
    return(row)

def get_token(auth1):
    """
    Accepts an auth object for a user
    Returns an edit token for the specified wiki
    """

    result = requests.get(
        url="https://en.wikipedia.org/w/api.php", #TODO add to config
        params={
            'action': "query",
            'meta': "tokens",
            'type': "csrf",
            'format': "json"
            },
        headers={'User-Agent': "hostbot (https://wikitech.wikimedia.org/wiki/Tool:HostBot, jonnymorgan.esq@gmail.com)"}, #TODO add to config
        auth=auth1,
        ).json()

#     print(result)
    edit_token = result['query']['tokens']['csrftoken']
#     print(edit_token)

    return(edit_token)

def publish_report(output, edit_sum, auth1, edit_token):
    """
    Accepts the page text, credentials and edit token
    Publishes the formatted page text to the specified wiki
    """
    response = requests.post(
    url = "https://en.wikipedia.org/w/api.php", #TODO add to config
    data={
        'action': "edit",
        'title': "User:HostBot/Top_1000_report", #TODO add to config
        'section': "1",
        'summary': edit_sum, #TODO add to config
        'text': output,
        'bot': 1,
        'token': edit_token,
        'format': "json"
        },
    headers={'User-Agent': "hostbot (https://wikitech.wikimedia.org/wiki/Tool:HostBot, jonnymorgan.esq@gmail.com)"}, #TODO add to config
    auth=auth1
        )

if __name__ == "__main__":

    auth1 = OAuth1("b5d87cbe96174f9435689a666110159c",
               hb_config.client_secret,
               "ca1b222d687be9ac33cfb49676f5bfd2",
               hb_config.resource_owner_secret)

    #get previous week's date info for query and reporting
    week_of_days = get_yesterdates(lookback=7)

    #get all of the articles that appeared on the topk list that week
    all_articles = get_all_topk_articles(week_of_days)

    #get counts for all days each article was in the top 1000
#     all_articles = get_daily_topk_counts(week_of_days, all_articles)

    #add number of days each article appears in the topk list. could do this in first function too
    all_articles = ar_days_in_topk(len(week_of_days), all_articles)

    #add page counts for days the article was not in the topk list
    all_articles = get_daily_non_topk_counts(week_of_days, all_articles)

    all_articles = fill_null_date_vals(week_of_days, all_articles)

    #now we're ready to make a dataframe!
    df_aa = pd.DataFrame.from_dict(all_articles, orient="index")

    #sum across the daily counts
    #https://stackoverflow.com/questions/25748683/pandas-sum-dataframe-rows-for-given-columns
    df_aa['week_total'] = df_aa.sum(axis=1)

    #make the title NOT the index. Should do this when creating the frame, instead
    df_aa.reset_index(inplace=True)

    #rename title column. Should do this when creating the frame, instead
    df_aa.rename(columns = {'index' : 'title'}, inplace=True)

    #remove blacklisted titles--pages we don't care about, for these purposes. Although... we could keep them I guess.
    blacklist = ["Main_Page", "Special:", "Category:", "Portal:", "Template:", "Wikipedia:", "Talk:", "User:", "_talk:", "Help:", "File:", "United_States_Senate",]
    df_aa = df_aa[~df_aa['title'].str.contains('|'.join(blacklist))]

    #sort by weekly views
    df_aa.sort_values('week_total', ascending=False, inplace=True)

    #add rank column based on weekly views
    new_rank = range(1, len(df_aa)+1)
    df_aa['rank'] = list(new_rank)

    #reset the index to reflect the final ranking, dropping the existing index this time
    df_aa.reset_index(drop=True, inplace=True)

    #start and end dates for header and edit comment
    header_dates = {'date1' : week_of_days[0]['display_date'],
                    'date7' : week_of_days[6]['display_date']
                        }

    #format the header template
    header = rt_header.format(**header_dates)

    report_rows = [format_row(a, b, c, d, rt_row) #this is messy
      for a, b, c, d
      in zip(df_aa['rank'],
             df_aa['title'],
             df_aa['week_total'],
             df_aa['topk_days'],
                )]

    rows_wiki = ''.join(report_rows)

    output = header + rows_wiki + footer
#     print(output)

    edit_token = get_token(auth1)

    edit_sum = "Popular articles from {date7} to {date1}".format(**header_dates)

    publish_report(output, edit_sum, auth1, edit_token)


"""
=======================================
Clustering text documents using k-means
=======================================

This is an example showing how the scikit-learn can be used to cluster
documents by topics using a bag-of-words approach. This example uses
a scipy.sparse matrix to store the features instead of standard numpy arrays.

Two feature extraction methods can be used in this example:

  - TfidfVectorizer uses a in-memory vocabulary (a python dict) to map the most
    frequent words to features indices and hence compute a word occurrence
    frequency (sparse) matrix. The word frequencies are then reweighted using
    the Inverse Document Frequency (IDF) vector collected feature-wise over
    the corpus.

  - HashingVectorizer hashes word occurrences to a fixed dimensional space,
    possibly with collisions. The word count vectors are then normalized to
    each have l2-norm equal to one (projected to the euclidean unit-ball) which
    seems to be important for k-means to work in high dimensional space.

    HashingVectorizer does not provide IDF weighting as this is a stateless
    model (the fit method does nothing). When IDF weighting is needed it can
    be added by pipelining its output to a TfidfTransformer instance.

Two algorithms are demoed: ordinary k-means and its more scalable cousin
minibatch k-means.

Additionally, latent sematic analysis can also be used to reduce dimensionality
and discover latent patterns in the data.

It can be noted that k-means (and minibatch k-means) are very sensitive to
feature scaling and that in this case the IDF weighting helps improve the
quality of the clustering by quite a lot as measured against the "ground truth"
provided by the class label assignments of the 20 newsgroups dataset.

This improvement is not visible in the Silhouette Coefficient which is small
for both as this measure seem to suffer from the phenomenon called
"Concentration of Measure" or "Curse of Dimensionality" for high dimensional
datasets such as text data. Other measures such as V-measure and Adjusted Rand
Index are information theoretic based evaluation scores: as they are only based
on cluster assignments rather than distances, hence not affected by the curse
of dimensionality.

Note: as k-means is optimizing a non-convex objective function, it will likely
end up in a local optimum. Several runs with independent random init might be
necessary to get a good convergence.

"""

# Author: Peter Prettenhofer <peter.prettenhofer@gmail.com>
#         Lars Buitinck <L.J.Buitinck@uva.nl>
# License: BSD 3 clause

from __future__ import print_function

from sklearn.datasets import fetch_20newsgroups
from sklearn.decomposition import TruncatedSVD
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.feature_extraction.text import HashingVectorizer
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import Normalizer
from sklearn import metrics

from sklearn.cluster import KMeans, MiniBatchKMeans

import logging
from optparse import OptionParser
import sys
from time import time

import numpy as np

# Display progress logs on stdout
logging.basicConfig(level=logging.INFO,
                    format='%(asctime)s %(levelname)s %(message)s')

# parse commandline arguments
op = OptionParser()
op.add_option("--lsa",
              dest="n_components", type="int",
              help="Preprocess documents with latent semantic analysis.")
op.add_option("--no-minibatch",
              action="store_false", dest="minibatch", default=True,
              help="Use ordinary k-means algorithm (in batch mode).")
op.add_option("--no-idf",
              action="store_false", dest="use_idf", default=True,
              help="Disable Inverse Document Frequency feature weighting.")
op.add_option("--use-hashing",
              action="store_true", default=False,
              help="Use a hashing feature vectorizer")
op.add_option("--n-features", type=int, default=10000,
              help="Maximum number of features (dimensions)"
                   " to extract from text.")
op.add_option("--verbose",
              action="store_true", dest="verbose", default=False,
              help="Print progress reports inside k-means algorithm.")

print(__doc__)
op.print_help()

(opts, args) = op.parse_args()
if len(args) > 0:
    op.error("this script takes no arguments.")
    sys.exit(1)

###############################################################################
# Load some categories from the training set
categories = [
    'alt.atheism',
    'talk.religion.misc',
    'comp.graphics',
    'sci.space',
]
# Uncomment the following to do the analysis on all the categories
#categories = None

print("Loading 20 newsgroups dataset for categories:")
print(categories)

dataset = fetch_20newsgroups(subset='all', categories=categories,
                             shuffle=True, random_state=42)

print("%d documents" % len(dataset.data))
print("%d categories" % len(dataset.target_names))
print()

labels = dataset.target
true_k = np.unique(labels).shape[0]

print("Extracting features from the training dataset using a sparse vectorizer")
t0 = time()
if opts.use_hashing:
    if opts.use_idf:
        # Perform an IDF normalization on the output of HashingVectorizer
        hasher = HashingVectorizer(n_features=opts.n_features,
                                   stop_words='english', non_negative=True,
                                   norm=None, binary=False)
        vectorizer = make_pipeline(hasher, TfidfTransformer())
    else:
        vectorizer = HashingVectorizer(n_features=opts.n_features,
                                       stop_words='english',
                                       non_negative=False, norm='l2',
                                       binary=False)
else:
    vectorizer = TfidfVectorizer(max_df=0.5, max_features=opts.n_features,
                                 min_df=2, stop_words='english',
                                 use_idf=opts.use_idf)
X = vectorizer.fit_transform(dataset.data)

print("done in %fs" % (time() - t0))
print("n_samples: %d, n_features: %d" % X.shape)
print()

if opts.n_components:
    print("Performing dimensionality reduction using LSA")
    t0 = time()
    # Vectorizer results are normalized, which makes KMeans behave as
    # spherical k-means for better results. Since LSA/SVD results are
    # not normalized, we have to redo the normalization.
    svd = TruncatedSVD(opts.n_components)
    normalizer = Normalizer(copy=False)
    lsa = make_pipeline(svd, normalizer)

    X = lsa.fit_transform(X)

    print("done in %fs" % (time() - t0))

    explained_variance = svd.explained_variance_ratio_.sum()
    print("Explained variance of the SVD step: {}%".format(
        int(explained_variance * 100)))

    print()

###############################################################################
# Do the actual clustering

if opts.minibatch:
    km = MiniBatchKMeans(n_clusters=true_k, init='k-means++', n_init=1,
                         init_size=1000, batch_size=1000, verbose=opts.verbose)
else:
    km = KMeans(n_clusters=true_k, init='k-means++', max_iter=100, n_init=1,
                verbose=opts.verbose)

print("Clustering sparse data with %s" % km)
t0 = time()
km.fit(X)
print("done in %0.3fs" % (time() - t0))
print()

print("Homogeneity: %0.3f" % metrics.homogeneity_score(labels, km.labels_))
print("Completeness: %0.3f" % metrics.completeness_score(labels, km.labels_))
print("V-measure: %0.3f" % metrics.v_measure_score(labels, km.labels_))
print("Adjusted Rand-Index: %.3f"
      % metrics.adjusted_rand_score(labels, km.labels_))
print("Silhouette Coefficient: %0.3f"
      % metrics.silhouette_score(X, km.labels_, sample_size=1000))

print()

if not opts.use_hashing:
    print("Top terms per cluster:")

    if opts.n_components:
        original_space_centroids = svd.inverse_transform(km.cluster_centers_)
        order_centroids = original_space_centroids.argsort()[:, ::-1]
    else:
        order_centroids = km.cluster_centers_.argsort()[:, ::-1]

    terms = vectorizer.get_feature_names()
    for i in range(true_k):
        print("Cluster %d:" % i, end='')
        for ind in order_centroids[i, :10]:
            print(' %s' % terms[ind], end='')
        print()


import sys

backends = {'tk': 'TkAgg',
            'gtk': 'GTKAgg',
            'wx': 'WXAgg',
            'qt': 'Qt4Agg', # qt3 not supported
            'qt4': 'Qt4Agg',
            'osx': 'MacOSX'}

# We also need a reverse backends2guis mapping that will properly choose which
# GUI support to activate based on the desired matplotlib backend.  For the
# most part it's just a reverse of the above dict, but we also need to add a
# few others that map to the same GUI manually:
backend2gui = dict(zip(backends.values(), backends.keys()))
backend2gui['Qt4Agg'] = 'qt'
# In the reverse mapping, there are a few extra valid matplotlib backends that
# map to the same GUI support
backend2gui['GTK'] = backend2gui['GTKCairo'] = 'gtk'
backend2gui['WX'] = 'wx'
backend2gui['CocoaAgg'] = 'osx'

def do_enable_gui(guiname):
    from pydev_versioncheck import versionok_for_gui
    if versionok_for_gui():
        try:
            from pydev_ipython.inputhook import enable_gui
            enable_gui(guiname)
        except:
            sys.stderr.write("Failed to enable GUI event loop integration for '%s'\n" % guiname)
            import traceback
            traceback.print_exc()
    elif guiname not in ['none', '', None]:
        # Only print a warning if the guiname was going to do something
        sys.stderr.write("Debug console: Python version does not support GUI event loop integration for '%s'\n" % guiname)
    # Return value does not matter, so return back what was sent
    return guiname

def find_gui_and_backend():
    """Return the gui and mpl backend."""
    matplotlib = sys.modules['matplotlib']
    # WARNING: this assumes matplotlib 1.1 or newer!!
    backend = matplotlib.rcParams['backend']
    # In this case, we need to find what the appropriate gui selection call
    # should be for IPython, so we can activate inputhook accordingly
    gui = backend2gui.get(backend, None)
    return gui, backend

def is_interactive_backend(backend):
    """ Check if backend is interactive """
    matplotlib = sys.modules['matplotlib']
    from matplotlib.rcsetup import interactive_bk, non_interactive_bk
    if backend in interactive_bk:
        return True
    elif backend in non_interactive_bk:
        return False
    else:
        return matplotlib.is_interactive()

def patch_use(enable_gui_function):
    """ Patch matplotlib function 'use' """
    matplotlib = sys.modules['matplotlib']
    def patched_use(*args, **kwargs):
        matplotlib.real_use(*args, **kwargs)
        gui, backend = find_gui_and_backend()
        enable_gui_function(gui)

    setattr(matplotlib, "real_use", getattr(matplotlib, "use"))
    setattr(matplotlib, "use", patched_use)

def patch_is_interactive():
    """ Patch matplotlib function 'use' """
    matplotlib = sys.modules['matplotlib']
    def patched_is_interactive():
        return matplotlib.rcParams['interactive']

    setattr(matplotlib, "real_is_interactive", getattr(matplotlib, "is_interactive"))
    setattr(matplotlib, "is_interactive", patched_is_interactive)

def activate_matplotlib(enable_gui_function):
    """Set interactive to True for interactive backends.
    enable_gui_function - Function which enables gui, should be run in the main thread.
    """
    matplotlib = sys.modules['matplotlib']
    gui, backend = find_gui_and_backend()
    is_interactive = is_interactive_backend(backend)
    if is_interactive:
        enable_gui_function(gui)
        if not matplotlib.is_interactive():
            sys.stdout.write("Backend %s is interactive backend. Turning interactive mode on.\n" % backend)
        matplotlib.interactive(True)
    else:
        if matplotlib.is_interactive():
            sys.stdout.write("Backend %s is non-interactive backend. Turning interactive mode off.\n" % backend)
        matplotlib.interactive(False)
    patch_use(enable_gui_function)
    patch_is_interactive()

def flag_calls(func):
    """Wrap a function to detect and flag when it gets called.

    This is a decorator which takes a function and wraps it in a function with
    a 'called' attribute. wrapper.called is initialized to False.

    The wrapper.called attribute is set to False right before each call to the
    wrapped function, so if the call fails it remains False.  After the call
    completes, wrapper.called is set to True and the output is returned.

    Testing for truth in wrapper.called allows you to determine if a call to
    func() was attempted and succeeded."""

    # don't wrap twice
    if hasattr(func, 'called'):
        return func

    def wrapper(*args,**kw):
        wrapper.called = False
        out = func(*args,**kw)
        wrapper.called = True
        return out

    wrapper.called = False
    wrapper.__doc__ = func.__doc__
    return wrapper

def activate_pylab():
    pylab = sys.modules['pylab']
    pylab.show._needmain = False
    # We need to detect at runtime whether show() is called by the user.
    # For this, we wrap it into a decorator which adds a 'called' flag.
    pylab.draw_if_interactive = flag_calls(pylab.draw_if_interactive)

def activate_pyplot():
    pyplot = sys.modules['matplotlib.pyplot']
    pyplot.show._needmain = False
    # We need to detect at runtime whether show() is called by the user.
    # For this, we wrap it into a decorator which adds a 'called' flag.
    pyplot.draw_if_interactive = flag_calls(pyplot.draw_if_interactive)


# Requires pandas, bs4, html5lib, and lxml
#
# Call script with the output from aws_resource_actions callback, e.g.
# python build_iam_policy_framework.py ['ec2:AuthorizeSecurityGroupEgress', 'ec2:AuthorizeSecurityGroupIngress', 'sts:GetCallerIdentity']
#
# The sample output:
# {
#     "Version": "2012-10-17",
#     "Statement": [
#         {
#             "Sid": "AnsibleEditor0",
#             "Effect": "Allow",
#             "Action": [
#                 "ec2:AuthorizeSecurityGroupEgress",
#                 "ec2:AuthorizeSecurityGroupIngress"
#             ],
#             "Resource": "arn:aws:ec2:${Region}:${Account}:security-group/${SecurityGroupId}"
#         },
#         {
#             "Sid": "AnsibleEditor1",
#             "Effect": "Allow",
#             "Action": [
#                 "sts:GetCallerIdentity"
#             ],
#             "Resource": "*"
#         }
#     ]
# }
#
# Policy troubleshooting:
#    - If there are more actions in the policy than you provided, AWS has documented dependencies for some of your actions and
#      those have been added to the policy.
#    - If there are fewer actions in the policy than you provided, some of your actions are not in the IAM table of actions for
#      that service. For example, the API call s3:DeleteObjects does not actually correlate to the permission needed in a policy.
#      In this case s3:DeleteObject is the permission required to allow both the s3:DeleteObjects action and the s3:DeleteObject action.
#    - The policies output are only as accurate as the AWS documentation. If the policy does not permit the
#      necessary actions, look for undocumented dependencies. For example, redshift:CreateCluster requires ec2:DescribeVpcs,
#      ec2:DescribeSubnets, ec2:DescribeSecurityGroups, and ec2:DescribeInternetGateways, but AWS does not document this.
#

import json
import requests
import sys

missing_dependencies = []
try:
    import pandas as pd
except ImportError:
    missing_dependencies.append('pandas')
try:
    import bs4
except ImportError:
    missing_dependencies.append('bs4')
try:
    import html5lib
except ImportError:
    missing_dependencies.append('html5lib')
try:
    import lxml
except ImportError:
    missing_dependencies.append('lxml')

irregular_service_names = {
    'a4b': 'alexaforbusiness',
    'appstream': 'appstream2.0',
    'acm': 'certificatemanager',
    'acm-pca': 'certificatemanagerprivatecertificateauthority',
    'aws-marketplace-management': 'marketplacemanagementportal',
    'ce': 'costexplorerservice',
    'cognito-identity': 'cognitoidentity',
    'cognito-sync': 'cognitosync',
    'cognito-idp': 'cognitouserpools',
    'cur': 'costandusagereport',
    'dax': 'dynamodbacceleratordax',
    'dlm': 'datalifecyclemanager',
    'dms': 'databasemigrationservice',
    'ds': 'directoryservice',
    'ec2messages': 'messagedeliveryservice',
    'ecr': 'ec2containerregistry',
    'ecs': 'elasticcontainerservice',
    'eks': 'elasticcontainerserviceforkubernetes',
    'efs': 'elasticfilesystem',
    'es': 'elasticsearchservice',
    'events': 'cloudwatchevents',
    'firehose': 'kinesisfirehose',
    'fms': 'firewallmanager',
    'health': 'healthapisandnotifications',
    'importexport': 'importexportdiskservice',
    'iot1click': 'iot1-click',
    'kafka': 'managedstreamingforkafka',
    'kinesisvideo': 'kinesisvideostreams',
    'kms': 'keymanagementservice',
    'license-manager': 'licensemanager',
    'logs': 'cloudwatchlogs',
    'opsworks-cm': 'opsworksconfigurationmanagement',
    'mediaconnect': 'elementalmediaconnect',
    'mediaconvert': 'elementalmediaconvert',
    'medialive': 'elementalmedialive',
    'mediapackage': 'elementalmediapackage',
    'mediastore': 'elementalmediastore',
    'mgh': 'migrationhub',
    'mobiletargeting': 'pinpoint',
    'pi': 'performanceinsights',
    'pricing': 'pricelist',
    'ram': 'resourceaccessmanager',
    'resource-groups': 'resourcegroups',
    'sdb': 'simpledb',
    'servicediscovery': 'cloudmap',
    'serverlessrepo': 'serverlessapplicationrepository',
    'sms': 'servermigrationservice',
    'sms-voice': 'pinpointsmsandvoiceservice',
    'sso-directory': 'ssodirectory',
    'ssm': 'systemsmanager',
    'ssmmessages': 'sessionmanagermessagegatewayservice',
    'states': 'stepfunctions',
    'sts': 'securitytokenservice',
    'swf': 'simpleworkflowservice',
    'tag': 'resourcegrouptaggingapi',
    'transfer': 'transferforsftp',
    'waf-regional': 'wafregional',
    'wam': 'workspacesapplicationmanager',
    'xray': 'x-ray'
}

irregular_service_links = {
    'apigateway': [
        'https://docs.aws.amazon.com/IAM/latest/UserGuide/list_manageamazonapigateway.html'
    ],
    'aws-marketplace': [
        'https://docs.aws.amazon.com/IAM/latest/UserGuide/list_awsmarketplace.html',
        'https://docs.aws.amazon.com/IAM/latest/UserGuide/list_awsmarketplacemeteringservice.html',
        'https://docs.aws.amazon.com/IAM/latest/UserGuide/list_awsprivatemarketplace.html'
    ],
    'discovery': [
        'https://docs.aws.amazon.com/IAM/latest/UserGuide/list_applicationdiscovery.html'
    ],
    'elasticloadbalancing': [
        'https://docs.aws.amazon.com/IAM/latest/UserGuide/list_elasticloadbalancing.html',
        'https://docs.aws.amazon.com/IAM/latest/UserGuide/list_elasticloadbalancingv2.html'
    ],
    'globalaccelerator': [
        'https://docs.aws.amazon.com/IAM/latest/UserGuide/list_globalaccelerator.html'
    ]
}

def get_docs_by_prefix(prefix):
    amazon_link_form = 'https://docs.aws.amazon.com/IAM/latest/UserGuide/list_amazon{0}.html'
    aws_link_form = 'https://docs.aws.amazon.com/IAM/latest/UserGuide/list_aws{0}.html'

    if prefix in irregular_service_links:
        links = irregular_service_links[prefix]
    else:
        if prefix in irregular_service_names:
            prefix = irregular_service_names[prefix]
        links = [amazon_link_form.format(prefix), aws_link_form.format(prefix)]

    return links

def get_html(links):
    html_list = []
    for link in links:
        html = requests.get(link).content
        try:
            parsed_html = pd.read_html(html)
            html_list.append(parsed_html)
        except ValueError as e:
            if 'No tables found' in str(e):
                pass
            else:
                raise e

    return html_list

def get_tables(service):
    links = get_docs_by_prefix(service)
    html_list = get_html(links)
    action_tables = []
    arn_tables = []
    for df_list in html_list:
        for df in df_list:
            table = json.loads(df.to_json(orient='split'))
            table_data = table['data'][0]
            if 'Actions' in table_data and 'Resource Types (*required)' in table_data:
                action_tables.append(table['data'][1::])
            elif 'Resource Types' in table_data and 'ARN' in table_data:
                arn_tables.append(table['data'][1::])

    # Action table indices:
    # 0: Action, 1: Description, 2: Access level, 3: Resource type, 4: Condition keys, 5: Dependent actions
    # ARN tables indices:
    # 0: Resource type, 1: ARN template, 2: Condition keys
    return action_tables, arn_tables

def add_dependent_action(resources, dependency):
    resource, action = dependency.split(':')
    if resource in resources:
        resources[resource].append(action)
    else:
        resources[resource] = [action]
    return resources

def get_dependent_actions(resources):
    for service in dict(resources):
        action_tables, arn_tables = get_tables(service)
        for found_action_table in action_tables:
            for action_stuff in found_action_table:
                if action_stuff is None:
                    continue
                if action_stuff[0] in resources[service] and action_stuff[5]:
                    dependencies = action_stuff[5].split()
                    if isinstance(dependencies, list):
                        for dependency in dependencies:
                            resources = add_dependent_action(resources, dependency)
                    else:
                        resources = add_dependent_action(resources, dependencies)
    return resources

def get_actions_by_service(resources):
    service_action_dict = {}
    dependencies = {}
    for service in resources:
        action_tables, arn_tables = get_tables(service)

        # Create dict of the resource type to the corresponding ARN
        arn_dict = {}
        for found_arn_table in arn_tables:
            for arn_stuff in found_arn_table:
                arn_dict["{0}*".format(arn_stuff[0])] = arn_stuff[1]

        # Create dict of the action to the corresponding ARN
        action_dict = {}
        for found_action_table in action_tables:
            for action_stuff in found_action_table:
                if action_stuff[0] is None:
                    continue
                if arn_dict.get(action_stuff[3]):
                    action_dict[action_stuff[0]] = arn_dict[action_stuff[3]]
                else:
                    action_dict[action_stuff[0]] = None
        service_action_dict[service] = action_dict
    return service_action_dict

def get_resource_arns(aws_actions, action_dict):
    resource_arns = {}
    for resource_action in aws_actions:
        resource, action = resource_action.split(':')
        if action not in action_dict:
            continue
        if action_dict[action] is None:
            resource = "*"
        else:
            resource = action_dict[action].replace("${Partition}", "aws")
        if resource not in resource_arns:
            resource_arns[resource] = []
        resource_arns[resource].append(resource_action)
    return resource_arns

def get_resources(actions):
    resources = {}
    for action in actions:
        resource, action = action.split(':')
        if resource not in resources:
            resources[resource] = []
        resources[resource].append(action)
    return resources

def combine_arn_actions(resources, service_action_arn_dict):
    arn_actions = {}
    for service in service_action_arn_dict:
        service_arn_actions = get_resource_arns(aws_actions, service_action_arn_dict[service])
        for resource in service_arn_actions:
            if resource in arn_actions:
                arn_actions[resource].extend(service_arn_actions[resource])
            else:
                arn_actions[resource] = service_arn_actions[resource]
    return arn_actions

def combine_actions_and_dependent_actions(resources):
    aws_actions = []
    for resource in resources:
        for action in resources[resource]:
            aws_actions.append('{0}:{1}'.format(resource, action))
    return set(aws_actions)

def get_actions_restricted_by_arn(aws_actions):
    resources = get_resources(aws_actions)
    resources = get_dependent_actions(resources)
    service_action_arn_dict = get_actions_by_service(resources)
    aws_actions = combine_actions_and_dependent_actions(resources)
    return combine_arn_actions(aws_actions, service_action_arn_dict)

def main(aws_actions):
    arn_actions = get_actions_restricted_by_arn(aws_actions)
    statement = []
    for resource_restriction in arn_actions:
        statement.append({
            "Sid": "AnsibleEditor{0}".format(len(statement)),
            "Effect": "Allow",
            "Action": arn_actions[resource_restriction],
            "Resource": resource_restriction
        })

    policy = {"Version": "2012-10-17", "Statement": statement}
    print(json.dumps(policy, indent=4))

if __name__ == '__main__':
    if missing_dependencies:
        sys.exit('Missing Python libraries: {0}'.format(', '.join(missing_dependencies)))
    actions = sys.argv[1:]
    if len(actions) == 1:
        actions = sys.argv[1].split(',')
    aws_actions = [action.strip('[], "\'') for action in actions]
    main(aws_actions)


# coding=utf-8

from sklearn.datasets import load_digits
from sklearn.cross_validation import train_test_split
from sklearn.metrics.classification import accuracy_score
import numpy as np

from dbn import SupervisedDBNClassification

# Loading dataset
digits = load_digits()
X, Y = digits.data, digits.target

# Data scaling
X = (X / 16).astype(np.float32)

# Splitting data
X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.2, random_state=0)

# Training
classifier = SupervisedDBNClassification(hidden_layers_structure=[256, 256],
                                         learning_rate_rbm=0.1,
                                         learning_rate=0.1,
                                         n_epochs_rbm=10,
                                         n_iter_backprop=100,
                                         l2_regularization=0.0,
                                         batch_size=32,
                                         activation_function='relu',
                                         dropout_p=0.2)
classifier.fit(X_train, Y_train)

# Test
Y_pred = classifier.predict(X_test)

print('Done.\nAccuracy: ')
print(accuracy_score(Y_test, Y_pred))


import os
import shutil
import tempfile
import warnings
import numpy
from pickle import loads
from pickle import dumps

from sklearn.datasets import get_data_home
from sklearn.datasets import clear_data_home
from sklearn.datasets import load_files
from sklearn.datasets import load_sample_images
from sklearn.datasets import load_sample_image
from sklearn.datasets import load_digits
from sklearn.datasets import load_diabetes
from sklearn.datasets import load_linnerud
from sklearn.datasets import load_iris
from sklearn.datasets import load_breast_cancer
from sklearn.datasets import load_boston
from sklearn.datasets.base import Bunch

from sklearn.externals.six import b, u

from sklearn.utils.testing import assert_false
from sklearn.utils.testing import assert_true
from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_raises
from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import with_setup

DATA_HOME = tempfile.mkdtemp(prefix="scikit_learn_data_home_test_")
LOAD_FILES_ROOT = tempfile.mkdtemp(prefix="scikit_learn_load_files_test_")
TEST_CATEGORY_DIR1 = ""
TEST_CATEGORY_DIR2 = ""

def _remove_dir(path):
    if os.path.isdir(path):
        shutil.rmtree(path)

def teardown_module():
    """Test fixture (clean up) run once after all tests of this module"""
    for path in [DATA_HOME, LOAD_FILES_ROOT]:
        _remove_dir(path)

def setup_load_files():
    global TEST_CATEGORY_DIR1
    global TEST_CATEGORY_DIR2
    TEST_CATEGORY_DIR1 = tempfile.mkdtemp(dir=LOAD_FILES_ROOT)
    TEST_CATEGORY_DIR2 = tempfile.mkdtemp(dir=LOAD_FILES_ROOT)
    sample_file = tempfile.NamedTemporaryFile(dir=TEST_CATEGORY_DIR1,
                                              delete=False)
    sample_file.write(b("Hello World!\n"))
    sample_file.close()

def teardown_load_files():
    _remove_dir(TEST_CATEGORY_DIR1)
    _remove_dir(TEST_CATEGORY_DIR2)

def test_data_home():
    # get_data_home will point to a pre-existing folder
    data_home = get_data_home(data_home=DATA_HOME)
    assert_equal(data_home, DATA_HOME)
    assert_true(os.path.exists(data_home))

    # clear_data_home will delete both the content and the folder it-self
    clear_data_home(data_home=data_home)
    assert_false(os.path.exists(data_home))

    # if the folder is missing it will be created again
    data_home = get_data_home(data_home=DATA_HOME)
    assert_true(os.path.exists(data_home))

def test_default_empty_load_files():
    res = load_files(LOAD_FILES_ROOT)
    assert_equal(len(res.filenames), 0)
    assert_equal(len(res.target_names), 0)
    assert_equal(res.DESCR, None)

@with_setup(setup_load_files, teardown_load_files)
def test_default_load_files():
    res = load_files(LOAD_FILES_ROOT)
    assert_equal(len(res.filenames), 1)
    assert_equal(len(res.target_names), 2)
    assert_equal(res.DESCR, None)
    assert_equal(res.data, [b("Hello World!\n")])

@with_setup(setup_load_files, teardown_load_files)
def test_load_files_w_categories_desc_and_encoding():
    category = os.path.abspath(TEST_CATEGORY_DIR1).split('/').pop()
    res = load_files(LOAD_FILES_ROOT, description="test",
                     categories=category, encoding="utf-8")
    assert_equal(len(res.filenames), 1)
    assert_equal(len(res.target_names), 1)
    assert_equal(res.DESCR, "test")
    assert_equal(res.data, [u("Hello World!\n")])

@with_setup(setup_load_files, teardown_load_files)
def test_load_files_wo_load_content():
    res = load_files(LOAD_FILES_ROOT, load_content=False)
    assert_equal(len(res.filenames), 1)
    assert_equal(len(res.target_names), 2)
    assert_equal(res.DESCR, None)
    assert_equal(res.get('data'), None)

def test_load_sample_images():
    try:
        res = load_sample_images()
        assert_equal(len(res.images), 2)
        assert_equal(len(res.filenames), 2)
        assert_true(res.DESCR)
    except ImportError:
        warnings.warn("Could not load sample images, PIL is not available.")

def test_load_digits():
    digits = load_digits()
    assert_equal(digits.data.shape, (1797, 64))
    assert_equal(numpy.unique(digits.target).size, 10)

    # test return_X_y option
    X_y_tuple = load_digits(return_X_y=True)
    bunch = load_digits()
    assert_true(isinstance(X_y_tuple, tuple))
    assert_array_equal(X_y_tuple[0], bunch.data)
    assert_array_equal(X_y_tuple[1], bunch.target)

def test_load_digits_n_class_lt_10():
    digits = load_digits(9)
    assert_equal(digits.data.shape, (1617, 64))
    assert_equal(numpy.unique(digits.target).size, 9)

def test_load_sample_image():
    try:
        china = load_sample_image('china.jpg')
        assert_equal(china.dtype, 'uint8')
        assert_equal(china.shape, (427, 640, 3))
    except ImportError:
        warnings.warn("Could not load sample images, PIL is not available.")

def test_load_missing_sample_image_error():
    have_PIL = True
    try:
        try:
            from scipy.misc import imread
        except ImportError:
            from scipy.misc.pilutil import imread
    except ImportError:
        have_PIL = False
    if have_PIL:
        assert_raises(AttributeError, load_sample_image,
                      'blop.jpg')
    else:
        warnings.warn("Could not load sample images, PIL is not available.")

def test_load_diabetes():
    res = load_diabetes()
    assert_equal(res.data.shape, (442, 10))
    assert_true(res.target.size, 442)
    assert_equal(len(res.feature_names), 10)

    # test return_X_y option
    X_y_tuple = load_diabetes(return_X_y=True)
    bunch = load_diabetes()
    assert_true(isinstance(X_y_tuple, tuple))
    assert_array_equal(X_y_tuple[0], bunch.data)
    assert_array_equal(X_y_tuple[1], bunch.target)

def test_load_linnerud():
    res = load_linnerud()
    assert_equal(res.data.shape, (20, 3))
    assert_equal(res.target.shape, (20, 3))
    assert_equal(len(res.target_names), 3)
    assert_true(res.DESCR)

    # test return_X_y option
    X_y_tuple = load_linnerud(return_X_y=True)
    bunch = load_linnerud()
    assert_true(isinstance(X_y_tuple, tuple))
    assert_array_equal(X_y_tuple[0], bunch.data)
    assert_array_equal(X_y_tuple[1], bunch.target)

def test_load_iris():
    res = load_iris()
    assert_equal(res.data.shape, (150, 4))
    assert_equal(res.target.size, 150)
    assert_equal(res.target_names.size, 3)
    assert_true(res.DESCR)

    # test return_X_y option
    X_y_tuple = load_iris(return_X_y=True)
    bunch = load_iris()
    assert_true(isinstance(X_y_tuple, tuple))
    assert_array_equal(X_y_tuple[0], bunch.data)
    assert_array_equal(X_y_tuple[1], bunch.target)

def test_load_breast_cancer():
    res = load_breast_cancer()
    assert_equal(res.data.shape, (569, 30))
    assert_equal(res.target.size, 569)
    assert_equal(res.target_names.size, 2)
    assert_true(res.DESCR)

    # test return_X_y option
    X_y_tuple = load_breast_cancer(return_X_y=True)
    bunch = load_breast_cancer()
    assert_true(isinstance(X_y_tuple, tuple))
    assert_array_equal(X_y_tuple[0], bunch.data)
    assert_array_equal(X_y_tuple[1], bunch.target)

def test_load_boston():
    res = load_boston()
    assert_equal(res.data.shape, (506, 13))
    assert_equal(res.target.size, 506)
    assert_equal(res.feature_names.size, 13)
    assert_true(res.DESCR)

    # test return_X_y option
    X_y_tuple = load_boston(return_X_y=True)
    bunch = load_boston()
    assert_true(isinstance(X_y_tuple, tuple))
    assert_array_equal(X_y_tuple[0], bunch.data)
    assert_array_equal(X_y_tuple[1], bunch.target)

def test_loads_dumps_bunch():
    bunch = Bunch(x="x")
    bunch_from_pkl = loads(dumps(bunch))
    bunch_from_pkl.x = "y"
    assert_equal(bunch_from_pkl['x'], bunch_from_pkl.x)

def test_bunch_pickle_generated_with_0_16_and_read_with_0_17():
    bunch = Bunch(key='original')
    # This reproduces a problem when Bunch pickles have been created
    # with scikit-learn 0.16 and are read with 0.17. Basically there
    # is a suprising behaviour because reading bunch.key uses
    # bunch.__dict__ (which is non empty for 0.16 Bunch objects)
    # whereas assigning into bunch.key uses bunch.__setattr__. See
    # https://github.com/scikit-learn/scikit-learn/issues/6196 for
    # more details
    bunch.__dict__['key'] = 'set from __dict__'
    bunch_from_pkl = loads(dumps(bunch))
    # After loading from pickle the __dict__ should have been ignored
    assert_equal(bunch_from_pkl.key, 'original')
    assert_equal(bunch_from_pkl['key'], 'original')
    # Making sure that changing the attr does change the value
    # associated with __getitem__ as well
    bunch_from_pkl.key = 'changed'
    assert_equal(bunch_from_pkl.key, 'changed')
    assert_equal(bunch_from_pkl['key'], 'changed')

def test_bunch_dir():
    # check that dir (important for autocomplete) shows attributes
    data = load_iris()
    assert_true("data" in dir(data))


import os
import logging
import pandas as pd
import glob
import re
import datetime as dt
from collections import Counter

logger = logging.getLogger('hr.chatbot.stats')
trace_pattern = re.compile(
    r'../(?P<fname>.*), (?P<tloc>\(.*\)), (?P<pname>.*), (?P<ploc>\(.*\))')

def collect_history_data(history_dir, days):
    today = dt.datetime.utcnow()
    dfs = []
    for d in glob.glob('{}/*'.format(history_dir)):
        if os.path.isdir(d):
            dirname = os.path.basename(d)
            dirdate = None
            try:
                dirdate = dt.datetime.strptime(dirname, '%Y%m%d')
            except Exception as ex:
                logger.error(ex)
            if dirdate and (days == -1 or (today - dirdate).days < days):
                for fname in glob.glob('{}/{}/*.csv'.format(history_dir, dirname)):
                    try:
                        dfs.append(pd.read_csv(fname))
                    except Exception as ex:
                        logger.warn("Reading {} error: {}".format(fname, ex))
    if not dfs:
        return None
    df = pd.concat(dfs, ignore_index=True)
    df = df[df.Datetime != 'Datetime'].sort(
        ['User', 'Datetime']).drop_duplicates()
    return df

def history_stats(history_dir, days):
    df = collect_history_data(history_dir, days)
    if df is None:
        return {}
    if days == -1:
        stats_csv = '{}/full_history.csv'.format(history_dir)
    else:
        stats_csv = '{}/last_{}_days.csv'.format(history_dir, days)
    columns = [u'Datetime', u'Revision', u'User', u'BotName',
               u'AnsweredBy', u'Question', u'Answer', u'Rate', u'Trace']
    df.to_csv(stats_csv, index=False, columns=columns)
    logger.info("Write statistic records to {}".format(stats_csv))
    records = len(df)
    rates = len(df[df.Rate.notnull()])
    good_rates = len(df[df.Rate.isin(['good'])])
    bad_rates = len(df[df.Rate.isin(['bad'])])
    if records > 0:
        csd = float(records - bad_rates) / records
    response = {
        'customers_satisfaction_degree': csd,
        'number_of_records': records,
        'number_of_rates': rates,
        'number_of_good_rates': good_rates,
        'number_of_bad_rates': bad_rates,
    }
    return response

def playback_history(df):
    from client import Client
    client = Client(os.environ.get('HR_CHATBOT_AUTHKEY', 'AAAAB3NzaC'), test=True)
    pattern_column = []
    for question in df.Question:
        answer = client.ask(question, True)
        traces = answer.get('trace')
        patterns = []
        if traces:
            for trace in traces:
                match_obj = trace_pattern.match(trace)
                if match_obj:
                    patterns.append(match_obj.group('pname'))
        pattern_column.append(patterns)
    df.loc[:,'Pattern'] = pd.Series(pattern_column, index=df.index)
    return df

def pattern_stats(history_dir, days):
    df = collect_history_data(history_dir, days)
    if df is None:
        return {}
    df = playback_history(df)
    patterns = sum(df.Pattern, [])
    counter = Counter(patterns)
    pattern_freq = pd.Series(counter)
    pattern_freq.sort(ascending=False)
    stats_csv = '{}/pattern_frequency.csv'.format(history_dir)
    pattern_freq.to_csv(stats_csv)
    logger.info("Write pattern statistic to {}".format(stats_csv))

if __name__ == '__main__':
    logging.basicConfig()
    logging.getLogger().setLevel(logging.INFO)
    history_stats(os.path.expanduser('~/.hr/chatbot/history'), -1)
    history_stats(os.path.expanduser('~/.hr/chatbot/history'), 7)
    pattern_stats(os.path.expanduser('~/.hr/chatbot/history'), -1)


'''
Created on Oct 6, 2010

@author: Peter
'''
from numpy import *
import matplotlib
import matplotlib.pyplot as plt
from matplotlib.patches import Rectangle
import logRegres

dataMat,labelMat=logRegres.loadDataSet()
dataArr = array(dataMat)
weights = logRegres.stocGradAscent0(dataArr,labelMat)

n = shape(dataArr)[0] #number of points to create
xcord1 = []; ycord1 = []
xcord2 = []; ycord2 = []

markers =[]
colors =[]
for i in range(n):
    if int(labelMat[i])== 1:
        xcord1.append(dataArr[i,1]); ycord1.append(dataArr[i,2])
    else:
        xcord2.append(dataArr[i,1]); ycord2.append(dataArr[i,2])

fig = plt.figure()
ax = fig.add_subplot(111)
#ax.scatter(xcord,ycord, c=colors, s=markers)
type1 = ax.scatter(xcord1, ycord1, s=30, c='red', marker='s')
type2 = ax.scatter(xcord2, ycord2, s=30, c='green')
x = arange(-3.0, 3.0, 0.1)
#weights = [-2.9, 0.72, 1.29]
#weights = [-5, 1.09, 1.42]
weights = [13.03822793,   1.32877317,  -1.96702074]
weights = [4.12,   0.48,  -0.6168]
y = (-weights[0]-weights[1]*x)/weights[2]
type3 = ax.plot(x, y)
#ax.legend([type1, type2, type3], ["Did Not Like", "Liked in Small Doses", "Liked in Large Doses"], loc=2)
#ax.axis([-5000,100000,-2,25])
plt.xlabel('X1')
plt.ylabel('X2')
plt.show()


"""
======================================
Decision Tree Regression with AdaBoost
======================================

A decision tree is boosted using the AdaBoost.R2 [1] algorithm on a 1D
sinusoidal dataset with a small amount of Gaussian noise.
299 boosts (300 decision trees) is compared with a single decision tree
regressor. As the number of boosts is increased the regressor can fit more
detail.

.. [1] H. Drucker, "Improving Regressors using Boosting Techniques", 1997.

"""
print(__doc__)

# Author: Noel Dawe <noel.dawe@gmail.com>
#
# License: BSD 3 clause

# importing necessary libraries
import numpy as np
import matplotlib.pyplot as plt
from sklearn.tree import DecisionTreeRegressor
from sklearn.ensemble import AdaBoostRegressor

# Create the dataset
rng = np.random.RandomState(1)
X = np.linspace(0, 6, 100)[:, np.newaxis]
y = np.sin(X).ravel() + np.sin(6 * X).ravel() + rng.normal(0, 0.1, X.shape[0])

# Fit regression model
regr_1 = DecisionTreeRegressor(max_depth=4)

regr_2 = AdaBoostRegressor(DecisionTreeRegressor(max_depth=4),
                          n_estimators=300, random_state=rng)

regr_1.fit(X, y)
regr_2.fit(X, y)

# Predict
y_1 = regr_1.predict(X)
y_2 = regr_2.predict(X)

# Plot the results
plt.figure()
plt.scatter(X, y, c="k", label="training samples")
plt.plot(X, y_1, c="g", label="n_estimators=1", linewidth=2)
plt.plot(X, y_2, c="r", label="n_estimators=300", linewidth=2)
plt.xlabel("data")
plt.ylabel("target")
plt.title("Boosted Decision Tree Regression")
plt.legend()
plt.show()


"""
This example makes custom 'pie charts' as the markers for a scatter plotqu

Thanks to Manuel Metz for the example
"""
import math
import numpy as np
import matplotlib.pyplot as plt

# first define the ratios
r1 = 0.2      # 20%
r2 = r1 + 0.4 # 40%

# define some sizes of the scatter marker
sizes = [60,80,120]

# calculate the points of the first pie marker
#
# these are just the origin (0,0) +
# some points on a circle cos,sin
x = [0] + np.cos(np.linspace(0, 2*math.pi*r1, 10)).tolist()
y = [0] + np.sin(np.linspace(0, 2*math.pi*r1, 10)).tolist()
xy1 = list(zip(x,y))

# ...
x = [0] + np.cos(np.linspace(2*math.pi*r1, 2*math.pi*r2, 10)).tolist()
y = [0] + np.sin(np.linspace(2*math.pi*r1, 2*math.pi*r2, 10)).tolist()
xy2 = list(zip(x,y))

x = [0] + np.cos(np.linspace(2*math.pi*r2, 2*math.pi, 10)).tolist()
y = [0] + np.sin(np.linspace(2*math.pi*r2, 2*math.pi, 10)).tolist()
xy3 = list(zip(x,y))

fig, ax = plt.subplots()
ax.scatter( np.arange(3), np.arange(3), marker=(xy1,0), s=sizes, facecolor='blue' )
ax.scatter( np.arange(3), np.arange(3), marker=(xy2,0), s=sizes, facecolor='green' )
ax.scatter( np.arange(3), np.arange(3), marker=(xy3,0), s=sizes, facecolor='red' )
plt.show()


from __future__ import print_function, division
import matplotlib
matplotlib.use('Agg') # Must be before importing matplotlib.pyplot or pylab!
from neuralnilm import Net, RealApplianceSource, BLSTMLayer, DimshuffleLayer
from lasagne.nonlinearities import sigmoid, rectify
from lasagne.objectives import crossentropy, mse
from lasagne.init import Uniform, Normal
from lasagne.layers import LSTMLayer, DenseLayer, Conv1DLayer, ReshapeLayer, FeaturePoolLayer
from lasagne.updates import nesterov_momentum
from functools import partial
import os
from neuralnilm.source import standardise, discretize, fdiff, power_and_fdiff
from neuralnilm.experiment import run_experiment
from neuralnilm.net import TrainingError
import __main__
from copy import deepcopy
from math import sqrt

NAME = os.path.splitext(os.path.split(__main__.__file__)[1])[0]
PATH = "/homes/dk3810/workspace/python/neuralnilm/figures"
SAVE_PLOT_INTERVAL = 250
GRADIENT_STEPS = 100

"""
e233
based on e131c but with:
* lag=32
* pool

e234
* init final layer and conv layer

235
no lag

236
should be exactly as 131c: no pool, no lag, no init for final and conv layer

237
putting the pool back

238
seems pooling hurts us! disable pooling.
enable lag = 32

239
BLSTM
lag = 20

240
LSTM not BLSTM
various lags

241
output is prediction

ideas for next TODO:
* 3 LSTM layers with smaller conv between them
* why does pooling hurt us?
"""

source_dict = dict(
    filename='/data/dk3810/ukdale.h5',
    appliances=[
        ['fridge freezer', 'fridge', 'freezer'],
        'hair straighteners',
        'television',
        'dish washer',
        ['washer dryer', 'washing machine']
    ],
    max_appliance_powers=[300, 500, 200, 2500, 2400],
    on_power_thresholds=[5] * 5,
    max_input_power=5900,
    min_on_durations=[60, 60, 60, 1800, 1800],
    min_off_durations=[12, 12, 12, 1800, 600],
    window=("2013-06-01", "2014-07-01"),
    seq_length=1500,
    output_one_appliance=False,
    boolean_targets=False,
    train_buildings=[1],
    validation_buildings=[1],
    # skip_probability=0.0,
    n_seq_per_batch=50,
    # subsample_target=5,
    include_diff=False,
    clip_appliance_power=True,
    target_is_prediction=True
    #lag=0
)

net_dict = dict(
    save_plot_interval=SAVE_PLOT_INTERVAL,
    loss_function=crossentropy,
    layers_config=[
        {
            'type': LSTMLayer,
            'num_units': 10,
            'gradient_steps': GRADIENT_STEPS,
            'peepholes': False,
            'W_in_to_cell': Normal(std=1.)
        }
    ]
)

def exp_x(name, learning_rate):
    global source
    try:
        a = source
    except NameError:
        source = RealApplianceSource(**source_dict)
    net_dict_copy = deepcopy(net_dict)
    net_dict_copy.update(dict(
        experiment_name=name,
        source=source,
        updates=partial(nesterov_momentum, learning_rate=learning_rate)
    ))
    net_dict_copy['layers_config'].append(
        {
            'type': DenseLayer,
            'num_units': source.n_outputs,
            'nonlinearity': sigmoid,
            'W': Normal(std=(1/sqrt(50)))
        }
    )
    net = Net(**net_dict_copy)
    return net

def main():
    for experiment, learning_rate in [('a', 1.0), ('b', 0.1), ('c', 0.01),
                                      ('d', 0.001), ('e', 0.0001)]:
        full_exp_name = NAME + experiment
        path = os.path.join(PATH, full_exp_name)
        print("***********************************")
        print("Preparing", full_exp_name, "...")
        try:
            net = exp_x(full_exp_name, learning_rate)
            run_experiment(net, path, epochs=1000)
        except KeyboardInterrupt:
            break
        except TrainingError as exception:
            print("EXCEPTION:", exception)
        except Exception as exception:
            print("EXCEPTION:", exception)

if __name__ == "__main__":
    main()


#!/usr/bin/env python
# -*- Mode: Python; coding: utf-8; indent-tabs-mode: nil; tab-width: 4 -*-

# Post-diagnostic of STATION_ASF /  L. Brodeau, 2019

import sys
from os import path as path
#from string import replace
import math
import numpy as nmp
from netCDF4 import Dataset,num2date
import matplotlib as mpl
mpl.use('Agg')
import matplotlib.pyplot as plt
import matplotlib.dates as mdates

reload(sys)
sys.setdefaultencoding('utf8')

cy1     = '2016' ; # First year
cy2     = '2018' ; # Last year

jt0 = 0
jt0 = 17519

dir_figs='.'
size_fig=(13,7)
fig_ext='png'

clr_red = '#AD0000'
clr_blu = '#3749A3'
clr_gre = '#548F64'
clr_sat = '#ffed00'
clr_mod = '#008ab8'

rDPI=200.

L_ALGOS = [ 'COARE3p6' , 'ECMWF'   , 'NCAR' ]
l_xtrns = [ '-noskin'  , '-noskin' ,  ''    ] ; # string to add to algo name (L_ALGOS) to get version without skin params turned on
l_color = [  '#ffed00' , '#008ab8' , '0.4'  ] ; # colors to differentiate algos on the plot
l_width = [     3      ,    2      ,  1     ] ; # line-width to differentiate algos on the plot
l_style = [    '-'     ,   '-'     , '--'   ] ; # line-style

L_VNEM  = [   'qla'     ,     'qsb'     ,     'qt'     ,   'qlw'     ,  'taum'     ,    'dt_skin'         ]
L_VARO  = [   'Qlat'    ,    'Qsen'     ,     'Qnet'   ,   'Qlw'     ,  'Tau'      ,    'dT_skin'         ] ; # name of variable on figure
L_VARL  = [ r'$Q_{lat}$', r'$Q_{sens}$' , r'$Q_{net}$' , r'$Q_{lw}$' , r'$|\tau|$' , r'$\Delta T_{skin}$' ] ; # name of variable in latex mode
L_VUNT  = [ r'$W/m^2$'  , r'$W/m^2$'    , r'$W/m^2$'   , r'$W/m^2$'  , r'$N/m^2$'  ,      'K'             ]
L_VMAX  = [     75.     ,     75.       ,    800.      ,     25.     ,    1.2      ,      -0.7            ]
L_VMIN  = [   -250.     ,   -125.       ,   -400.      ,   -150.     ,    0.       ,       0.7            ]
L_ANOM  = [   True      ,    True       ,    True      ,    True     ,   True      ,      False           ]

#L_VNEM  = [   'qlw'      ]
#L_VARO  = [   'Qlw'      ] ; # name of variable on figure
#L_VARL  = [ r'$Q_{lw}$'  ] ; # name of variable in latex mode
#L_VUNT  = [ r'$W/m^2$'   ]
#L_VMAX  = [     25.      ]
#L_VMIN  = [   -150.      ]
#L_ANOM  = [    True      ]

nb_algos = len(L_ALGOS) ; print(nb_algos)

# Getting arguments:
narg = len(sys.argv)
if narg != 2:
    print 'Usage: '+sys.argv[0]+' <DIR_OUT_SASF>'; sys.exit(0)
cdir_data = sys.argv[1]

# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
# Populating and checking existence of files to be read
# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
def chck4f(cf):
    cmesg = 'ERROR: File '+cf+' does not exist !!!'
    if not path.exists(cf): print cmesg ; sys.exit(0)

###cf_in = nmp.empty((), dtype="S10")
cf_in = [] ; cf_in_ns = []
for ja in range(nb_algos):
    cfi = cdir_data+'/output/'+'STATION_ASF-'+L_ALGOS[ja]+'_1h_'+cy1+'0101_'+cy2+'1231_gridT.nc'
    chck4f(cfi)
    cf_in.append(cfi)
# Same but without skin params:
for ja in range(nb_algos):
    cfi = cdir_data+'/output/'+'STATION_ASF-'+L_ALGOS[ja]+l_xtrns[ja]+'_1h_'+cy1+'0101_'+cy2+'1231_gridT.nc'
    chck4f(cfi)
    cf_in_ns.append(cfi)
print('Files we are goin to use:')
for ja in range(nb_algos): print(cf_in[ja])
print('   --- same without cool-skin/warm-layer:')
for ja in range(nb_algos): print(cf_in_ns[ja])
#-----------------------------------------------------------------

# Getting time array from the first file:
id_in = Dataset(cf_in[0])
vt = id_in.variables['time_counter'][jt0:]
cunit_t = id_in.variables['time_counter'].units
clndr_t = id_in.variables['time_counter'].calendar
id_in.close()
Nt = len(vt)
print(' "time" => units = '+cunit_t+', calendar = "'+clndr_t+'"')

vtime = num2date(vt, units=cunit_t) ; # something understandable!

ii=Nt/300
ib=max(ii-ii%10,1)
xticks_d=int(30*ib)

font_inf = { 'fontname':'Open Sans', 'fontweight':'normal', 'fontsize':14 }

nb_var = len(L_VNEM)

xF  = nmp.zeros((Nt,nb_algos))
xFa = nmp.zeros((Nt,nb_algos))

for ctest in ['skin','noskin']:

    for jv in range(nb_var):
        print('\n *** Treating variable: '+L_VARO[jv]+' ('+ctest+') !')

        for ja in range(nb_algos):
            #
            if ctest == 'skin':   id_in = Dataset(cf_in[ja])
            if ctest == 'noskin': id_in = Dataset(cf_in_ns[ja])
            xF[:,ja] = id_in.variables[L_VNEM[jv]][jt0:,1,1] # only the center point of the 3x3 spatial domain!
            if ja == 0: cvar_lnm = id_in.variables[L_VNEM[jv]].long_name
            id_in.close()

        fig = plt.figure(num = jv, figsize=size_fig, facecolor='w', edgecolor='k')

        ax1 = plt.axes([0.07, 0.22, 0.9, 0.75])

        ax1.set_xticks(vtime[::xticks_d])
        ax1.xaxis.set_major_formatter(mdates.DateFormatter('%Y-%m-%d %H:%M:%S'))
        plt.xticks(rotation='60')

        for ja in range(nb_algos):
            plt.plot(vtime, xF[:,ja], '-', color=l_color[ja], linestyle=l_style[ja], linewidth=l_width[ja], label=L_ALGOS[ja], zorder=10+ja)

        ax1.set_ylim(L_VMIN[jv], L_VMAX[jv]) ; ax1.set_xlim(vtime[0],vtime[Nt-1])
        plt.ylabel(L_VARL[jv]+' ['+L_VUNT[jv]+']')

        ax1.grid(color='k', linestyle='-', linewidth=0.3)
        plt.legend(bbox_to_anchor=(0.45, 0.2), ncol=1, shadow=True, fancybox=True)
        ax1.annotate(cvar_lnm+' ('+ctest+')', xy=(0.3, 0.97), xycoords='axes fraction',  bbox={'facecolor':'w', 'alpha':1., 'pad':10}, zorder=50, **font_inf)
        plt.savefig(L_VARO[jv]+'_'+ctest+'.'+fig_ext, dpi=int(rDPI), transparent=False)
        plt.close(jv)

        if L_ANOM[jv]:

            for ja in range(nb_algos): xFa[:,ja] = xF[:,ja] - nmp.mean(xF,axis=1)

            if nmp.sum(xFa[:,:]) == 0.0:
                print('     Well! Seems that for variable '+L_VARO[jv]+', choice of algo has no impact a all!')
                print('          ==> skipping anomaly plot...')

            else:

                # Want a symetric y-range that makes sense for the anomaly we're looking at:
                rmax = nmp.max(xFa) ; rmin = nmp.min(xFa)
                rmax = max( abs(rmax) , abs(rmin) )
                romagn = math.floor(math.log(rmax, 10)) ; # order of magnitude of the anomaly  we're dealing with
                rmlt = 10.**(int(romagn)) / 2.
                yrng = math.copysign( math.ceil(abs(rmax)/rmlt)*rmlt , rmax)
                #print 'yrng = ', yrng ;  #sys.exit(0)

                fig = plt.figure(num = 10+jv, figsize=size_fig, facecolor='w', edgecolor='k')
                ax1 = plt.axes([0.07, 0.22, 0.9, 0.75])

                ax1.set_xticks(vtime[::xticks_d])
                ax1.xaxis.set_major_formatter(mdates.DateFormatter('%Y-%m-%d %H:%M:%S'))
                plt.xticks(rotation='60')

                for ja in range(nb_algos):
                    plt.plot(vtime, xFa[:,ja], '-', color=l_color[ja], linewidth=l_width[ja], label=L_ALGOS[ja], zorder=10+ja)

                ax1.set_ylim(-yrng,yrng) ; ax1.set_xlim(vtime[0],vtime[Nt-1])
                plt.ylabel(L_VARL[jv]+' ['+L_VUNT[jv]+']')
                ax1.grid(color='k', linestyle='-', linewidth=0.3)
                plt.legend(bbox_to_anchor=(0.45, 0.2), ncol=1, shadow=True, fancybox=True)
                ax1.annotate('Anomaly of '+cvar_lnm+' ('+ctest+')', xy=(0.3, 0.97), xycoords='axes fraction',  bbox={'facecolor':'w', 'alpha':1., 'pad':10}, zorder=50, **font_inf)
                plt.savefig(L_VARO[jv]+'_'+ctest+'_anomaly.'+fig_ext, dpi=int(rDPI), transparent=False)
                plt.close(10+jv)

# Difference skin vs noskin:
xFns = nmp.zeros((Nt,nb_algos))

for jv in range(nb_var-1):
    print('\n *** Treating variable: '+L_VARO[jv]+' ('+ctest+') !')

    for ja in range(nb_algos-1):
        id_in = Dataset(cf_in[ja])
        xF[:,ja]   = id_in.variables[L_VNEM[jv]][jt0:,1,1] # only the center point of the 3x3 spatial domain!
        if ja == 0: cvar_lnm = id_in.variables[L_VNEM[jv]].long_name
        id_in.close()
        #
        id_in = Dataset(cf_in_ns[ja])
        xFns[:,ja] = id_in.variables[L_VNEM[jv]][jt0:,1,1] # only the center point of the 3x3 spatial domain!
        if ja == 0: cvar_lnm = id_in.variables[L_VNEM[jv]].long_name
        id_in.close()

        xFa[:,ja] = xF[:,ja] - xFns[:,ja]  ; # difference!

    # Want a symetric y-range that makes sense for the anomaly we're looking at:
    rmax = nmp.max(xFa) ; rmin = nmp.min(xFa)
    rmax = max( abs(rmax) , abs(rmin) )
    romagn = math.floor(math.log(rmax, 10)) ; # order of magnitude of the anomaly  we're dealing with
    rmlt = 10.**(int(romagn)) / 2.
    yrng = math.copysign( math.ceil(abs(rmax)/rmlt)*rmlt , rmax)
    print 'yrng = ', yrng ;  #sys.exit(0)

    for ja in range(nb_algos-1):

        calgo = L_ALGOS[ja]

        if nmp.sum(xFa[:,ja]) == 0.0:
            print('     Well! Seems that for variable '+L_VARO[jv]+', and algo '+calgo+', skin param has no impact')
            print('          ==> skipping difference plot...')

        else:

            fig = plt.figure(num = jv, figsize=size_fig, facecolor='w', edgecolor='k')
            ax1 = plt.axes([0.07, 0.22, 0.9, 0.75])

            ax1.set_xticks(vtime[::xticks_d])
            ax1.xaxis.set_major_formatter(mdates.DateFormatter('%Y-%m-%d %H:%M:%S'))
            plt.xticks(rotation='60')

            plt.plot(vtime, xFa[:,ja], '-', color=l_color[ja], linestyle=l_style[ja], linewidth=l_width[ja], label=None, zorder=10+ja)

            ax1.set_ylim(-yrng,yrng) ; ax1.set_xlim(vtime[0],vtime[Nt-1])
            plt.ylabel(L_VARL[jv]+' ['+L_VUNT[jv]+']')

            ax1.grid(color='k', linestyle='-', linewidth=0.3)
            #plt.legend(bbox_to_anchor=(0.45, 0.2), ncol=1, shadow=True, fancybox=True)
            ax1.annotate(cvar_lnm+' ('+ctest+')', xy=(0.3, 0.97), xycoords='axes fraction',  bbox={'facecolor':'w', 'alpha':1., 'pad':10}, zorder=50, **font_inf)
            plt.savefig('diff_skin-noskin_'+L_VARO[jv]+'_'+calgo+'_'+ctest+'.'+fig_ext, dpi=int(rDPI), transparent=False)
            plt.close(jv)


# --------------------------------------------------------
# Fast R-CNN
# Copyright (c) 2015 Microsoft
# Licensed under The MIT License [see LICENSE for details]
# Written by Ross Girshick
# --------------------------------------------------------

"""Compute minibatch blobs for training a Fast R-CNN network."""

import numpy as np
import numpy.random as npr
import cv2
import os

# TODO: make fast_rcnn irrelevant
# >>>> obsolete, because it depends on sth outside of this project
from ..fast_rcnn.config import cfg
# <<<< obsolete
from ..utils.blob import prep_im_for_blob, im_list_to_blob

def get_minibatch(roidb, num_classes):
    """Given a roidb, construct a minibatch sampled from it."""
    num_images = len(roidb)
    # Sample random scales to use for each image in this batch
    random_scale_inds = npr.randint(0, high=len(cfg.TRAIN.SCALES),
                                    size=num_images)
    assert(cfg.TRAIN.BATCH_SIZE % num_images == 0), \
        'num_images ({}) must divide BATCH_SIZE ({})'. \
        format(num_images, cfg.TRAIN.BATCH_SIZE)
    rois_per_image = cfg.TRAIN.BATCH_SIZE / num_images
    fg_rois_per_image = np.round(cfg.TRAIN.FG_FRACTION * rois_per_image)

    # Get the input image blob, formatted for caffe
    im_blob, im_scales = _get_image_blob(roidb, random_scale_inds)

    blobs = {'data': im_blob}

    if cfg.TRAIN.HAS_RPN:
        assert len(im_scales) == 1, "Single batch only"
        assert len(roidb) == 1, "Single batch only"
        # gt boxes: (x1, y1, x2, y2, cls)
        gt_inds = np.where(roidb[0]['gt_classes'] != 0)[0]
        gt_boxes = np.empty((len(gt_inds), 5), dtype=np.float32)
        gt_boxes[:, 0:4] = roidb[0]['boxes'][gt_inds, :] * im_scales[0]
        gt_boxes[:, 4] = roidb[0]['gt_classes'][gt_inds]
        blobs['gt_boxes'] = gt_boxes
        blobs['gt_ishard'] = roidb[0]['gt_ishard'][gt_inds]  \
            if 'gt_ishard' in roidb[0] else np.zeros(gt_inds.size, dtype=int)
        # blobs['gt_ishard'] = roidb[0]['gt_ishard'][gt_inds]
        blobs['dontcare_areas'] = roidb[0]['dontcare_areas'] * im_scales[0] \
            if 'dontcare_areas' in roidb[0] else np.zeros([0, 4], dtype=float)
        blobs['im_info'] = np.array(
            [[im_blob.shape[1], im_blob.shape[2], im_scales[0]]],
            dtype=np.float32)
        blobs['im_name'] = os.path.basename(roidb[0]['image'])

    else: # not using RPN
        # Now, build the region of interest and label blobs
        rois_blob = np.zeros((0, 5), dtype=np.float32)
        labels_blob = np.zeros((0), dtype=np.float32)
        bbox_targets_blob = np.zeros((0, 4 * num_classes), dtype=np.float32)
        bbox_inside_blob = np.zeros(bbox_targets_blob.shape, dtype=np.float32)
        # all_overlaps = []
        for im_i in xrange(num_images):
            labels, overlaps, im_rois, bbox_targets, bbox_inside_weights \
                = _sample_rois(roidb[im_i], fg_rois_per_image, rois_per_image,
                               num_classes)

            # Add to RoIs blob
            rois = _project_im_rois(im_rois, im_scales[im_i])
            batch_ind = im_i * np.ones((rois.shape[0], 1))
            rois_blob_this_image = np.hstack((batch_ind, rois))
            rois_blob = np.vstack((rois_blob, rois_blob_this_image))

            # Add to labels, bbox targets, and bbox loss blobs
            labels_blob = np.hstack((labels_blob, labels))
            bbox_targets_blob = np.vstack((bbox_targets_blob, bbox_targets))
            bbox_inside_blob = np.vstack((bbox_inside_blob, bbox_inside_weights))
            # all_overlaps = np.hstack((all_overlaps, overlaps))

        # For debug visualizations
        # _vis_minibatch(im_blob, rois_blob, labels_blob, all_overlaps)

        blobs['rois'] = rois_blob
        blobs['labels'] = labels_blob

        if cfg.TRAIN.BBOX_REG:
            blobs['bbox_targets'] = bbox_targets_blob
            blobs['bbox_inside_weights'] = bbox_inside_blob
            blobs['bbox_outside_weights'] = \
                np.array(bbox_inside_blob > 0).astype(np.float32)

    return blobs

def _sample_rois(roidb, fg_rois_per_image, rois_per_image, num_classes):
    """Generate a random sample of RoIs comprising foreground and background
    examples.
    """
    # label = class RoI has max overlap with
    labels = roidb['max_classes']
    overlaps = roidb['max_overlaps']
    rois = roidb['boxes']

    # Select foreground RoIs as those with >= FG_THRESH overlap
    fg_inds = np.where(overlaps >= cfg.TRAIN.FG_THRESH)[0]
    # Guard against the case when an image has fewer than fg_rois_per_image
    # foreground RoIs
    fg_rois_per_this_image = np.minimum(fg_rois_per_image, fg_inds.size)
    # Sample foreground regions without replacement
    if fg_inds.size > 0:
        fg_inds = npr.choice(
                fg_inds, size=fg_rois_per_this_image, replace=False)

    # Select background RoIs as those within [BG_THRESH_LO, BG_THRESH_HI)
    bg_inds = np.where((overlaps < cfg.TRAIN.BG_THRESH_HI) &
                       (overlaps >= cfg.TRAIN.BG_THRESH_LO))[0]
    # Compute number of background RoIs to take from this image (guarding
    # against there being fewer than desired)
    bg_rois_per_this_image = rois_per_image - fg_rois_per_this_image
    bg_rois_per_this_image = np.minimum(bg_rois_per_this_image,
                                        bg_inds.size)
    # Sample foreground regions without replacement
    if bg_inds.size > 0:
        bg_inds = npr.choice(
                bg_inds, size=bg_rois_per_this_image, replace=False)

    # The indices that we're selecting (both fg and bg)
    keep_inds = np.append(fg_inds, bg_inds)
    # Select sampled values from various arrays:
    labels = labels[keep_inds]
    # Clamp labels for the background RoIs to 0
    labels[fg_rois_per_this_image:] = 0
    overlaps = overlaps[keep_inds]
    rois = rois[keep_inds]

    bbox_targets, bbox_inside_weights = _get_bbox_regression_labels(
            roidb['bbox_targets'][keep_inds, :], num_classes)

    return labels, overlaps, rois, bbox_targets, bbox_inside_weights

def _get_image_blob(roidb, scale_inds):
    """Builds an input blob from the images in the roidb at the specified
    scales.
    """
    num_images = len(roidb)
    processed_ims = []
    im_scales = []
    for i in xrange(num_images):
        im = cv2.imread(roidb[i]['image'])
        if roidb[i]['flipped']:
            im = im[:, ::-1, :]
        target_size = cfg.TRAIN.SCALES[scale_inds[i]]
        im, im_scale = prep_im_for_blob(im, cfg.PIXEL_MEANS, target_size,
                                        cfg.TRAIN.MAX_SIZE)
        im_scales.append(im_scale)
        processed_ims.append(im)

    # Create a blob to hold the input images
    blob = im_list_to_blob(processed_ims)

    return blob, im_scales

def _project_im_rois(im_rois, im_scale_factor):
    """Project image RoIs into the rescaled training image."""
    rois = im_rois * im_scale_factor
    return rois

def _get_bbox_regression_labels(bbox_target_data, num_classes):
    """Bounding-box regression targets are stored in a compact form in the
    roidb.

    This function expands those targets into the 4-of-4*K representation used
    by the network (i.e. only one class has non-zero targets). The loss weights
    are similarly expanded.

    Returns:
        bbox_target_data (ndarray): N x 4K blob of regression targets
        bbox_inside_weights (ndarray): N x 4K blob of loss weights
    """
    clss = bbox_target_data[:, 0]
    bbox_targets = np.zeros((clss.size, 4 * num_classes), dtype=np.float32)
    bbox_inside_weights = np.zeros(bbox_targets.shape, dtype=np.float32)
    inds = np.where(clss > 0)[0]
    for ind in inds:
        cls = clss[ind]
        start = 4 * cls
        end = start + 4
        bbox_targets[ind, start:end] = bbox_target_data[ind, 1:]
        bbox_inside_weights[ind, start:end] = cfg.TRAIN.BBOX_INSIDE_WEIGHTS
    return bbox_targets, bbox_inside_weights

def _vis_minibatch(im_blob, rois_blob, labels_blob, overlaps):
    """Visualize a mini-batch for debugging."""
    import matplotlib.pyplot as plt
    for i in xrange(rois_blob.shape[0]):
        rois = rois_blob[i, :]
        im_ind = rois[0]
        roi = rois[1:]
        im = im_blob[im_ind, :, :, :].transpose((1, 2, 0)).copy()
        im += cfg.PIXEL_MEANS
        im = im[:, :, (2, 1, 0)]
        im = im.astype(np.uint8)
        cls = labels_blob[i]
        plt.imshow(im)
        print 'class: ', cls, ' overlap: ', overlaps[i]
        plt.gca().add_patch(
            plt.Rectangle((roi[0], roi[1]), roi[2] - roi[0],
                          roi[3] - roi[1], fill=False,
                          edgecolor='r', linewidth=3)
            )
        plt.show()