import itertools
import operator

from sqlalchemy_searchable import search

from scoda.app import app
from flask import request, url_for, redirect, flash, make_response, session, render_template, jsonify, Response, \
    send_file
from flask_security import current_user
from itertools import zip_longest
from sqlalchemy.sql import select
from sqlalchemy import func, extract, desc
from .models import db
from .models import *
from .models.user import UserAnalysis
from .models.datasets import ExploreForm
from .models.maps import MapForm, NightFormETH, NightFormJHB
from pandas import read_sql_query
import gviz_api
import geojson, json
import pandas as pd
from .app import csrf
from werkzeug.datastructures import MultiDict
from urllib.parse import urlencode, urlparse, parse_qsl, urlsplit, parse_qs

def grouper(iterable, n, fillvalue=None):
    "Collect data into fixed-length chunks or blocks"
    # grouper('ABCDEFG', 3, 'x') --> ABC DEF Gxx"
    args = [iter(iterable)] * n
    return zip_longest(*args, fillvalue=fillvalue)

@app.route('/help')
def help():
    return render_template('help/help.html')

@app.route('/api/indicators-list/', defaults={'check': ''})
@app.route('/api/indicators-list/<check>', methods=['GET', 'POST'])
def api_indicators_list(check):
    remove_list = ['Poverty rate', 'Gini Coefficient', 'Gross Value Add', 'Exports', 'Multiple deprivation index',
                   'Human Development Index']
    if check == "codebook":
        indicators_list = [[str(c.id), c.name] for c in CbIndicator.query.join(CbDataPoint,CbDataPoint.indicator_id == CbIndicator.id).all() if c.name not in remove_list]
    else:
        indicators_list = [[str(c.id), c.in_name] for c in Indicator.all() if c.in_name not in remove_list]
    # print(indicators_list)
    # payload = {"indicators_list": indicators_list}
    return jsonify(indicators_list)

@app.route('/api/explore/', defaults={'check': ''})
@app.route('/api/explore/<check>', methods=['GET', 'POST'])
def api_explore(check):
    form = ExploreForm()
    status = 200
    plot = 0
    tour = 1
    #ind = 76
    #Note: Riaan Snyders: 10 June 2020 - Removed for now. Only functions on GET at the moment.
    #if request.method == 'POST':
        #if form.validate():
            #data_json = request.get_json()
            #ind = data_json["indicator_id"]
    if request.args.get('indicator_id'):
        ind = request.args.get('indicator_id')
    else:
        ind = 76
    print(ind)
    plot = 1
    tour = 2
    # codebook query
    if check == "codebook":
        query = db.session.query(CbRegion.name.label('re_name'), CbDataPoint.start_dt, CbIndicator.name.label('ds_name'), CbDataPoint.value). \
            filter(CbDataPoint.indicator_id == ind).filter(CbDataPoint.indicator_id == CbIndicator.id). \
            filter(CbDataPoint.region_id == CbRegion.id)
        df = read_sql_query(query.statement, query.session.bind)
        df = df.rename(columns={'name': 're_name', 'name.1': 'ds_name'})
        df["year"] = df["start_dt"].apply(lambda x: int(x.strftime('%Y')))
        df["start_dt"] = df["year"]
    else:
        query = db.session.query(Region.re_name, DataPoint.year, DataSet.ds_name, DataPoint.value). \
            filter(DataPoint.indicator_id == ind).filter(DataPoint.dataset_id == DataSet.id). \
            filter(DataPoint.region_id == Region.id)
        df = read_sql_query(query.statement, query.session.bind)
    df.to_csv('%s/data/%s' % (app.root_path, "data_test.csv"), index=False)
    table = []
    table_plot = []
    years, cities, datasets = [list(df.year.unique()), list(df.re_name.unique()), list(df.ds_name.unique())]
    cities = [c for c in cities]

    options_list = [{'optid': i, 'optname': d} for i, d in enumerate(datasets, start=1)]
    years_list = [{'optid': i, 'optname': 'Year: %s' % d} for i, d in enumerate(sorted(years), start=1)]

    plot_type = 1
    print(len(years))
    if (len(datasets) > 1) or (len(years) == 1):
        plot_type = 2

    colours = ['#f44336', '#03a9f4', '#4caf50', '#ffc107', '#03a9f4', '#ff5722', '#9c27b0', '#8bc34a',
               '#ffeb3b', '#9e9e9e', '#3f51b5', '#e91e63']
    series = {i: {'color': colours[i]} for i in range(len(datasets))}
    view = list(range(2, len(datasets) + 2))
    view.insert(0, 0)

    minVal = min(map(float, list(df.value.unique())))
    maxVal = max(map(float, list(df.value.unique()))) * 1.1

    head = ['City', 'Year']
    for i in datasets:
        head.append(str(i))
    table.append(head)
    table_plot.append(head)

    # df.re_name = df.re_name.str.encode('utf-8')
    if plot_type == 1:
        df_i = df.iloc[:, [0, 1, 3]]

        schema = [('City', 'string'), ('Year', 'string'), ('%s' % datasets[0], 'number')]

        data_table = gviz_api.DataTable(schema)
        data_table.LoadData(df_i.values)
        table_plot = data_table.ToJSon(columns_order=('City', '%s' % datasets[0], 'Year'))

        for c in cities:
            for y in years:
                row = [str(c), str(y)]
                for d in datasets:
                    datapoint = df.loc[(df["re_name"] == c) & (df["year"] == y) & (df["ds_name"] == d), "value"]
                    if len(datapoint) == 0:
                        row.append(None)
                    else:
                        row.append(
                            float(df.loc[(df["re_name"] == c) & (df["year"] == y) & (
                            df["ds_name"] == d), "value"]))
                table.append(row)
    else:
        for c in cities:
            for y in years:
                row = [str(c), str(y)]
                for d in datasets:
                    datapoint = df.loc[(df["re_name"] == c) & (df["year"] == y) & (df["ds_name"] == d), "value"]
                    if len(datapoint) == 0:
                        row.append(None)
                    else:
                        row.append(
                            float(df.loc[(df["re_name"] == c) & (df["year"] == y) & (
                            df["ds_name"] == d), "value"]))
                table.append(row)
    yrs = ['Year'] + [str(y) for y in years[::-1]]
    payload = {"plot":plot, "table":table, "table_plot":table_plot,"colours":colours,"year":str(max(years)), "series":series,
             "view":view, "plot_type":plot_type,"min":minVal,"max":maxVal, "cities":cities, "options_list":options_list,
             "years_list":years_list,"tour":tour, "years":yrs}
    return jsonify(payload)
        # else:
        #     form_errors = form.errors
        #     return {"form_errors":form_errors}

@app.route('/explore', methods=['GET', 'POST'])
def explore():
    analyses = []

    if current_user.is_authenticated:
        query = db.session.query(UserAnalysis.id, UserAnalysis.ds_name, UserAnalysis.description) \
            .filter(UserAnalysis.user_id == current_user.id).order_by(UserAnalysis.id.desc())

        analyses = []

        for i in grouper(query, 4):
            analyses.append(i)

    session['explore'] = []
    form = ExploreForm()
    status = 200
    plot = 0
    tour = 1
    if request.method == 'POST':
        if form.validate():
            plot = 1
            tour = 2

            ind = form.indicator_id.data

            query = db.session.query(Region.re_name, DataPoint.year, DataSet.ds_name, DataPoint.value). \
                filter(DataPoint.indicator_id == ind).filter(DataPoint.dataset_id == DataSet.id). \
                filter(DataPoint.region_id == Region.id)
            print(query.all())
            indicator = Indicator.query.get(ind)

            df = read_sql_query(query.statement, query.session.bind)
            # df.to_csv('%s/data/%s' % (app.root_path, "data_test.csv"), index=False)
            table = []
            years, cities, datasets = [list(df.year.unique()), list(df.re_name.unique()), list(df.ds_name.unique())]
            cities = [c for c in cities]

            options_list = [{'optid': i, 'optname': d} for i, d in enumerate(datasets, start=1)]
            years_list = [{'optid': i, 'optname': 'Year: %s' % d} for i, d in enumerate(sorted(years), start=1)]

            plot_type = 1
            if (len(datasets) > 1) or (len(years) == 1):
                plot_type = 2

            colours = ['#f44336', '#03a9f4', '#4caf50', '#ffc107', '#03a9f4', '#ff5722', '#9c27b0', '#8bc34a',
                       '#ffeb3b', '#9e9e9e', '#3f51b5', '#e91e63']
            series = {i: {'color': colours[i]} for i in range(len(datasets))}
            view = list(range(2, len(datasets) + 2))
            view.insert(0, 0)

            minVal = min(map(float, list(df.value.unique())))
            maxVal = max(map(float, list(df.value.unique()))) * 1.1

            head = ['City', 'Year']
            for i in datasets:
                head.append(str(i))
            table.append(head)
            print(df)
            # df.re_name = df.re_name.str.encode('utf-8')
            if plot_type == 1:
                df = df.iloc[:, [0, 1, 3]]

                schema = [('City', 'string'), ('Year', 'string'), ('%s' % datasets[0], 'number')]

                data_table = gviz_api.DataTable(schema)
                data_table.LoadData(df.values)
                table = data_table.ToJSon(columns_order=('City', '%s' % datasets[0], 'Year'))

            else:
                for c in cities:
                    for y in years:
                        row = [str(c), str(y)]
                        for d in datasets:
                            datapoint = df.loc[(df["re_name"] == c) & (df["year"] == y) & (df["ds_name"] == d), "value"]
                            if len(datapoint) == 0:
                                row.append(None)
                            else:
                                row.append(
                                    float(df.loc[(df["re_name"] == c) & (df["year"] == y) & (
                                    df["ds_name"] == d), "value"]))
                        table.append(row)
            yrs = ['Year'] + [str(y) for y in years[::-1]]
            return render_template('explore/explore.html', form=form, plot=plot, table=table, colours=colours,
                                   year=str(max(years)), series=series, view=view, plot_type=plot_type, min=minVal,
                                   max=maxVal, cities=cities, options_list=options_list, years_list=years_list,
                                   tour=tour, indicator=indicator, analyses=analyses, years=yrs)
        else:
            if request.is_xhr:
                status = 412
            else:
                flash('Please correct the problems below and try again.', 'warning')

    else:
        return render_template('explore/explore.html', form=form, tour=tour)

    if not request.is_xhr:
        resp = make_response(
            render_template('explore/explore.html', form=form, plot=plot, tour=tour, analyses=analyses))

    else:
        resp = ''

    return (resp, status,
            # ensure the browser refreshes the page when Back is pressed
            {'Cache-Control': 'no-cache, no-store, must-revalidate'})

@app.route('/demographics/<region_id>/<city_ward_code>/download', methods=['GET'])
def demographics_download(region_id, city_ward_code):
    region = Region.query.get(region_id).re_name

    if city_ward_code == 'None':
        query = db.session.query(Ward.data, Ward.city_ward_code). \
            filter(Ward.region_id == region_id).all()

        df = pd.DataFrame()
        df['Year'] = range(1996, 2031)
        for g in query:
            df['%s - Ward %s' % (region, g[1])] = list(g[0])

    else:
        query = db.session.query(Ward.data, Ward.city_ward_code) \
            .filter(Ward.city_ward_code == city_ward_code) \
            .filter(Ward.region_id == region_id).all()

        df = pd.DataFrame()
        df['Year'] = range(1996, 2031)
        for g in query:
            df['%s - Ward %s' % (region, g[1])] = list(g[0])

    return Response(df.to_csv(index=False), mimetype="text/csv",
                    headers={"Content-disposition": "attachment; filename=demographics.csv"})

@app.route('/demographics', methods=['GET', 'POST'])
def demographics():
    analyses = []

    if current_user.is_authenticated:
        query = db.session.query(UserAnalysis.id, UserAnalysis.ds_name, UserAnalysis.description) \
            .filter(UserAnalysis.user_id == current_user.id).order_by(UserAnalysis.id.desc())

        analyses = []

        for i in grouper(query, 4):
            analyses.append(i)

    session['demo'] = []

    if 'maps' not in session.keys():
        session['maps'] = {0: {}, 1: {}}

    form1 = MapForm(prefix='form1', region_id='1', year=1)
    print(form1.city_ward_code.choices)
    status = 200
    tour = 1
    geometries1 = {}

    forms = [form1]
    if request.method == 'POST':

        if all(f.validate() for f in forms):

            for f, F in enumerate(forms):
                for field in F:
                    if str(field.data) == 'None':
                        field.data = session['maps'][str(f)][field.name[6:]]
                    else:
                        session['maps'][str(f)][field.name[6:]] = field.data

            tour = 0
            # query = db.session.query(Area.geom.ST_AsGeoJSON(), Area.data)
            year1 = int(form1.year.data)
            year_ind1 = range(1996, 2031)

            if form1.city_ward_code.data == '':

                query = db.session.query(Ward.geom.ST_AsGeoJSON(), Ward.data, Ward.city_ward_code). \
                    filter(Ward.region_id == form1.region_id.data)

                geometries1 = {"type": "FeatureCollection",
                               "features": []}
                for g in query:
                    d = json.loads(g[0])

                    if year1 == 0:
                        flow = 0
                    else:
                        flow = round(g[1][year1] - g[1][year1 - 1])

                    geometries1['features'].append({"type": "Feature", "properties": {"density": round(g[1][year1]),
                                                                                "flow": flow,
                                                                                "name": 'Ward %s' % g[2],
                                                                                "year": year_ind1[year1]},
                                                    "geometry": {"type": "Polygon", "coordinates": d['coordinates']}})

                query = db.session.query(Ward.data).filter(Ward.region_id == form1.region_id.data).all()
                region = db.session.query(Region.re_name).filter(Region.id == form1.region_id.data).first()

                results = []

                for r in query:
                    row = [val for val in list(r)[0]]
                    results.append(row)

                df = pd.DataFrame(results).fillna(value=0)

                table1 = [['Year', '%s' % str(region[0])]]

                for y, val in zip(range(1996, 2031), df.sum(axis=0).tolist()):
                    table1.append([str(y), val])

                m1 = 1.05 * max(df.sum(axis=0).tolist())

            else:
                query = db.session.query(Area.geom.ST_AsGeoJSON(), Area.data, Area.city_ward_code) \
                    .filter(Area.city_ward_code == form1.city_ward_code.data) \
                    .filter(Area.region_id == form1.region_id.data)

                geometries1 = {"type": "FeatureCollection",
                               "features": []}

                for g in query:
                    d = json.loads(g[0])

                    if year1 == 0:
                        flow = 0
                    else:
                        flow = round(g[1][year1] - g[1][year1 - 1])

                    geometries1['features'].append(
                        {"type": "Feature", "properties": {"density": round(g[1][year1]),
                                                           "flow": flow,
                                                           "name": 'Area %s' % g[2],
                                                           "year": year_ind1[year1]},
                         "geometry": {"type": "Polygon", "coordinates": d['coordinates']}})

                query = db.session.query(Ward.data).filter(Ward.city_ward_code == form1.city_ward_code.data). \
                    filter(Ward.region_id == form1.region_id.data).first()

                region = db.session.query(Region.re_name).filter(Region.id == form1.region_id.data).first()
                region2 = db.session.query(Ward.city_ward_code).filter(Ward.city_ward_code == form1.city_ward_code.data) \
                    .first()

                results = []

                for r in query:
                    row = [val for val in list(r)]
                    results.append(row)

                df = pd.DataFrame(results).fillna(value=0)

                table1 = [['Year', '%s - Ward %s' % (str(region[0]), str(region2[0]))]]

                for y, val in zip(range(1996, 2031), df.sum(axis=0).tolist()):
                    table1.append([str(y), val])

                m1 = 1.05 * max(df.sum(axis=0).tolist())

            query = db.session.query(Ward.city_ward_code).filter(Ward.region_id == form1.region_id.data).order_by(
                Ward.city_ward_code).distinct()

            form1.city_ward_code.choices = [[str(i), 'Ward %s' % row.city_ward_code] for i, row in enumerate(query.all()
                                                                                , start=1)]
            form1.city_ward_code.choices.insert(0, ('', 'View All'))

            return render_template('demographics/demographics.html', form1=form1, geometries1=geometries1,
                                   table1=table1, tour=tour, max1=m1, region1=form1.region_id.data,
                                   ward1=form1.city_ward_code.data, analyses=analyses)

        else:
            if request.is_xhr:
                status = 412
            else:
                flash('Please correct the problems below and try again.', 'warning')

    else:

        session['maps'][0] = {'city_ward_code': '', 'region_id': 1, 'year': 1}
        session['maps'][1] = {'city_ward_code': '', 'region_id': 4, 'year': 1}

        query = db.session.query(Ward.geom.ST_AsGeoJSON(), Ward.data, Ward.city_ward_code). \
            filter(Ward.region_id == 1)

        geometries1 = {"type": "FeatureCollection",
                       "features": []}

        geometries2 = {"type": "FeatureCollection",
                       "features": []}

        for g in query:
            d = json.loads(g[0])
            geometries1['features'].append({"type": "Feature", "properties": {"density": round(g[1][1]),
                                                                              "flow": round(g[1][1] - g[1][0]),
                                                                              "name": 'Ward %s' % g[2],
                                                                              "year": 1997},
                                            "geometry": {"type": "Polygon", "coordinates": d['coordinates']}})

        query = db.session.query(Ward.data).filter(Ward.region_id == 1).all()

        results = []

        for r in query:
            row = [val for val in list(r)[0]]
            results.append(row)

        df = pd.DataFrame(results).fillna(value=0)

        table1 = [['Year', 'Johannesburg']]

        for y, val in zip(range(1996, 2031), df.sum(axis=0).tolist()):
            table1.append([str(y), val])

        m = 1.05 * max(df.sum(axis=0).tolist())

        query = db.session.query(Ward.geom.ST_AsGeoJSON(), Ward.data, Ward.city_ward_code). \
            filter(Ward.region_id == 4)

        for g in query:
            d = json.loads(g[0])
            geometries2['features'].append({"type": "Feature", "properties": {"density": round(g[1][1]),
                                                                              "flow": round(g[1][1] - g[1][0]),
                                                                              "name": 'Ward %s' % g[2],
                                                                              "year": 1997},
                                            "geometry": {"type": "Polygon", "coordinates": d['coordinates']}})

        query = db.session.query(Ward.data).filter(Ward.region_id == 4).all()

        results = []

        for r in query:
            row = [val for val in list(r)[0]]
            results.append(row)

        df = pd.DataFrame(results).fillna(value=0)

        table2 = [['Year', 'EThekwini']]

        for y, val in zip(range(1996, 2031), df.sum(axis=0).tolist()):
            table2.append([str(y), val])

        m2 = 1.05 * max(df.sum(axis=0).tolist())

        return render_template('demographics/demographics.html', form1=form1, geometries1=geometries1,
                               tour=tour, table1=table1, max1=m, region1=1, ward1=None, ward2=None, analyses=analyses
                               )

    if not request.is_xhr:
        query = db.session.query(Ward.geom.ST_AsGeoJSON(), Ward.data, Ward.city_ward_code). \
            filter(Ward.region_id == 1)
        geometries1 = {"type": "FeatureCollection",
                       "features": []}

        geometries2 = {"type": "FeatureCollection",
                       "features": []}

        for g in query:
            d = json.loads(g[0])
            geometries1['features'].append(
                {"type": "Feature", "properties": {"density": round(g[1][0]), "flow": 0, "name": g[2],
                                                   "year": 1996},
                 "geometry": {"type": "Polygon", "coordinates": d['coordinates']}})

            geometries2['features'].append(
                {"type": "Feature", "properties": {"density": round(g[1][0]), "flow": 0, "name": g[2],
                                                   "year": 1996},
                 "geometry": {"type": "Polygon", "coordinates": d['coordinates']}})

        query = db.session.query(Ward.data).filter(Ward.region_id == 1).all()

        results = []

        for r in query:
            row = [val for val in list(r)[0]]
            results.append(row)

        df = pd.DataFrame(results).fillna(value=0)

        table1 = [['Year', 'Johannesburg']]

        for y, val in zip(range(1996, 2031), df.sum(axis=0).tolist()):
            table1.append([str(y), val])

        m = 1.05 * max(df.sum(axis=0).tolist())

        resp = make_response(render_template('demographics/demographics.html', form1=form1,
                                             geometries1=geometries1, table1=table1,
                                             tour=tour, max1=m, region1=1,
                                             ward1=None, analyses=analyses))

    else:
        resp = ''

    return (resp, status,
            # ensure the browser refreshes the page when Back is pressed
            {'Cache-Control': 'no-cache, no-store, must-revalidate'})

@app.route('/api/demographics', methods=['GET', 'POST'])
@csrf.exempt
def api_demographics():
    analyses = []

    session['demo'] = []

    if 'maps' not in session.keys():
        session['maps'] = {0: {}, 1: {}}

    form1 = MapForm(prefix='form1', region_id='1', year=1)
    geometries1 = {}

    if request.method == 'POST':
        data = request.get_json()
        print(data)
        #data = request.data.decode('utf-8')
        #object = parse_qs(urlsplit('?' + data).query)
        #object = {key: str(value[0]) for key, value in object.items()}

        #if 'csrf_token' in object: del object['csrf_token']
        #form1 = MapForm(MultiDict(object))
        form1 = data

        print(form1['year'])

        #if form1.validate():
        if form1:
            tour = 0
            # query = db.session.query(Area.geom.ST_AsGeoJSON(), Area.data)
            #year1 = int(form1.year)
            year1 = int(form1['year'])
            year_ind1 = range(1996, 2031)

            #if form1.city_ward_code.data == '':
            if form1['city_ward_code'] == '':
                query = db.session.query(Ward.geom.ST_AsGeoJSON(), Ward.data, Ward.city_ward_code). \
                    filter(Ward.region_id == form1['region_id'])

                geometries1 = {"type": "FeatureCollection",
                               "features": []}
                for g in query:
                    d = json.loads(g[0])

                    if year1 == 0:
                        flow = 0
                    else:
                        flow = round(g[1][year1] - g[1][year1 - 1])

                    geometries1['features'].append({"type": "Feature", "properties": {"density": round(g[1][year1]),
                                                                                "flow": flow,
                                                                                "name": 'Ward %s' % g[2],
                                                                                "year": year_ind1[year1]},
                                                    "geometry": {"type": "Polygon", "coordinates": d['coordinates']}})

                query = db.session.query(Ward.data).filter(Ward.region_id == form1['region_id']).all()
                region = db.session.query(Region.re_name).filter(Region.id == form1['region_id']).first()

                results = []

                for r in query:
                    row = [val for val in list(r)[0]]
                    results.append(row)

                df = pd.DataFrame(results).fillna(value=0)

                table1 = [['Year', '%s' % str(region[0])]]

                for y, val in zip(range(1996, 2031), df.sum(axis=0).tolist()):
                    table1.append([str(y), val])

                m1 = 1.05 * max(df.sum(axis=0).tolist())

            else:
                query = db.session.query(Area.geom.ST_AsGeoJSON(), Area.data, Area.city_ward_code) \
                    .filter(Area.city_ward_code == int(form1['city_ward_code'])) \
                    .filter(Area.region_id == int(form1['region_id']))

                geometries1 = {"type": "FeatureCollection",
                               "features": []}

                for g in query:
                    d = json.loads(g[0])

                    if year1 == 0:
                        flow = 0
                    else:
                        flow = round(g[1][year1] - g[1][year1 - 1])

                    geometries1['features'].append(
                        {"type": "Feature", "properties": {"density": round(g[1][year1]),
                                                           "flow": flow,
                                                           "name": 'Area %s' % g[2],
                                                           "year": year_ind1[year1]},
                         "geometry": {"type": "Polygon", "coordinates": d['coordinates']}})

                query = db.session.query(Ward.data).filter(Ward.city_ward_code == int(form1['city_ward_code'])). \
                    filter(Ward.region_id == int(form1['region_id'])).first()

                region = db.session.query(Region.re_name).filter(Region.id == int(form1['region_id'])).first()
                region2 = db.session.query(Ward.city_ward_code).filter(Ward.city_ward_code == int(form1['city_ward_code'])) \
                    .first()

                results = []

                for r in query:
                    row = [val for val in list(r)]
                    results.append(row)

                df = pd.DataFrame(results).fillna(value=0)

                table1 = [['Year', '%s - Ward %s' % (str(region[0]), str(region2[0]))]]

                for y, val in zip(range(1996, 2031), df.sum(axis=0).tolist()):
                    table1.append([str(y), val])

                m1 = 1.05 * max(df.sum(axis=0).tolist())

            query = db.session.query(Ward.city_ward_code).filter(Ward.region_id == int(form1['region_id'])).order_by(
                Ward.city_ward_code).distinct()

            #form1.city_ward_code.choices = [[str(i), 'Ward %s' % row.city_ward_code] for i, row in enumerate(query.all()
                                                                                #, start=1)]
            #form1.city_ward_code.choices.insert(0, ('', 'View All'))

            resp = jsonify({'success': True, 'geometries1': geometries1,'table1':table1,
                            'tour':tour, 'max1':m1, 'region1':form1['region_id'],'ward1':form1['city_ward_code']})
            resp.status_code = 200
            return resp
        else:
            message = 'Please correct the problems below and try again.'
            resp = jsonify(message=message)
            resp.status_code = 500
            return resp

    else:
        session['maps'][0] = {'city_ward_code': '', 'region_id': 1, 'year': 1}
        session['maps'][1] = {'city_ward_code': '', 'region_id': 4, 'year': 1}

        query = db.session.query(Ward.geom.ST_AsGeoJSON(), Ward.data, Ward.city_ward_code). \
            filter(Ward.region_id == 1)

        geometries1 = {"type": "FeatureCollection",
                       "features": []}

        for g in query:
            d = json.loads(g[0])
            geometries1['features'].append({"type": "Feature", "properties": {"density": round(g[1][1]),
                                                                              "flow": round(g[1][1] - g[1][0]),
                                                                              "name": 'Ward %s' % g[2],
                                                                              "year": 1997},
                                            "geometry": {"type": "Polygon", "coordinates": d['coordinates']}})

        query = db.session.query(Ward.data).filter(Ward.region_id == 1).all()

        results = []

        for r in query:
            row = [val for val in list(r)[0]]
            results.append(row)

        df = pd.DataFrame(results).fillna(value=0)

        table1 = [['Year', 'Johannesburg']]

        for y, val in zip(range(1996, 2031), df.sum(axis=0).tolist()):
            table1.append([str(y), val])

        m = 1.05 * max(df.sum(axis=0).tolist())

        resp = jsonify({'success': True, 'table1': table1,
                         'max1': m, 'region1': 1, 'ward1': None,'ward2':None, 'geometries1': geometries1,
                        'form_year':form1.year.choices,'form_ward':form1.city_ward_code.choices,'form_city':form1.region_id.choices})
        resp.status_code = 200
        return resp

@app.route('/nightlights_jhb', methods=['GET', 'POST'])
def demographics_night_jhb():
    analyses = []

    if current_user.is_authenticated:
        query = db.session.query(UserAnalysis.id, UserAnalysis.ds_name, UserAnalysis.description) \
            .filter(UserAnalysis.user_id == current_user.id).order_by(UserAnalysis.id.desc())

        analyses = []

        for i in grouper(query, 4):
            analyses.append(i)

    session['night'] = []

    form = NightFormJHB()
    status = 200
    tour = 1

    if request.method == 'POST':

        if form.validate():

            tour = 0

            if form.city_ward_code.data == '':

                query = db.session.query(Grid.geom.ST_AsGeoJSON(), Grid.data, Grid.city_grid_id, Grid.reference). \
                    filter(Grid.region_id == 1)

                geometries = {"type": "FeatureCollection",
                              "features": []}

                bias_ind = [x / 10.0 for x in range(5, 21, 1)].index(float(form.grid_bias.data))

                for g in query:

                    d = json.loads(g[0])

                    geometries['features'].append({"type": "Feature", "properties": {"density": round(g[1][bias_ind] - g[3]),
                                                                                "name": 'Grid %s' % g[2],
                                                                                "year": 2016},
                                                   "geometry": {"type": "Polygon", "coordinates": d['coordinates']}})

                query = db.session.query(Ward.city_ward_code).filter(Ward.region_id == 1).order_by(
                    Ward.city_ward_code).distinct()

                form.city_ward_code.choices = [[str(i), 'Ward %s' % row.city_ward_code] for i, row in
                                               enumerate(query.all()
                                                         , start=1)]
                form.city_ward_code.choices.insert(0, ('', 'View All'))

                return render_template('demographics/demographics_night.html', form=form, geometries=geometries,
                                       bias_val=form.grid_bias.data)

            else:

                w = db.session.query(Ward.id).filter(Ward.city_ward_code == form.city_ward_code.data)\
                    .filter(Ward.region_id == 1).first()

                w = Ward.query.get(w[0])

                query = db.session.query(Grid.geom.ST_AsGeoJSON(), Grid.data, Grid.city_grid_id, Grid.reference) \
                    .filter(Grid.geom.intersects(w.geom))

                geometries = {"type": "FeatureCollection",
                              "features": []}

                bias_ind = [x / 10.0 for x in range(5, 21, 1)].index(float(form.grid_bias.data))

                for g in query:
                    d = json.loads(g[0])

                    geometries['features'].append(
                        {"type": "Feature", "properties": {"density": round(g[1][bias_ind] - g[3]),
                                                           "name": 'Grid %s' % g[2],
                                                           "year": 2016},
                         "geometry": {"type": "Polygon", "coordinates": d['coordinates']}})

                query = db.session.query(Ward.geom.ST_AsGeoJSON(), Ward.data, Ward.city_ward_code)\
                    .filter(Ward.city_ward_code == form.city_ward_code.data).filter(Ward.region_id == 1)

                geometries2 = {"type": "FeatureCollection",
                               "features": []}

                for g in query:
                    d = json.loads(g[0])

                    geometries2['features'].append(
                        {"type": "Feature", "properties": {"density": 0,
                                                           "name": 'Ward %s' % form.city_ward_code.data,
                                                           "year": 2016},
                         "geometry": {"type": "Polygon", "coordinates": d['coordinates']}})

                query = db.session.query(Ward.city_ward_code).filter(Ward.region_id == 1).order_by(
                    Ward.city_ward_code).distinct()

                form.city_ward_code.choices = [[str(i), 'Ward %s' % row.city_ward_code] for i, row in enumerate(query.all()
                                                                                , start=1)]
                form.city_ward_code.choices.insert(0, ('', 'View All'))

                return render_template('demographics/demographics_night.html', form=form, geometries=geometries,
                                   bias_val=form.grid_bias.data, geometries2=geometries2, ward=form.city_ward_code.data)

        else:
            if request.is_xhr:
                status = 412
            else:
                flash('Please correct the problems below and try again.', 'warning')

    else:

        query = db.session.query(Grid.geom.ST_AsGeoJSON(), Grid.data, Grid.city_grid_id, Grid.reference). \
            filter(Grid.region_id == 1)

        geometries = {"type": "FeatureCollection",
                      "features": []}

        for g in query:
            d = json.loads(g[0])

            geometries['features'].append({"type": "Feature", "properties": {"density": g[1][0] - g[3],
                                                                             "name": 'Grid %s' % g[2],
                                                                             "year": 2016},
                                           "geometry": {"type": "Polygon", "coordinates": d['coordinates']}})

        return render_template('demographics/demographics_night.html', form=form, bias_val=0.5, geometries=geometries,
                               analyses=analyses)

    if not request.is_xhr:
        query = db.session.query(Ward.geom.ST_AsGeoJSON(), Ward.data, Ward.city_ward_code). \
            filter(Ward.region_id == 1)
        geometries1 = {"type": "FeatureCollection",
                       "features": []}

        geometries2 = {"type": "FeatureCollection",
                       "features": []}

        for g in query:
            d = json.loads(g[0])
            geometries1['features'].append(
                {"type": "Feature", "properties": {"density": round(g[1][0]), "flow": 0, "name": g[2],
                                                   "year": 1996},
                 "geometry": {"type": "Polygon", "coordinates": d['coordinates']}})

            geometries2['features'].append(
                {"type": "Feature", "properties": {"density": round(g[1][0]), "flow": 0, "name": g[2],
                                                   "year": 1996},
                 "geometry": {"type": "Polygon", "coordinates": d['coordinates']}})

        query = db.session.query(Ward.data).filter(Ward.region_id == 1).all()

        results = []

        for r in query:
            row = [val for val in list(r)[0]]
            results.append(row)

        df = pd.DataFrame(results).fillna(value=0)

        table1 = [['Year', 'Johannesburg']]

        for y, val in zip(range(1996, 2031), df.sum(axis=0).tolist()):
            table1.append([str(y), val])

        m = 1.05 * max(df.sum(axis=0).tolist())

        resp = make_response(render_template('demographics/demographics.html', form1=form1, form2=form2,
                                             geometries1=geometries1, geometries2=geometries2, table1=table1,
                                             table2=table1, tour=tour, max1=m, max2=m, region1=1, region2=1,
                                             ward1=None, ward2=None, analyses=analyses))

    else:
        resp = ''

    return (resp, status,
            # ensure the browser refreshes the page when Back is pressed
            {'Cache-Control': 'no-cache, no-store, must-revalidate'})

@app.route('/nightlights_eth', methods=['GET', 'POST'])
def demographics_night_eth():
    analyses = []

    if current_user.is_authenticated:
        query = db.session.query(UserAnalysis.id, UserAnalysis.ds_name, UserAnalysis.description) \
            .filter(UserAnalysis.user_id == current_user.id).order_by(UserAnalysis.id.desc())

        analyses = []

        for i in grouper(query, 4):
            analyses.append(i)

    session['night'] = []

    form = NightFormETH()
    status = 200
    tour = 1

    if request.method == 'POST':

        if form.validate():

            tour = 0

            if form.city_ward_code.data == '':

                query = db.session.query(Grid.geom.ST_AsGeoJSON(), Grid.data, Grid.city_grid_id, Grid.reference). \
                    filter(Grid.region_id == 4)

                geometries = {"type": "FeatureCollection",
                              "features": []}

                for g in query:

                    d = json.loads(g[0])

                    geometries['features'].append({"type": "Feature", "properties": {"density": round(g[1][0]-g[3]),
                                                                                "name": 'Grid %s' % g[2],
                                                                                "year": 2016},
                                                   "geometry": {"type": "Polygon", "coordinates": d['coordinates']}})

                query = db.session.query(Ward.city_ward_code).filter(Ward.region_id == 4).order_by(
                    Ward.city_ward_code).distinct()

                form.city_ward_code.choices = [[str(i), 'Ward %s' % row.city_ward_code] for i, row in
                                               enumerate(query.all()
                                                         , start=1)]
                form.city_ward_code.choices.insert(0, ('', 'View All'))

                return render_template('demographics/demographics_night_ETH.html', form=form, geometries=geometries)

            else:

                w = db.session.query(Ward.id).filter(Ward.city_ward_code == form.city_ward_code.data)\
                    .filter(Ward.region_id == 4).first()

                w = Ward.query.get(w[0])

                query = db.session.query(Grid.geom.ST_AsGeoJSON(), Grid.data, Grid.city_grid_id, Grid.reference) \
                    .filter(Grid.geom.intersects(w.geom)).filter(Grid.region_id == 4)

                geometries = {"type": "FeatureCollection",
                              "features": []}

                for g in query:
                    d = json.loads(g[0])

                    geometries['features'].append(
                        {"type": "Feature", "properties": {"density": round(g[1][0]-g[3]),
                                                           "name": 'Grid %s' % g[2],
                                                           "year": 2016},
                         "geometry": {"type": "Polygon", "coordinates": d['coordinates']}})

                query = db.session.query(Ward.geom.ST_AsGeoJSON(), Ward.data, Ward.city_ward_code)\
                    .filter(Ward.city_ward_code == form.city_ward_code.data).filter(Ward.region_id == 4)

                geometries2 = {"type": "FeatureCollection",
                               "features": []}

                for g in query:
                    d = json.loads(g[0])

                    geometries2['features'].append(
                        {"type": "Feature", "properties": {"density": 0,
                                                           "name": 'Ward %s' % form.city_ward_code.data,
                                                           "year": 2016},
                         "geometry": {"type": "Polygon", "coordinates": d['coordinates']}})

                query = db.session.query(Ward.city_ward_code).filter(Ward.region_id == 4).order_by(
                    Ward.city_ward_code).distinct()

                form.city_ward_code.choices = [[str(i), 'Ward %s' % row.city_ward_code] for i, row in enumerate(query.all()
                                                                                , start=1)]
                form.city_ward_code.choices.insert(0, ('', 'View All'))

                return render_template('demographics/demographics_night_ETH.html', form=form, geometries=geometries,
                                   geometries2=geometries2, ward=form.city_ward_code.data)

        else:
            if request.is_xhr:
                status = 412
            else:
                flash('Please correct the problems below and try again.', 'warning')

    else:

        query = db.session.query(Grid.geom.ST_AsGeoJSON(), Grid.data, Grid.city_grid_id, Grid.reference). \
            filter(Grid.region_id == 4)

        geometries = {"type": "FeatureCollection",
                      "features": []}

        for g in query:
            d = json.loads(g[0])

            geometries['features'].append({"type": "Feature", "properties": {"density": round(g[1][0]-g[3]),
                                                                             "name": 'Grid %s' % g[2],
                                                                             "year": 2016},
                                           "geometry": {"type": "Polygon", "coordinates": d['coordinates']}})

        return render_template('demographics/demographics_night_ETH.html', form=form, geometries=geometries,
                               analyses=analyses)

    if not request.is_xhr:
        query = db.session.query(Ward.geom.ST_AsGeoJSON(), Ward.data, Ward.city_ward_code). \
            filter(Ward.region_id == 1)
        geometries1 = {"type": "FeatureCollection",
                       "features": []}

        geometries2 = {"type": "FeatureCollection",
                       "features": []}

        for g in query:
            d = json.loads(g[0])
            geometries1['features'].append(
                {"type": "Feature", "properties": {"density": round(g[1][0]), "flow": 0, "name": g[2],
                                                   "year": 1996},
                 "geometry": {"type": "Polygon", "coordinates": d['coordinates']}})

            geometries2['features'].append(
                {"type": "Feature", "properties": {"density": round(g[1][0]), "flow": 0, "name": g[2],
                                                   "year": 1996},
                 "geometry": {"type": "Polygon", "coordinates": d['coordinates']}})

        query = db.session.query(Ward.data).filter(Ward.region_id == 1).all()

        results = []

        for r in query:
            row = [val for val in list(r)[0]]
            results.append(row)

        df = pd.DataFrame(results).fillna(value=0)

        table1 = [['Year', 'Johannesburg']]

        for y, val in zip(range(1996, 2031), df.sum(axis=0).tolist()):
            table1.append([str(y), val])

        m = 1.05 * max(df.sum(axis=0).tolist())

        resp = make_response(render_template('demographics/demographics.html', form1=form1, form2=form2,
                                             geometries1=geometries1, geometries2=geometries2, table1=table1,
                                             table2=table1, tour=tour, max1=m, max2=m, region1=1, region2=1,
                                             ward1=None, ward2=None, analyses=analyses))

    else:
        resp = ''

    return (resp, status,
            # ensure the browser refreshes the page when Back is pressed
            {'Cache-Control': 'no-cache, no-store, must-revalidate'})

@app.route('/return-land/')
def land_gen():

    return send_file('data/Ethekwini_Region_Data.xlsx', as_attachment=True)

@app.route('/_parse_data', methods=['GET'])
def parse_data():
    kwargs = {}
    for i in ['dataset_id', 'indicator_id', 'region_id', 'type_id', 'theme_id', 'year']:
        param = request.args.get(i)
        if (i == 'year'):
            if (str(param) != 'Empty') and (param is not None) and (str(param) != ''):
                kwargs[i] = int(param)
            else:
                pass

        elif (param is not None) and (str(param) != ''):
            kwargs[i] = param

    session['explore'] = [i for i in kwargs]

    datasets = db.session.query(DataPoint.dataset_id).filter_by(**kwargs).distinct()
    indicators = db.session.query(DataPoint.indicator_id).filter_by(**kwargs).distinct()
    regions = db.session.query(DataPoint.region_id).filter_by(**kwargs).distinct()
    types = db.session.query(DataPoint.type_id).filter_by(**kwargs).distinct()
    themes = db.session.query(DataPoint.theme_id).filter_by(**kwargs).distinct()
    years = db.session.query(DataPoint.year).filter_by(**kwargs).distinct()

    response = {}

    remove_list = ['Poverty rate', 'Gini Coefficient', 'Gross Value Add', 'Exports', 'Multiple deprivation index',
                   'Human Development Index']

    dataset_list = [(i[0], str(DataSet.query.filter_by(id=i).first().ds_name)) for i in datasets if
                    str(DataSet.query.filter_by(id=i).first().ds_name) not in remove_list]
    if 'dataset_id' not in session['explore']:
        dataset_list.insert(0, ('', 'Empty'))
    else:
        dataset_list.insert(1, ('', 'Empty'))

    response['dataset'] = dataset_list

    indicator_list = [[i[0], str(Indicator.query.filter_by(id=i).first().in_name)] for i in indicators if
                      str(Indicator.query.filter_by(id=i).first().in_name) not in remove_list]

    if 'indicator_id' not in session['explore']:
        indicator_list.insert(0, ('', 'Empty'))
        response['ind_ready'] = 0
    else:
        indicator_list.insert(1, ('', 'Empty'))
        response['ind_ready'] = 1

    response['indicator'] = indicator_list
    region_list = [(i[0], str(Region.query.filter_by(id=i).first().re_name)) for i in regions]

    if 'region_id' not in session['explore']:
        region_list.insert(0, ('', 'Empty'))
    else:
        region_list.insert(1, ('', 'Empty'))

    response['region'] = region_list

    type_list = [(i[0], str(Type.query.filter_by(id=i).first().ty_name)) for i in types]

    if 'type_id' not in session['explore']:
        type_list.insert(0, ('', 'Empty'))
    else:
        type_list.insert(1, ('', 'Empty'))

    response['type'] = type_list

    theme_list = [(i[0], str(Theme.query.filter_by(id=i).first().th_name)) for i in themes]

    if 'theme_id' not in session['explore']:
        theme_list.insert(0, ('', 'Empty'))
    else:
        theme_list.insert(1, ('', 'Empty'))

    response['theme'] = theme_list

    year_list = [(str(i), str(y[0])) for i, y in enumerate(sorted(years))]

    if 'year' not in session['explore']:
        year_list.insert(0, ('', 'Empty'))
    else:
        year_list.insert(1, ('', 'Empty'))

    response['year'] = year_list

    return jsonify(response)

@app.route('/_parse_demo', methods=['GET'])
def parse_demo():
    kwargs = {}
    for i in ['region_id', 'ward_id']:
        param = request.args.get(i)
        if (param is not None) and (str(param) != ''):
            kwargs[i] = param

    session['demo'] = [i for i in kwargs]

    wards = db.session.query(Ward.city_ward_code).filter_by(**kwargs).distinct().order_by(Ward.city_ward_code)

    response = {}

    ward_list = [(str(i[0]), 'Ward %s' % Ward.query.filter_by(id=i).first().city_ward_code) for i in wards]

    if 'ward_id' not in session['demo']:
        ward_list.insert(0, ('', 'View All'))
    else:
        ward_list.insert(1, ('', 'View All'))

    response['wards'] = ward_list

    return jsonify(response)

@app.route('/api/codebook', methods=['GET', 'POST'])
@app.route('/api/codebook/<int:page>', methods=['GET', 'POST'])
@csrf.exempt
def api_codebook(page=1):
    query = db.session.query(CbIndicator). \
        outerjoin(CbTheme, CbTheme.id == CbIndicator.theme_id). \
        outerjoin(CbSource, CbSource.id == CbIndicator.source_id). \
        outerjoin(CbUnit, CbUnit.id == CbIndicator.unit_id)

    if request.method == 'POST':
        data = request.get_json()

        if data['c88']:
            query = query.filter(CbIndicator.c88_theme.in_(data['c88']))

        if data['socr']:
            query = query.filter(CbIndicator.socr_theme.in_(data['socr']))

        if data['sdg']:
            query = query.filter(CbIndicator.sdg_theme.in_(data['sdg']))

        if data['search']:
            query = search(query, data['search'], sort=True)

    else:
        query = query.limit(150).offset((page - 1) * 20)

    row_count = query.count()
    query = query.all()
    # query.sort(key=lambda x: x.code)

    result_list = [row_count]
    for day, dicts_for_group_code in itertools.groupby(query, key=lambda x:x.group_code):
        dicts_for_group_code = list(dicts_for_group_code)
        day_dict = {
            "id": str(dicts_for_group_code[0].id),
            "varCode": dicts_for_group_code[0].code,
            "groupCode": dicts_for_group_code[0].group_code,
            "indicator": dicts_for_group_code[0].name,
            "c88": dicts_for_group_code[0].c88_theme,
            "socr": dicts_for_group_code[0].socr_theme,
            "sdg": dicts_for_group_code[0].sdg_theme,
            "definition": dicts_for_group_code[0].definition,
            "source": dicts_for_group_code[0].source.name if dicts_for_group_code[0].source else None,
            "reportingResponsibility": dicts_for_group_code[0].reporting_responsibility,
            "notesOnCalculation": dicts_for_group_code[0].notes_on_calculation,
            "variableType": dicts_for_group_code[0].unit.name,
            "frequencyOfCollection": dicts_for_group_code[0].frequency_of_collection,
            "automatibility": dicts_for_group_code[0].automatable,
            "granulity": dicts_for_group_code[0].granularity,
            "gathering_method": dicts_for_group_code[0].gathering_method,
            "expandability": dicts_for_group_code[0].expandable,
            "period": dicts_for_group_code[0].period,
            "unit_of_measurement": dicts_for_group_code[0].unit.name,
            "source_link": dicts_for_group_code[0].url_link,
            "data_check":True if dicts_for_group_code[0].indicator_data else False
        }
        children = []
        dicts_for_group_code.pop(0)
        for d in dicts_for_group_code:
            child = {
                "id": str(d.id),
                "varCode": d.code,
                "groupCode": d.group_code,
                "indicator": d.name,
                "c88": d.c88_theme,
                "socr": d.socr_theme,
                "sdg": d.sdg_theme,
                "definition": d.definition,
                "source": d.source.name if d.source else None,
                "reportingResponsibility": d.reporting_responsibility,
                "notesOnCalculation": d.notes_on_calculation,
                "variableType": d.unit.name,
                "frequencyOfCollection": d.frequency_of_collection,
                "automatibility": d.automatable,
                "granulity": d.granularity,
                "gathering_method": d.gathering_method,
                "expandability": d.expandable,
                "period": d.period,
                "unit_of_measurement": d.unit.name,
                "source_link": d.url_link,
                "data_check": bool(d.indicator_data),
            }

            children.append(child)
        day_dict.update({"children": children})
        result_list.append(day_dict)

    return jsonify(result_list)


import numpy as np
import matplotlib.pyplot as plt
from scipy.stats import linregress
from scipy.optimize import curve_fit

oberwelle3, amplitude3 = np.genfromtxt('Rohdaten/dreieckspannung.txt',unpack=True)
plt.plot(oberwelle3, amplitude3,'k.',label="Messdaten")
#plt.legend(loc='best')
plt.grid()
#plt.xlim(0,1.5)
plt.xscale('log')
plt.yscale('log')
plt.xlabel(r'Oberwellen')
plt.ylabel(r'$\mathrm{U}/V$')
plt.tight_layout()
#plt.show()
#plt.savefig('build/50ohm.pdf')

####################
newX = np.logspace(-4, 1, base=10)  # Makes a nice domain for the fitted curves.

                                   # This avoids the sorting and the swarm of lines.

# Let's fit an exponential function.
# This looks like a line on a lof-log plot.
def myExpFunc(x, a, b):
    return a * np.power(x, b)
popt, pcov = curve_fit(myExpFunc, oberwelle3, amplitude3)
plt.plot(newX, myExpFunc(newX, *popt), 'r-',
         label="Fit".format(*popt))
plt.xlim(10**-2, 10**1)
plt.ylim(10**-0.5, 10**1)
print('Exponential Fit: y = (a*(x**b))')
print('\ta = popt[0] = {0}\n\tb = popt[1] = {1}'.format(*popt))
####################

plt.legend(loc='best')
#plt.show()
plt.savefig('build/dreieckspannung.pdf')


"""
==================================================
Plot different SVM classifiers in the iris dataset
==================================================

Comparison of different linear SVM classifiers on a 2D projection of the iris
dataset. We only consider the first 2 features of this dataset:

- Sepal length
- Sepal width

This example shows how to plot the decision surface for four SVM classifiers
with different kernels.

The linear models ``LinearSVC()`` and ``SVC(kernel='linear')`` yield slightly
different decision boundaries. This can be a consequence of the following
differences:

- ``LinearSVC`` minimizes the squared hinge loss while ``SVC`` minimizes the
  regular hinge loss.

- ``LinearSVC`` uses the One-vs-All (also known as One-vs-Rest) multiclass
  reduction while ``SVC`` uses the One-vs-One multiclass reduction.

Both linear models have linear decision boundaries (intersecting hyperplanes)
while the non-linear kernel models (polynomial or Gaussian RBF) have more
flexible non-linear decision boundaries with shapes that depend on the kind of
kernel and its parameters.

.. NOTE:: while plotting the decision function of classifiers for toy 2D
   datasets can help get an intuitive understanding of their respective
   expressive power, be aware that those intuitions don't always generalize to
   more realistic high-dimensional problem.

"""
print(__doc__)

import numpy as np
import matplotlib.pyplot as plt
from sklearn import svm, datasets

# import some data to play with
iris = datasets.load_iris()
X = iris.data[:, :2]  # we only take the first two features. We could
                      # avoid this ugly slicing by using a two-dim dataset
y = iris.target

h = .02  # step size in the mesh

# we create an instance of SVM and fit out data. We do not scale our
# data since we want to plot the support vectors
C = 1.0  # SVM regularization parameter
svc = svm.SVC(kernel='linear', C=C).fit(X, y)
rbf_svc = svm.SVC(kernel='rbf', gamma=0.7, C=C).fit(X, y)
poly_svc = svm.SVC(kernel='poly', degree=3, C=C).fit(X, y)
lin_svc = svm.LinearSVC(C=C).fit(X, y)

# create a mesh to plot in
x_min, x_max = X[:, 0].min() - 1, X[:, 0].max() + 1
y_min, y_max = X[:, 1].min() - 1, X[:, 1].max() + 1
xx, yy = np.meshgrid(np.arange(x_min, x_max, h),
                     np.arange(y_min, y_max, h))

# title for the plots
titles = ['SVC with linear kernel',
          'LinearSVC (linear kernel)',
          'SVC with RBF kernel',
          'SVC with polynomial (degree 3) kernel']

for i, clf in enumerate((svc, lin_svc, rbf_svc, poly_svc)):
    # Plot the decision boundary. For that, we will assign a color to each
    # point in the mesh [x_min, m_max]x[y_min, y_max].
    plt.subplot(2, 2, i + 1)
    plt.subplots_adjust(wspace=0.4, hspace=0.4)

    Z = clf.predict(np.c_[xx.ravel(), yy.ravel()])

    # Put the result into a color plot
    Z = Z.reshape(xx.shape)
    plt.contourf(xx, yy, Z, cmap=plt.cm.Paired, alpha=0.8)

    # Plot also the training points
    plt.scatter(X[:, 0], X[:, 1], c=y, cmap=plt.cm.Paired)
    plt.xlabel('Sepal length')
    plt.ylabel('Sepal width')
    plt.xlim(xx.min(), xx.max())
    plt.ylim(yy.min(), yy.max())
    plt.xticks(())
    plt.yticks(())
    plt.title(titles[i])

plt.show()


"""
=====================================================================
The Johnson-Lindenstrauss bound for embedding with random projections
=====================================================================

The `Johnson-Lindenstrauss lemma`_ states that any high dimensional
dataset can be randomly projected into a lower dimensional Euclidean
space while controlling the distortion in the pairwise distances.

.. _`Johnson-Lindenstrauss lemma`: http://en.wikipedia.org/wiki/Johnson%E2%80%93Lindenstrauss_lemma

Theoretical bounds
==================

The distortion introduced by a random projection `p` is asserted by
the fact that `p` is defining an eps-embedding with good probability
as defined by:

  (1 - eps) ||u - v||^2 < ||p(u) - p(v)||^2 < (1 + eps) ||u - v||^2

Where u and v are any rows taken from a dataset of shape [n_samples,
n_features] and p is a projection by a random Gaussian N(0, 1) matrix
with shape [n_components, n_features] (or a sparse Achlioptas matrix).

The minimum number of components to guarantees the eps-embedding is
given by:

  n_components >= 4 log(n_samples) / (eps^2 / 2 - eps^3 / 3)

The first plot shows that with an increasing number of samples ``n_samples``,
the minimal number of dimensions ``n_components`` increased logarithmically
in order to guarantee an ``eps``-embedding.

The second plot shows that an increase of the admissible
distortion ``eps`` allows to reduce drastically the minimal number of
dimensions ``n_components`` for a given number of samples ``n_samples``

Empirical validation
====================

We validate the above bounds on the the digits dataset or on the 20 newsgroups
text document (TF-IDF word frequencies) dataset:

- for the digits dataset, some 8x8 gray level pixels data for 500
  handwritten digits pictures are randomly projected to spaces for various
  larger number of dimensions ``n_components``.

- for the 20 newsgroups dataset some 500 documents with 100k
  features in total are projected using a sparse random matrix to smaller
  euclidean spaces with various values for the target number of dimensions
  ``n_components``.

The default dataset is the digits dataset. To run the example on the twenty
newsgroups dataset, pass the --twenty-newsgroups command line argument to this
script.

For each value of ``n_components``, we plot:

- 2D distribution of sample pairs with pairwise distances in original
  and projected spaces as x and y axis respectively.

- 1D histogram of the ratio of those distances (projected / original).

We can see that for low values of ``n_components`` the distribution is wide
with many distorted pairs and a skewed distribution (due to the hard
limit of zero ratio on the left as distances are always positives)
while for larger values of n_components the distortion is controlled
and the distances are well preserved by the random projection.

Remarks
=======

According to the JL lemma, projecting 500 samples without too much distortion
will require at least several thousands dimensions, irrespective of the
number of features of the original dataset.

Hence using random projections on the digits dataset which only has 64 features
in the input space does not make sense: it does not allow for dimensionality
reduction in this case.

On the twenty newsgroups on the other hand the dimensionality can be decreased
from 56436 down to 10000 while reasonably preserving pairwise distances.

"""
print(__doc__)

import sys
from time import time
import numpy as np
import matplotlib.pyplot as plt
from sklearn.random_projection import johnson_lindenstrauss_min_dim
from sklearn.random_projection import SparseRandomProjection
from sklearn.datasets import fetch_20newsgroups_vectorized
from sklearn.datasets import load_digits
from sklearn.metrics.pairwise import euclidean_distances

# Part 1: plot the theoretical dependency between n_components_min and
# n_samples

# range of admissible distortions
eps_range = np.linspace(0.1, 0.99, 5)
colors = plt.cm.Blues(np.linspace(0.3, 1.0, len(eps_range)))

# range of number of samples (observation) to embed
n_samples_range = np.logspace(1, 9, 9)

plt.figure()
for eps, color in zip(eps_range, colors):
    min_n_components = johnson_lindenstrauss_min_dim(n_samples_range, eps=eps)
    plt.loglog(n_samples_range, min_n_components, color=color)

plt.legend(["eps = %0.1f" % eps for eps in eps_range], loc="lower right")
plt.xlabel("Number of observations to eps-embed")
plt.ylabel("Minimum number of dimensions")
plt.title("Johnson-Lindenstrauss bounds:\nn_samples vs n_components")

# range of admissible distortions
eps_range = np.linspace(0.01, 0.99, 100)

# range of number of samples (observation) to embed
n_samples_range = np.logspace(2, 6, 5)
colors = plt.cm.Blues(np.linspace(0.3, 1.0, len(n_samples_range)))

plt.figure()
for n_samples, color in zip(n_samples_range, colors):
    min_n_components = johnson_lindenstrauss_min_dim(n_samples, eps=eps_range)
    plt.semilogy(eps_range, min_n_components, color=color)

plt.legend(["n_samples = %d" % n for n in n_samples_range], loc="upper right")
plt.xlabel("Distortion eps")
plt.ylabel("Minimum number of dimensions")
plt.title("Johnson-Lindenstrauss bounds:\nn_components vs eps")

# Part 2: perform sparse random projection of some digits images which are
# quite low dimensional and dense or documents of the 20 newsgroups dataset
# which is both high dimensional and sparse

if '--twenty-newsgroups' in sys.argv:
    # Need an internet connection hence not enabled by default
    data = fetch_20newsgroups_vectorized().data[:500]
else:
    data = load_digits().data[:500]

n_samples, n_features = data.shape
print("Embedding %d samples with dim %d using various random projections"
      % (n_samples, n_features))

n_components_range = np.array([300, 1000, 10000])
dists = euclidean_distances(data, squared=True).ravel()

# select only non-identical samples pairs
nonzero = dists != 0
dists = dists[nonzero]

for n_components in n_components_range:
    t0 = time()
    rp = SparseRandomProjection(n_components=n_components)
    projected_data = rp.fit_transform(data)
    print("Projected %d samples from %d to %d in %0.3fs"
          % (n_samples, n_features, n_components, time() - t0))
    if hasattr(rp, 'components_'):
        n_bytes = rp.components_.data.nbytes
        n_bytes += rp.components_.indices.nbytes
        print("Random matrix with size: %0.3fMB" % (n_bytes / 1e6))

    projected_dists = euclidean_distances(
        projected_data, squared=True).ravel()[nonzero]

    plt.figure()
    plt.hexbin(dists, projected_dists, gridsize=100, cmap=plt.cm.PuBu)
    plt.xlabel("Pairwise squared distances in original space")
    plt.ylabel("Pairwise squared distances in projected space")
    plt.title("Pairwise distances distribution for n_components=%d" %
              n_components)
    cb = plt.colorbar()
    cb.set_label('Sample pairs counts')

    rates = projected_dists / dists
    print("Mean distances rate: %0.2f (%0.2f)"
          % (np.mean(rates), np.std(rates)))

    plt.figure()
    plt.hist(rates, bins=50, normed=True, range=(0., 2.))
    plt.xlabel("Squared distances rate: projected / original")
    plt.ylabel("Distribution of samples pairs")
    plt.title("Histogram of pairwise distance rates for n_components=%d" %
              n_components)

    # TODO: compute the expected value of eps and add them to the previous plot
    # as vertical lines / region

plt.show()


# add to kfkd.py
from lasagne import layers
from lasagne.updates import nesterov_momentum
from nolearn.lasagne import NeuralNet,BatchIterator
import os
import numpy as np
from sklearn.utils import shuffle
import cPickle as pickle
import matplotlib.pyplot as plt
import Image
import ImageOps
from scipy import misc
import scipy.io
import theano

def load():
    f = file('data_Salicon_T.cPickle', 'rb')
    loaded_obj = pickle.load(f)
    f.close()
    X, y = loaded_obj
    return X, y

def float32(k):
    return np.cast['float32'](k)

class AdjustVariable(object):
    def __init__(self, name, start=0.03, stop=0.001):
        self.name = name
        self.start, self.stop = start, stop
        self.ls = None

    def __call__(self, nn, train_history):
        if self.ls is None:
            self.ls = np.linspace(self.start, self.stop, nn.max_epochs)

        epoch = train_history[-1]['epoch']
        new_value = float32(self.ls[epoch - 1])
        getattr(nn, self.name).set_value(new_value)

class FlipBatchIterator(BatchIterator):
    def transform(self, Xb, yb):
        Xb, yb = super(FlipBatchIterator, self).transform(Xb, yb)

        # Flip half of the images in this batch at random:
        bs = Xb.shape[0]
        indices = np.random.choice(bs, bs / 2, replace=False)
        Xb[indices] = Xb[indices, :, :, ::-1]
        tmp =  yb[indices].reshape(bs/2,1,48,48)
        mirror = tmp[ :,:,:, ::-1]
        yb[indices] =  mirror.reshape(bs/2,48*48)
        return Xb, yb

net2 = NeuralNet(
    layers=[
        ('input', layers.InputLayer),
        ('conv1', layers.Conv2DLayer),
        ('pool1', layers.MaxPool2DLayer),
        ('conv2', layers.Conv2DLayer),
        ('pool2', layers.MaxPool2DLayer),
        ('conv3', layers.Conv2DLayer),
        ('pool3', layers.MaxPool2DLayer),
        ('hidden4', layers.DenseLayer),
        ('maxout6',layers.FeaturePoolLayer),
        ('output', layers.DenseLayer),
        ],
    input_shape=(None, 3, 96, 96),
    conv1_num_filters=32, conv1_filter_size=(5, 5), pool1_pool_size=(2, 2),
    conv2_num_filters=64, conv2_filter_size=(3, 3), pool2_pool_size=(2, 2),
    conv3_num_filters=64, conv3_filter_size=(3, 3), pool3_pool_size=(2, 2),
    hidden4_num_units=48*48*2,
    maxout6_pool_size=2,output_num_units=48*48,output_nonlinearity=None,

    update_learning_rate=theano.shared(float32(0.05)),
    update_momentum=theano.shared(float32(0.9)),

    regression=True,

    on_epoch_finished=[
        AdjustVariable('update_learning_rate', start=0.05, stop=0.0001),
        AdjustVariable('update_momentum', start=0.9, stop=0.999),
        ],
    batch_iterator_train=FlipBatchIterator(batch_size=128),
    max_epochs=1200,
    verbose=1,
    )

X, y = load()

print("X.shape == {}; X.min == {:.3f}; X.max == {:.3f}".format(
    X.shape, X.min(), X.max()))
print("y.shape == {}; y.min == {:.3f}; y.max == {:.3f}".format(
    y.shape, y.min(), y.max()))

X = X.astype(np.float32)
y = y.astype(np.float32)
net.fit(X, y)

with open('JuntingNet_SALICON.pickle', 'wb') as f:
   pickle.dump(net2, f, -1)


import pandas as pd
from sqlalchemy import create_engine, MetaData, Table, select

# Parameters
TableName = "data"

DB = {
    'drivername': 'mssql+pyodbc',
    'servername': 'DAVID-THINK',
    #'port': '5432',
    #'username': 'lynn',
    #'password': '',
    'database': 'BizIntel',
    'driver': 'SQL Server Native Client 11.0',
    'trusted_connection': 'yes',
    'legacy_schema_aliasing': False
}

# Create the connection
engine = create_engine(DB['drivername'] + '://' + DB['servername'] + '/' + DB['database'] + '?' + 'driver=' + DB['driver'] + ';' + 'trusted_connection=' + DB['trusted_connection'], legacy_schema_aliasing=DB['legacy_schema_aliasing'])
conn = engine.connect()

# Required for querying tables
metadata = MetaData(conn)

# Table to query
tbl = Table(TableName, metadata, autoload=True, schema="dbo")
#tbl.create(checkfirst=True)

# Select all
sql = tbl.select()

# run sql code
result = conn.execute(sql)

# Insert to a dataframe
df = pd.DataFrame(data=list(result), columns=result.keys())

# Close connection
conn.close()

print('Done')

df.to_csv('DimDate.csv', index=False)
df.to_excel('DimDate.xls', index=False)
df.to_csv('DimDate.txt', index=False)


#TSTOP
#
#This program is free software: you can redistribute it and/or modify
#it under the terms of the GNU General Public License as published by
#the Free Software Foundation, either version 3 of the License, or
#(at your option) any later version.
#
#This program is distributed in the hope that it will be useful,
#but WITHOUT ANY WARRANTY; without even the implied warranty of
#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#GNU General Public License for more details.
#
#You should have received a copy of the GNU General Public License
#along with this program.  If not, see <http://www.gnu.org/licenses/>.

import random
import os
import sys
import argparse
from math import ceil
import numpy

from sklearn.cross_validation import StratifiedKFold

from Datatypes.JSONObject import load_data, save_data
from Datatypes.Segments import SegmentInfo
from Datatypes.Configuration import Configuration
from Datatypes.TrainTestPartitions import TrainTestPartition, TrainTestPartitions

def PartitionData(segment_info, split, avoid_overlap=False, segment_size=0,
                  file_based=False, preserve_labels=False, override_preset=False, surpress_warning=False, seed=None) :
    '''
    Accepts a list of Datatype.Segments.SegmentInfo and a float between 0 and 1,
    and outputs a pair of lists of indices, (train, test) corresponding
    to a parition of the input list
    len(train) approximates split * len(segment_info)
    Intersection of train and test is an empty set
    Union of train and test is not guaranteed to be range(len(segment_info))

    Optional arguments:
    avoid_overlap omits entries in test that would have overlapping data with entries in train,
        as indicated by the range [segment_start:segment_start+segment_size]
    segment_size interacts with avoid overlap, because only segment_start is contained in the
        SegmentInfo class
    file_based creates partitions where segments with the same filename for source data are
        in the same partition
    preserve_label tries to split the populations of labels evenly
    '''

    segment_count = len(segment_info)
    segment_range = range(segment_count)

    # check to see if we have a preset train / test split for all data and we aren't overriding that
    if not override_preset and [0 for s in segment_info if s.learning == None] == [] :
        return TrainTestPartition([i for i in segment_range if segment_info[i].learning == 'train'],
                                  [i for i in segment_range if segment_info[i].learning == 'test'], None)

    train_goal_len = int(ceil(segment_count * split))

    if preserve_labels :
        labels = [s.max_label() for s in segment_info]
        label_set = list(set(labels))
        label_count = [(l0,len([l for l in labels if l == l0])) for l0 in label_set]
        label_goal = [(str(l), int(round(c * split))) for (l,c) in label_count]
        for ((l0,g),(l1,c)) in zip(label_goal, label_count) :
            if (g == 0) or (g == c) and not surpress_warning:
                print "PartitionData warning: not enough entries (%d) of label %s to properly make a train / test split of ratio %s" % (c, l0, split)
        label_goal = dict(label_goal)

    train = []
    test = []
    if seed != None :
        random.seed(seed)
    state = random.getstate()
    if file_based :
        files = list(set([s.filename for s in segment_info]))
        random.shuffle(files)
        for f in files :
            f_indices = [x for (x,y) in zip(segment_range, segment_info) if y.filename == f]
            if preserve_labels :
                f_labels = [str(labels[i]) for i in f_indices]
                extend_train = True
                for l in label_goal.keys() :
                    count = len([l0 for l0 in f_labels if l0 == l])
                    if count > label_goal[l] :
                        extend_train = False
                        break
                if extend_train :
                    train.extend(f_indices)
                    for l in label_goal.keys() :
                        count = len([l0 for l0 in f_labels if l0 == l])
                        label_goal[l] = label_goal[l] - count
                else :
                    test.extend(f_indices)
            else :
                if len(train) + len(f_indices) < train_goal_len :
                    train.extend(f_indices)
                else :
                    test.extend(f_indices)
    else :
        random.shuffle(segment_range)
        if preserve_labels :
            for i in segment_range:
                l = str(labels[i])
                if label_goal[l] > 0 :
                    train.append(i)
                    label_goal[l] = label_goal[l] - 1
                else :
                    test.append(i)
        else :
            train = segment_range[0:train_goal_len]
            test = segment_range[train_goal_len:]

    return TrainTestPartition(train,test,state)

def generate_partitions(config, segment_info, cv_iterations=0, seed=None) :
    partition = PartitionData(segment_info,
                              config.learning_split,
                              avoid_overlap=True,
                              segment_size=config.segment_size,
                              file_based=True if (config.data_type == "BirdSoundsSegments" or
                                                  config.data_type == "KitchenMocapSegments") \
                              else False,
                              preserve_labels=True,
                              seed=seed)

    all_labels = [segment_info[i].max_label() for i in partition.train]
    if cv_iterations > 0 :
        skf = StratifiedKFold(all_labels, n_folds=cv_iterations)

        cross_validation = [TrainTestPartition([partition.train[i] for i in train_index],
                                               [partition.train[i] for i in test_index], None) \
                            for train_index, test_index in skf]
    else :
        cross_validation = None
    learning_trials = [PartitionData(segment_info,
                                     config.learning_split,
                                     avoid_overlap=True,
                                     segment_size=config.segment_size,
                                     file_based=True if (config.data_type == "BirdSoundsSegments" or
                                                         config.data_type == "KitchenMocapSegments") \
                                     else False,
                                     preserve_labels=True,
                                     seed=None) for i in range(config.learning_iterations)]
    return TrainTestPartitions(config, segment_info, cross_validation, learning_trials)

if __name__ == "__main__" :
    parser = argparse.ArgumentParser("Tool to generate train / test splits for testing and cross validation")
    parser.add_argument("--segments", "-i")
    parser.add_argument("--outfile", "-o")
    parser.add_argument("--learning-split", "-s", type=float)
    parser.add_argument("--learning-iterations", "-I", type=int)
    parser.add_argument("--cv-iterations", "-v", default=5, type=int)
    parser.add_argument("--seed", "-S")
    args = parser.parse_args(sys.argv[1:])

    segments_json = load_data(args.segments, 'segments', None, None, sys.argv[0] + " : ")
    if segments_json == None :
        print "Could not load Segments from %s" % (args.segments,)
        sys.exit(1)
    segment_info = [SegmentInfo.fromJSONDict(s) for s in segments_json['segments']]
    config = Configuration.fromJSONDict(segments_json['config'])
    if args.learning_split != None :
        config.learning_split = args.learning_split
    if args.learning_iterations != None :
        config.learning_iterations = args.learning_iterations

    output = generate_partitions(config, segment_info, cv_iterations=args.cv_iterations, seed=args.seed)

    if args.outfile == None :
        args.outfile = TrainTestPartitions.get_partition_filename(config)

    print "Writing %s" % (args.outfile,)
    save_data(args.outfile, output.toJSONDict())


import datetime
import logging
import math
import random
import pandas as pd

logging.basicConfig(level=logging.INFO)

INPUT_FIELDS = ('holiday', 'workingday', 'temp', 'atemp', 'humidity',
                'windspeed', 'hour', 'day_of_year', 'day_of_week')
PERIODICS    = ('hour', 'day_of_year', 'day_of_week')
RESULT_FIELD = 'count'

def normalise(df, normalise=[]):
    mins  = dict((field, min(df[field])) for field in normalise)
    maxes = dict((field, max(df[field])) for field in normalise)

    for field in normalise:
        f = lambda x: float(x - mins[field]) / (maxes[field] - mins[field])
        df[field] = map(f, df[field])

    return df

def load_and_munge_training_data(filename):

    parse_hour = lambda dt: int(dt.split()[1].split(':')[0])
    parse_day_of_week = lambda dt: parse_date(dt).weekday()

    def parse_date(dt):
        return datetime.date(*(int(x) for x in dt.split()[0].split('-')))

    def parse_day_of_year(dt):
        _date = parse_date(dt)
        year_start = datetime.date(_date.year, 1, 1)
        return (_date - year_start).days

    df = pd.read_csv(open(filename))
    df['hour']  = map(parse_hour, df['datetime'])
    df['day_of_year'] = map(parse_day_of_year, df['datetime'])
    df['day_of_week'] = map(parse_day_of_week, df['datetime'])
    return normalise(df, INPUT_FIELDS)

def euclidean_dist(a, b):
    diff = lambda m, n, field: (m - n) % 1 if field in PERIODICS else m - n
    return math.sqrt(sum((diff(a[f], b[f], f)**2 for f in INPUT_FIELDS)))

def shuffle(df):
    length = len(df)
    chosen_indices = random.sample(range(length), length)
    return df.irow(chosen_indices)

def most_influential(training, fields):

    def homogeneity(field):
        return training.groupby(field)[RESULT_FIELD].apply(np.std).sum()

    return sorted((homogeneity(f), f) for f in fields)[0][1]

def knn(vector, neighbours, k=3):
    ds = [(euclidean_dist(vector, n), n) for _, n in neighbours.iterrows()]
    return sorted(ds, key=lambda a: a[0])[:k]

def gaussian_weight(dist, sigma=12.0):
    return math.exp(-dist**2/(2*sigma**2))

def estimate(test, training):
    neighbour_dists = knn(test, training)
    weights = [(gaussian_weight(d), n) for d, n in neighbour_dists]
    sum_weights = sum(w for w, _ in weights)
    mean = sum(w * n[RESULT_FIELD] for w, n in weights) / sum_weights
    return int(mean)

def main():
    dry_run   = False
    all_train = load_and_munge_training_data('train.csv')

    if dry_run:
        train_on  = 0.6

        all_train = shuffle(all_train)
        split = int(train_on * len(all_train))
        train = all_train[: split]
        test  = all_train[split+1:]
    else:
        train = all_train
        test  = load_and_munge_training_data('test.csv')

    filename = 'knn-normalising.csv'
    with open(filename, 'w') as f:
        f.write('datetime,count\n')
        for n, t in test.iterrows():
            f.write('%s,%i\n' % (t['datetime'], estimate(t, train)))
            print '%s,%i' % (t['datetime'], estimate(t, train))

if __name__ == '__main__':
    main()