"""
=========================
Bayesian Ridge Regression
=========================

Computes a Bayesian Ridge Regression on a synthetic dataset.

See :ref:`bayesian_ridge_regression` for more information on the regressor.

Compared to the OLS (ordinary least squares) estimator, the coefficient
weights are slightly shifted toward zeros, which stabilises them.

As the prior on the weights is a Gaussian prior, the histogram of the
estimated weights is Gaussian.

The estimation of the model is done by iteratively maximizing the
marginal log-likelihood of the observations.
"""
print(__doc__)

import numpy as np
import matplotlib.pyplot as plt
from scipy import stats

from sklearn.linear_model import BayesianRidge, LinearRegression

###############################################################################
# Generating simulated data with Gaussian weigthts
np.random.seed(0)
n_samples, n_features = 100, 100
X = np.random.randn(n_samples, n_features)  # Create Gaussian data
# Create weigts with a precision lambda_ of 4.
lambda_ = 4.
w = np.zeros(n_features)
# Only keep 10 weights of interest
relevant_features = np.random.randint(0, n_features, 10)
for i in relevant_features:
    w[i] = stats.norm.rvs(loc=0, scale=1. / np.sqrt(lambda_))
# Create noise with a precision alpha of 50.
alpha_ = 50.
noise = stats.norm.rvs(loc=0, scale=1. / np.sqrt(alpha_), size=n_samples)
# Create the target
y = np.dot(X, w) + noise

###############################################################################
# Fit the Bayesian Ridge Regression and an OLS for comparison
clf = BayesianRidge(compute_score=True)
clf.fit(X, y)

ols = LinearRegression()
ols.fit(X, y)

###############################################################################
# Plot true weights, estimated weights and histogram of the weights
plt.figure(figsize=(6, 5))
plt.title("Weights of the model")
plt.plot(clf.coef_, 'b-', label="Bayesian Ridge estimate")
plt.plot(w, 'g-', label="Ground truth")
plt.plot(ols.coef_, 'r--', label="OLS estimate")
plt.xlabel("Features")
plt.ylabel("Values of the weights")
plt.legend(loc="best", prop=dict(size=12))

plt.figure(figsize=(6, 5))
plt.title("Histogram of the weights")
plt.hist(clf.coef_, bins=n_features, log=True)
plt.plot(clf.coef_[relevant_features], 5 * np.ones(len(relevant_features)),
         'ro', label="Relevant features")
plt.ylabel("Features")
plt.xlabel("Values of the weights")
plt.legend(loc="lower left")

plt.figure(figsize=(6, 5))
plt.title("Marginal log-likelihood")
plt.plot(clf.scores_)
plt.ylabel("Score")
plt.xlabel("Iterations")
plt.show()


#!/usr/bin/python

import numpy as np
import numpy.random as rnd
import sys
import matplotlib
matplotlib.use('TkAgg')
from matplotlib import pyplot as plt
from numpy import pi

poligono_p    = lambda n,rot: [(1,i*2*np.pi/n+rot) for i in range(1,n+1)]
pol2cart      = lambda ro,te: (ro*np.cos(te),ro*np.sin(te))
poligono_c    = lambda L: [pol2cart(x[0],x[1]) for x in L]
genera_coords = lambda L,p: dict(zip(L,p))
pmedio        = lambda x,y: (0.5*(x[0]+y[0]) , 0.5*(x[1]+y[1]) )

class JdelC(object):
    def __init__(self):
        pass

def juego(n,m=100000, rot=pi/2):
    C = genera_coords(range(n), poligono_c(poligono_p(n,rot)))
    P = [C[rnd.choice(range(n))]]
    for i in range(m):
        up = P[-1]
        vz = C[rnd.choice(range(n))]
        P.append(pmedio(up,vz))
    return np.array(P), C

def juego_sec(V,S,m=100000,rot=pi/4):
    n = len(V)
    C = genera_coords(V, poligono_c(poligono_p(n,rot)))
    P = [C[S[0]]]
    cont = 0
    for i in range(1,m):
        up = P[-1]
        vz = C[S[i]]
        P.append(pmedio(up,vz))
    return np.array(P), C

def secciones_nucleotidos(f,m):
    cont=0
    for r in f:
        l = r.strip()
        if(l[0]=='>'):
            continue
        acum = m-cont
        sec = ''.join([ s for s in l[:acum] if s!='N' ])
        cont+=len(sec)
        if(cont<=m):
            yield sec

def secciones(f,m):
    cont=0
    for r in f:
        l = r.strip()
        try:
            if(l[0]=='>'):
                continue
        except:
            continue
        acum = m-cont
        sec = ''.join([ s for s in l[:acum] ])
        cont+=len(sec)
        if(cont<=m):
            yield sec

def grafica(R):
    plt.scatter(R[:,0],R[:,1],s=0.1, c='k')

def grafcoords(*D):
    R,C = D
    plt.scatter(R[:,0],R[:,1],s=0.1, c='k')
    for c in C:
        plt.annotate(c,C[c])

if __name__=='__main__':
    n = int(sys.argv[0])
# Ejemplo
# In [150]: G = open('Saccharomyces_cerevisiae_aa.fasta','r')
#
# In [151]: secs = jdelc.secciones(G,1000)
#
# In [152]: secuencia = ''
#
# In [153]: for sec in secs:
#      ...:     secuencia += sec
#      ...:
#
# In [154]: R,C = jdelc.juego_sec(aminos,secuencia, len(secuencia),pi/4); jdelc.grafcoords(R,C); show()


from operator import (
    and_,
    ge,
    gt,
    le,
    lt,
    methodcaller,
    ne,
    or_,
)
from unittest import TestCase

import numpy
from numpy import (
    arange,
    eye,
    full,
    isnan,
    zeros,
)
from pandas import (
    DataFrame,
    date_range,
    Int64Index,
)

from zipline.modelling.expression import (
    NumericalExpression,
    NUMEXPR_MATH_FUNCS,
)
from zipline.modelling.factor import TestingFactor
from zipline.utils.test_utils import check_arrays

class F(TestingFactor):
    inputs = ()
    window_length = 0

class G(TestingFactor):
    inputs = ()
    window_length = 0

class H(TestingFactor):
    inputs = ()
    window_length = 0

class NumericalExpressionTestCase(TestCase):

    def setUp(self):
        self.dates = date_range('2014-01-01', periods=5, freq='D')
        self.assets = Int64Index(range(5))
        self.f = F()
        self.g = G()
        self.h = H()
        self.fake_raw_data = {
            self.f: full((5, 5), 3),
            self.g: full((5, 5), 2),
            self.h: full((5, 5), 1),
        }
        self.mask = DataFrame(True, index=self.dates, columns=self.assets)

    def check_output(self, expr, expected):
        result = expr.compute_from_arrays(
            [self.fake_raw_data[input_] for input_ in expr.inputs],
            self.mask,
        )
        check_arrays(result, expected)

    def check_constant_output(self, expr, expected):
        self.assertFalse(isnan(expected))
        return self.check_output(expr, full((5, 5), expected))

    def test_validate_good(self):
        f = self.f
        g = self.g

        NumericalExpression("x_0", (f,))
        NumericalExpression("x_0 ", (f,))
        NumericalExpression("x_0 + x_0", (f,))
        NumericalExpression("x_0 + 2", (f,))
        NumericalExpression("2 * x_0", (f,))
        NumericalExpression("x_0 + x_1", (f, g))
        NumericalExpression("x_0 + x_1 + x_0", (f, g))
        NumericalExpression("x_0 + 1 + x_1", (f, g))

    def test_validate_bad(self):
        f, g, h = F(), G(), H()

        # Too few inputs.
        with self.assertRaises(ValueError):
            NumericalExpression("x_0", ())
        with self.assertRaises(ValueError):
            NumericalExpression("x_0 + x_1", (f,))

        # Too many inputs.
        with self.assertRaises(ValueError):
            NumericalExpression("x_0", (f, g))
        with self.assertRaises(ValueError):
            NumericalExpression("x_0 + x_1", (f, g, h))

        # Invalid variable name.
        with self.assertRaises(ValueError):
            NumericalExpression("x_0x_1", (f,))
        with self.assertRaises(ValueError):
            NumericalExpression("x_0x_1", (f, g))

        # Variable index must start at 0.
        with self.assertRaises(ValueError):
            NumericalExpression("x_1", (f,))

        # Scalar operands must be numeric.
        with self.assertRaises(TypeError):
            "2" + f
        with self.assertRaises(TypeError):
            f + "2"
        with self.assertRaises(TypeError):
            f > "2"

        # Boolean binary operators must be between filters.
        with self.assertRaises(TypeError):
            f + (f > 2)
        with self.assertRaises(TypeError):
            (f > f) > f

    def test_negate(self):
        f, g = self.f, self.g

        self.check_constant_output(-f, -3.0)
        self.check_constant_output(--f, 3.0)
        self.check_constant_output(---f, -3.0)

        self.check_constant_output(-(f + f), -6.0)
        self.check_constant_output(-f + -f, -6.0)
        self.check_constant_output(-(-f + -f), 6.0)

        self.check_constant_output(f + -g, 1.0)
        self.check_constant_output(f - -g, 5.0)

        self.check_constant_output(-(f + g) + (f + g), 0.0)
        self.check_constant_output((f + g) + -(f + g), 0.0)
        self.check_constant_output(-(f + g) + -(f + g), -10.0)

    def test_add(self):
        f, g = self.f, self.g

        self.check_constant_output(f + g, 5.0)

        self.check_constant_output((1 + f) + g, 6.0)
        self.check_constant_output(1 + (f + g), 6.0)
        self.check_constant_output((f + 1) + g, 6.0)
        self.check_constant_output(f + (1 + g), 6.0)
        self.check_constant_output((f + g) + 1, 6.0)
        self.check_constant_output(f + (g + 1), 6.0)

        self.check_constant_output((f + f) + f, 9.0)
        self.check_constant_output(f + (f + f), 9.0)

        self.check_constant_output((f + g) + f, 8.0)
        self.check_constant_output(f + (g + f), 8.0)

        self.check_constant_output((f + g) + (f + g), 10.0)
        self.check_constant_output((f + g) + (g + f), 10.0)
        self.check_constant_output((g + f) + (f + g), 10.0)
        self.check_constant_output((g + f) + (g + f), 10.0)

    def test_subtract(self):
        f, g = self.f, self.g

        self.check_constant_output(f - g, 1.0)  # 3 - 2

        self.check_constant_output((1 - f) - g, -4.)   # (1 - 3) - 2
        self.check_constant_output(1 - (f - g), 0.0)   # 1 - (3 - 2)
        self.check_constant_output((f - 1) - g, 0.0)   # (3 - 1) - 2
        self.check_constant_output(f - (1 - g), 4.0)   # 3 - (1 - 2)
        self.check_constant_output((f - g) - 1, 0.0)   # (3 - 2) - 1
        self.check_constant_output(f - (g - 1), 2.0)   # 3 - (2 - 1)

        self.check_constant_output((f - f) - f, -3.)   # (3 - 3) - 3
        self.check_constant_output(f - (f - f), 3.0)   # 3 - (3 - 3)

        self.check_constant_output((f - g) - f, -2.)   # (3 - 2) - 3
        self.check_constant_output(f - (g - f), 4.0)   # 3 - (2 - 3)

        self.check_constant_output((f - g) - (f - g), 0.0)  # (3 - 2) - (3 - 2)
        self.check_constant_output((f - g) - (g - f), 2.0)  # (3 - 2) - (2 - 3)
        self.check_constant_output((g - f) - (f - g), -2.)  # (2 - 3) - (3 - 2)
        self.check_constant_output((g - f) - (g - f), 0.0)  # (2 - 3) - (2 - 3)

    def test_multiply(self):
        f, g = self.f, self.g

        self.check_constant_output(f * g, 6.0)

        self.check_constant_output((2 * f) * g, 12.0)
        self.check_constant_output(2 * (f * g), 12.0)
        self.check_constant_output((f * 2) * g, 12.0)
        self.check_constant_output(f * (2 * g), 12.0)
        self.check_constant_output((f * g) * 2, 12.0)
        self.check_constant_output(f * (g * 2), 12.0)

        self.check_constant_output((f * f) * f, 27.0)
        self.check_constant_output(f * (f * f), 27.0)

        self.check_constant_output((f * g) * f, 18.0)
        self.check_constant_output(f * (g * f), 18.0)

        self.check_constant_output((f * g) * (f * g), 36.0)
        self.check_constant_output((f * g) * (g * f), 36.0)
        self.check_constant_output((g * f) * (f * g), 36.0)
        self.check_constant_output((g * f) * (g * f), 36.0)

        self.check_constant_output(f * f * f * 0 * f * f, 0.0)

    def test_divide(self):
        f, g = self.f, self.g

        self.check_constant_output(f / g, 3.0 / 2.0)

        self.check_constant_output(
            (2 / f) / g,
            (2 / 3.0) / 2.0
        )
        self.check_constant_output(
            2 / (f / g),
            2 / (3.0 / 2.0),
        )
        self.check_constant_output(
            (f / 2) / g,
            (3.0 / 2) / 2.0,
        )
        self.check_constant_output(
            f / (2 / g),
            3.0 / (2 / 2.0),
        )
        self.check_constant_output(
            (f / g) / 2,
            (3.0 / 2.0) / 2,
        )
        self.check_constant_output(
            f / (g / 2),
            3.0 / (2.0 / 2),
        )
        self.check_constant_output(
            (f / f) / f,
            (3.0 / 3.0) / 3.0
        )
        self.check_constant_output(
            f / (f / f),
            3.0 / (3.0 / 3.0),
        )
        self.check_constant_output(
            (f / g) / f,
            (3.0 / 2.0) / 3.0,
        )
        self.check_constant_output(
            f / (g / f),
            3.0 / (2.0 / 3.0),
        )

        self.check_constant_output(
            (f / g) / (f / g),
            (3.0 / 2.0) / (3.0 / 2.0),
        )
        self.check_constant_output(
            (f / g) / (g / f),
            (3.0 / 2.0) / (2.0 / 3.0),
        )
        self.check_constant_output(
            (g / f) / (f / g),
            (2.0 / 3.0) / (3.0 / 2.0),
        )
        self.check_constant_output(
            (g / f) / (g / f),
            (2.0 / 3.0) / (2.0 / 3.0),
        )

    def test_pow(self):
        f, g = self.f, self.g

        self.check_constant_output(f ** g, 3.0 ** 2)
        self.check_constant_output(2 ** f, 2.0 ** 3)
        self.check_constant_output(f ** 2, 3.0 ** 2)

        self.check_constant_output((f + g) ** 2, (3.0 + 2.0) ** 2)
        self.check_constant_output(2 ** (f + g), 2 ** (3.0 + 2.0))

        self.check_constant_output(f ** (f ** g), 3.0 ** (3.0 ** 2.0))
        self.check_constant_output((f ** f) ** g, (3.0 ** 3.0) ** 2.0)

        self.check_constant_output((f ** g) ** (f ** g), 9.0 ** 9.0)
        self.check_constant_output((f ** g) ** (g ** f), 9.0 ** 8.0)
        self.check_constant_output((g ** f) ** (f ** g), 8.0 ** 9.0)
        self.check_constant_output((g ** f) ** (g ** f), 8.0 ** 8.0)

    def test_mod(self):
        f, g = self.f, self.g

        self.check_constant_output(f % g, 3.0 % 2.0)
        self.check_constant_output(f % 2.0, 3.0 % 2.0)
        self.check_constant_output(g % f, 2.0 % 3.0)

        self.check_constant_output((f + g) % 2, (3.0 + 2.0) % 2)
        self.check_constant_output(2 % (f + g), 2 % (3.0 + 2.0))

        self.check_constant_output(f % (f % g), 3.0 % (3.0 % 2.0))
        self.check_constant_output((f % f) % g, (3.0 % 3.0) % 2.0)

        self.check_constant_output((f + g) % (f * g), 5.0 % 6.0)

    def test_math_functions(self):
        f, g = self.f, self.g

        fake_raw_data = self.fake_raw_data
        alt_fake_raw_data = {
            self.f: full((5, 5), .5),
            self.g: full((5, 5), -.5),
        }

        for funcname in NUMEXPR_MATH_FUNCS:
            method = methodcaller(funcname)
            func = getattr(numpy, funcname)

            # These methods have domains in [0, 1], so we need alternate inputs
            # that are in the domain.
            if funcname in ('arcsin', 'arccos', 'arctanh'):
                self.fake_raw_data = alt_fake_raw_data
            else:
                self.fake_raw_data = fake_raw_data

            f_val = self.fake_raw_data[f][0, 0]
            g_val = self.fake_raw_data[g][0, 0]

            self.check_constant_output(method(f), func(f_val))
            self.check_constant_output(method(g), func(g_val))

            self.check_constant_output(method(f) + 1, func(f_val) + 1)
            self.check_constant_output(1 + method(f), 1 + func(f_val))

            self.check_constant_output(method(f + .25), func(f_val + .25))
            self.check_constant_output(method(.25 + f), func(.25 + f_val))

            self.check_constant_output(
                method(f) + method(g),
                func(f_val) + func(g_val),
            )
            self.check_constant_output(
                method(f + g),
                func(f_val + g_val),
            )

    def test_comparisons(self):
        f, g, h = self.f, self.g, self.h
        self.fake_raw_data = {
            f: arange(25).reshape(5, 5),
            g: arange(25).reshape(5, 5) - eye(5),
            h: full((5, 5), 5),
        }
        f_data = self.fake_raw_data[f]
        g_data = self.fake_raw_data[g]

        cases = [
            # Sanity Check with hand-computed values.
            (f, g, eye(5), zeros((5, 5))),
            (f, 10, f_data, 10),
            (10, f, 10, f_data),
            (f, f, f_data, f_data),
            (f + 1, f, f_data + 1, f_data),
            (1 + f, f, 1 + f_data, f_data),
            (f, g, f_data, g_data),
            (f + 1, g, f_data + 1, g_data),
            (f, g + 1, f_data, g_data + 1),
            (f + 1, g + 1, f_data + 1, g_data + 1),
            ((f + g) / 2, f ** 2, (f_data + g_data) / 2, f_data ** 2),
        ]
        for op in (gt, ge, lt, le, ne):
            for expr_lhs, expr_rhs, expected_lhs, expected_rhs in cases:
                self.check_output(
                    op(expr_lhs, expr_rhs),
                    op(expected_lhs, expected_rhs),
                )

    def test_boolean_binops(self):
        f, g, h = self.f, self.g, self.h
        self.fake_raw_data = {
            f: arange(25).reshape(5, 5),
            g: arange(25).reshape(5, 5) - eye(5),
            h: full((5, 5), 5),
        }

        # Should be True on the diagonal.
        eye_filter = f > g
        # Should be True in the first row only.
        first_row_filter = f < h

        eye_mask = eye(5, dtype=bool)
        first_row_mask = zeros((5, 5), dtype=bool)
        first_row_mask[0] = 1

        self.check_output(eye_filter, eye_mask)
        self.check_output(first_row_filter, first_row_mask)

        for op in (and_, or_):  # NumExpr doesn't support xor.
            self.check_output(
                op(eye_filter, first_row_filter),
                op(eye_mask, first_row_mask),
            )


# coding: utf-8

# In[1]:

from __future__ import division,unicode_literals
# get_ipython().magic('matplotlib inline')
import numpy as np
import pandas as pd
import json
import runProcs
from urllib.request import urlopen

import matplotlib.pyplot as plt

# In[2]:

# 0. State abbreviations

# 0.1 dictionary:
stateAbbr = {
u'Alabama':u'AL',
u'Alaska':u'AK',
u'Arizona':u'AZ',
u'Arkansas':u'AR',
u'California':u'CA',
u'Colorado':u'CO',
u'Connecticut':u'CT',
u'Delaware':u'DE',
u'District of Columbia':u'DC',
u'Florida':u'FL',
u'Georgia':u'GA',
u'Hawaii':u'HI',
u'Idaho':u'ID',
u'Illinois':u'IL',
u'Indiana':u'IN',
u'Iowa':u'IA',
u'Kansas':u'KS',
u'Kentucky':u'KY',
u'Louisiana':u'LA',
u'Maine':u'ME',
u'Maryland':u'MD',
u'Massachusetts':u'MA',
u'Michigan':u'MI',
u'Minnesota':u'MN',
u'Mississippi':u'MS',
u'Missouri':u'MO',
u'Montana':u'MT',
u'Nebraska':u'NE',
u'Nevada':u'NV',
u'New Hampshire':u'NH',
u'New Jersey':u'NJ',
u'New Mexico':u'NM',
u'New York':u'NY',
u'North Carolina':u'NC',
u'North Dakota':u'ND',
u'Ohio':u'OH',
u'Oklahoma':u'OK',
u'Oregon':u'OR',
u'Pennsylvania':u'PA',
u'Rhode Island':u'RI',
u'South Carolina':u'SC',
u'South Dakota':u'SD',
u'Tennessee':u'TN',
u'Texas':u'TX',
u'Utah':u'UT',
u'Vermont':u'VT',
u'Virginia':u'VA',
u'Washington':u'WA',
u'West Virginia':u'WV',
u'Wisconsin':u'WI',
u'Wyoming':u'WY'
}

# 0.2 List of states in the US
stateList = [s for s in stateAbbr]

# In[3]:

# 1. Construct series for price deflator

# 1.1 Obtain data from BEA
gdpDeflator = urlopen('http://bea.gov/api/data/?UserID=3EDEAA66-4B2B-4926-83C9-FD2089747A5B&method=GetData&datasetname=NIPA&TableID=13&Frequency=A&Year=X&ResultFormat=JSON&')

# result = gdpDeflator.readall().decode('utf-8')
result = gdpDeflator.read().decode('utf-8')
jsonResponse = json.loads(result)

# In[4]:

# 1.2 Construct the data frame for the deflator series
values = []
years = []
for element in jsonResponse['BEAAPI']['Results']['Data']:
#     if element['LineDescription'] == 'Personal consumption expenditures':
    if element['LineDescription'] == 'Gross domestic product':
        years.append(element['TimePeriod'])
        values.append(float(element['DataValue'])/100)

values = np.array([values]).T
dataP = pd.DataFrame(values,index = years,columns = ['price level'])

# 1.3 Display the data
print(dataP)

# In[5]:

# 2. Construct series for per capita income by state, region, and the entire us

# 2.1 Obtain data from BEA
stateYpc = urlopen('http://bea.gov/api/data/?UserID=3EDEAA66-4B2B-4926-83C9-FD2089747A5B&method=GetData&datasetname=RegionalData&KeyCode=PCPI_SI&Year=ALL&GeoFips=STATE&ResultFormat=JSON&')
# result = stateYpc.readall().decode('utf-8')
result = stateYpc.read().decode('utf-8')
jsonResponse = json.loads(result)
# jsonResponse['BEAAPI']['Results']['Data'][0]['GeoName']

# In[6]:

# 2.2 Construct the data frame for the per capita income series

# 2.2.1 Initialize the dataframe
regions = []
years = []
for element in jsonResponse['BEAAPI']['Results']['Data']:
    if element['GeoName'] not in regions:
        regions.append(element['GeoName'])
    if element['TimePeriod'] not in years:
        years.append(element['TimePeriod'])

df = np.zeros([len(years),len(regions)])
dataY = pd.DataFrame(df,index = years,columns = regions)
# 2.2.2 Populate the dataframe with values
for element in jsonResponse['BEAAPI']['Results']['Data']:
    try:
        dataY[element['GeoName']][element['TimePeriod']] = np.round(float(element[u'DataValue'])/float(dataP.loc[element['TimePeriod']]),2)# real
    except:
        dataY[element['GeoName']][element['TimePeriod']] = np.nan

# 2.2.3 Replace the state names in the index with abbreviations
columns=[]
for r in regions:
    if r in stateList:
        columns.append(stateAbbr[r])
    else:
        columns.append(r)

dataY.columns=columns

# 2.2.4 Display the data obtained from the BEA
dataY

# In[7]:

# 3. State income data for 1840, 1880, and 1900

# 3.1.1 Import Easterlin's income data
easterlin_data = pd.read_csv('Historical Statistics of the US - Easterlin State Income Data.csv',index_col=0)

# 3.1.2 Import historic CPI data
historic_cpi_data=pd.read_csv('Historical Statistics of the US - cpi.csv',index_col=0)
historic_cpi_data = historic_cpi_data/historic_cpi_data.loc[1929]*float(dataP.loc['1929'])

# In[8]:

# Const
df_1840 = easterlin_data['Income per capita - 1840 - A [cur dollars]']/float(historic_cpi_data.loc[1840])
df_1880 = easterlin_data['Income per capita - 1880 [cur dollars]']/float(historic_cpi_data.loc[1890])
df_1900 = easterlin_data['Income per capita - 1900 [cur dollars]']/float(historic_cpi_data.loc[1900])

df = pd.DataFrame({'1840':df_1840,'1880':df_1880,'1900':df_1900}).transpose()

# In[9]:

df = pd.concat([dataY,df]).sort_index()

# In[17]:

df.loc['1880'].sort_values()

# In[10]:

# 3. Export data to csv
series = dataY.sort_index()
series = df.sort_index()
dropCols = [u'AK', u'HI', u'New England', u'Mideast', u'Great Lakes', u'Plains', u'Southeast', u'Southwest', u'Rocky Mountain', u'Far West']
for c in dropCols:
    series = series.drop([c],axis=1)

series.to_csv('stateIncomeData.csv',na_rep='NaN')

# In[11]:

len(dataY.columns)

# In[12]:

# 4. Export notebook to .py
runProcs.exportNb('stateIncomeData')


"""
========================================
Lasso and Elastic Net for Sparse Signals
========================================

Estimates Lasso and Elastic-Net regression models on a manually generated
sparse signal corrupted with an additive noise. Estimated coefficients are
compared with the ground-truth.

"""
print(__doc__)

import numpy as np
import matplotlib.pyplot as plt

from sklearn.metrics import r2_score

###############################################################################
# generate some sparse data to play with
np.random.seed(42)

n_samples, n_features = 50, 200
X = np.random.randn(n_samples, n_features)
coef = 3 * np.random.randn(n_features)
inds = np.arange(n_features)
np.random.shuffle(inds)
coef[inds[10:]] = 0  # sparsify coef
y = np.dot(X, coef)

# add noise
y += 0.01 * np.random.normal((n_samples,))

# Split data in train set and test set
n_samples = X.shape[0]
X_train, y_train = X[:n_samples / 2], y[:n_samples / 2]
X_test, y_test = X[n_samples / 2:], y[n_samples / 2:]

###############################################################################
# Lasso
from sklearn.linear_model import Lasso

alpha = 0.1
lasso = Lasso(alpha=alpha)

y_pred_lasso = lasso.fit(X_train, y_train).predict(X_test)
r2_score_lasso = r2_score(y_test, y_pred_lasso)
print(lasso)
print("r^2 on test data : %f" % r2_score_lasso)

###############################################################################
# ElasticNet
from sklearn.linear_model import ElasticNet

enet = ElasticNet(alpha=alpha, l1_ratio=0.7)

y_pred_enet = enet.fit(X_train, y_train).predict(X_test)
r2_score_enet = r2_score(y_test, y_pred_enet)
print(enet)
print("r^2 on test data : %f" % r2_score_enet)

plt.plot(enet.coef_, label='Elastic net coefficients')
plt.plot(lasso.coef_, label='Lasso coefficients')
plt.plot(coef, '--', label='original coefficients')
plt.legend(loc='best')
plt.title("Lasso R^2: %f, Elastic Net R^2: %f"
          % (r2_score_lasso, r2_score_enet))
plt.show()