## Robust Linear Models

from __future__ import print_function
import numpy as np
import statsmodels.api as sm
import matplotlib.pyplot as plt
from statsmodels.sandbox.regression.predstd import wls_prediction_std

# ## Estimation
#
# Load data:

data = sm.datasets.stackloss.load()
data.exog = sm.add_constant(data.exog)

# Huber's T norm with the (default) median absolute deviation scaling

huber_t = sm.RLM(data.endog, data.exog, M=sm.robust.norms.HuberT())
hub_results = huber_t.fit()
print(hub_results.params)
print(hub_results.bse)
print(hub_results.summary(yname='y',
            xname=['var_%d' % i for i in range(len(hub_results.params))]))

# Huber's T norm with 'H2' covariance matrix

hub_results2 = huber_t.fit(cov="H2")
print(hub_results2.params)
print(hub_results2.bse)

# Andrew's Wave norm with Huber's Proposal 2 scaling and 'H3' covariance matrix

andrew_mod = sm.RLM(data.endog, data.exog, M=sm.robust.norms.AndrewWave())
andrew_results = andrew_mod.fit(scale_est=sm.robust.scale.HuberScale(), cov="H3")
print('Parameters: ', andrew_results.params)

# See ``help(sm.RLM.fit)`` for more options and ``module sm.robust.scale`` for scale options
#
# ## Comparing OLS and RLM
#
# Artificial data with outliers:

nsample = 50
x1 = np.linspace(0, 20, nsample)
X = np.column_stack((x1, (x1-5)**2))
X = sm.add_constant(X)
sig = 0.3   # smaller error variance makes OLS<->RLM contrast bigger
beta = [5, 0.5, -0.0]
y_true2 = np.dot(X, beta)
y2 = y_true2 + sig*1. * np.random.normal(size=nsample)
y2[[39,41,43,45,48]] -= 5   # add some outliers (10% of nsample)

# ### Example 1: quadratic function with linear truth
#
# Note that the quadratic term in OLS regression will capture outlier effects.

res = sm.OLS(y2, X).fit()
print(res.params)
print(res.bse)
print(res.predict())

# Estimate RLM:

resrlm = sm.RLM(y2, X).fit()
print(resrlm.params)
print(resrlm.bse)

# Draw a plot to compare OLS estimates to the robust estimates:

fig = plt.figure(figsize=(12,8))
ax = fig.add_subplot(111)
ax.plot(x1, y2, 'o',label="data")
ax.plot(x1, y_true2, 'b-', label="True")
prstd, iv_l, iv_u = wls_prediction_std(res)
ax.plot(x1, res.fittedvalues, 'r-', label="OLS")
ax.plot(x1, iv_u, 'r--')
ax.plot(x1, iv_l, 'r--')
ax.plot(x1, resrlm.fittedvalues, 'g.-', label="RLM")
ax.legend(loc="best")

# ### Example 2: linear function with linear truth
#
# Fit a new OLS model using only the linear term and the constant:

X2 = X[:,[0,1]]
res2 = sm.OLS(y2, X2).fit()
print(res2.params)
print(res2.bse)

# Estimate RLM:

resrlm2 = sm.RLM(y2, X2).fit()
print(resrlm2.params)
print(resrlm2.bse)

# Draw a plot to compare OLS estimates to the robust estimates:

prstd, iv_l, iv_u = wls_prediction_std(res2)

fig, ax = plt.subplots()
ax.plot(x1, y2, 'o', label="data")
ax.plot(x1, y_true2, 'b-', label="True")
ax.plot(x1, res2.fittedvalues, 'r-', label="OLS")
ax.plot(x1, iv_u, 'r--')
ax.plot(x1, iv_l, 'r--')
ax.plot(x1, resrlm2.fittedvalues, 'g.-', label="RLM")
ax.legend(loc="best")


"""
Tools for working with Computer Programs in Seismology velocity models
"""
import os
import numpy as np
import datetime
import pandas as pd
from scipy.interpolate import interp1d
import matplotlib.pyplot as plt
from rockfish2 import logging
from rockfish2.models.profile import Profile

class CPSModel1d(Profile):

    def __init__(self, *args, **kwargs):

        self.NAME = kwargs.pop('name', '1D model')
        self.UNITS = kwargs.pop('units', 'KGS')
        self.ISOTROPY = kwargs.pop('isotropy', 'ISOTROPIC')
        self.SHAPE = kwargs.pop('shape', 'FLAT EARTH')
        self.DIM = kwargs.pop('dim', '1-D')

        Profile.__init__(self, *args, **kwargs)

    def __str__(self):

        return self.write()

    def write(self, path_or_buf=None, float_format='%10.6f', **kwargs):
        """
        Write profile to the Computer Programs in Seismology model format

        Parameters
        ----------
        path_or_buf : string or file handle, default None
            File path or object, if None is provided the result is returned as
            a string.
        """
        model = self.model.copy()
        col = ['hr'] + [k for k in model if k != 'hr']
        model['hr'] = np.concatenate((np.diff(np.asarray(model.index)), [0.0]))
        model.index = np.arange(len(model))
        #model = model[0:len(model) - 1]

        sng = "MODEL\n"
        sng += "{:}\n".format(self.NAME)
        sng += "{:}\n".format(self.ISOTROPY)
        sng += "{:}\n".format(self.UNITS)
        sng += "{:}\n".format(self.SHAPE)
        sng += "{:}\n".format(self.DIM)
        sng += "CONSTANT VELOCITY\n"
        sng += "#\n"
        sng += "Created by: {:}{:}\n"\
                .format(self.__module__, self.__class__.__name__)
        sng += "Created on: {:}\n".format(datetime.datetime.now())
        sng += "#\n"
        sng += model[col].to_csv(sep='\t', index=False,
                float_format=float_format, **kwargs)

        if path_or_buf is None:
            return sng

        if hasattr(path_or_buf, 'write'):
            path_or_buf.write(sng)

        else:
            f = open(path_or_buf, 'w')
            f.write(sng)

    def read(self, filename, sep='\t'):
        """
        Write profile from the Computer Programs in Seismology model format
        """
        f = open(filename, 'rb')
        kind = f.readline().replace('\n', '')
        assert kind.startswith('MODEL'),\
                'File does not appear to be CPS format'
        self.NAME = f.readline().replace('\n', '')
        self.ISOTROPY = f.readline().replace('\n', '')
        self.UNITS = f.readline().replace('\n', '')
        self.SHAPE = f.readline().replace('\n', '')
        self.DIM = f.readline().replace('\n', '')
        _ = f.readline().replace('\n', '')
        _ = f.readline().replace('\n', '')
        _ = f.readline().replace('\n', '')
        _ = f.readline().replace('\n', '')
        _ = f.readline().replace('\n', '')
        cols = f.readline().replace('\n', '').split()

        self.model = pd.read_csv(filename, sep=sep, skiprows=11,
                index_col=0)

        try:
            dz = self.model.index[:]
            z = np.cumsum(np.asarray(dz)) - dz[0]
            if z[-1] == 0:
                z[-1] = dz[-2]

            self.model.index = z
            self.model.index.name = 'depth'

        except:
            pass


# -*- coding: utf-8 -*-
"""

Vincent Map Examples

"""

#Build a map from scratch

from vincent import *

world_topo = r'world-countries.topo.json'
state_topo = r'us_states.topo.json'
lake_topo = r'lakes_50m.topo.json'
county_geo = r'us_counties.geo.json'
county_topo = r'us_counties.topo.json'
or_topo = r'or_counties.topo.json'

vis = Visualization(width=960, height=500)
vis.data['countries'] = Data(
    name='countries',
    url=world_topo,
    format={'type': 'topojson', 'feature': 'world-countries'}
    )

geo_transform = Transform(
                type='geopath', value="data", projection='winkel3', scale=200,
                translate=[480, 250]
                )

geo_from = MarkRef(data='countries', transform=[geo_transform])

enter_props = PropertySet(
    stroke=ValueRef(value='#000000'),
    path=ValueRef(field='path')
    )

update_props = PropertySet(fill=ValueRef(value='steelblue'))

mark_props = MarkProperties(enter=enter_props, update=update_props)

vis.marks.append(
    Mark(type='path', from_=geo_from, properties=mark_props)
    )

vis.to_json('vega.json')

#Convenience Method

geo_data = [{'name': 'countries',
             'url': world_topo,
             'feature': 'world-countries'}]

vis = Map(geo_data=geo_data, scale=200)
vis.to_json('vega.json')

#States & Counties

geo_data = [{'name': 'counties',
             'url': county_topo,
             'feature': 'us_counties.geo'},
            {'name': 'states',
             'url': state_topo,
             'feature': 'us_states.geo'}
             ]

vis = Map(geo_data=geo_data, scale=1000, projection='albersUsa')
del vis.marks[1].properties.update
vis.marks[0].properties.update.fill.value = '#084081'
vis.marks[1].properties.enter.stroke.value = '#fff'
vis.marks[0].properties.enter.stroke.value = '#7bccc4'
vis.to_json('vega.json')

#Choropleth
import json
import pandas as pd
#Map the county codes we have in our geometry to those in the
#county_data file, which contains additional rows we don't need
with open('us_counties.topo.json', 'r') as f:
    get_id = json.load(f)

#A little FIPS code munging
new_geoms = []
for geom in get_id['objects']['us_counties.geo']['geometries']:
    geom['properties']['FIPS'] = int(geom['properties']['FIPS'])
    new_geoms.append(geom)

get_id['objects']['us_counties.geo']['geometries'] = new_geoms

with open('us_counties.topo.json', 'w') as f:
    json.dump(get_id, f)

#Grab the FIPS codes and load them into a dataframe
geometries = get_id['objects']['us_counties.geo']['geometries']
county_codes = [x['properties']['FIPS'] for x in geometries]
county_df = pd.DataFrame({'FIPS': county_codes}, dtype=str)
county_df = county_df.astype(int)

#Read into Dataframe, cast to int for consistency
df = pd.read_csv('data/us_county_data.csv', na_values=[' '])
df['FIPS'] = df['FIPS'].astype(int)

#Perform an inner join, pad NA's with data from nearest county
merged = pd.merge(df, county_df, on='FIPS', how='inner')
merged = merged.fillna(method='pad')

geo_data = [{'name': 'counties',
             'url': county_topo,
             'feature': 'us_counties.geo'}]

vis = Map(data=merged, geo_data=geo_data, scale=1100, projection='albersUsa',
          data_bind='Employed_2011', data_key='FIPS',
          map_key={'counties': 'properties.FIPS'})
vis.marks[0].properties.enter.stroke_opacity = ValueRef(value=0.5)
#Change our domain for an even inteager
vis.scales['color'].domain = [0, 189000]
vis.legend(title='Number Employed 2011')
vis.to_json('vega.json')

#Lets look at different stats
vis.rebind(column='Civilian_labor_force_2011', brew='BuPu')
vis.to_json('vega.json')

vis.rebind(column='Unemployed_2011', brew='PuBu')
vis.to_json('vega.json')

vis.rebind(column='Unemployment_rate_2011', brew='YlGnBu')
vis.to_json('vega.json')

vis.rebind(column='Median_Household_Income_2011', brew='RdPu')
vis.to_json('vega.json')

#Mapping US State Level Data

state_data = pd.read_csv('data/US_Unemployment_Oct2012.csv')
geo_data = [{'name': 'states',
             'url': state_topo,
             'feature': 'us_states.geo'}]
vis = Map(data=state_data, geo_data=geo_data, scale=1000,
          projection='albersUsa', data_bind='Unemployment', data_key='NAME',
          map_key={'states': 'properties.NAME'})
vis.legend(title='Unemployment (%)')
vis.to_json('vega.json')

#Iterating State Level Data
yoy = pd.read_table('data/State_Unemp_YoY.txt', delim_whitespace=True)
#Standardize State names to match TopoJSON for keying
names = []
for row in yoy.iterrows():
    pieces = row[1]['NAME'].split('_')
    together = ' '.join(pieces)
    names.append(together.title())
yoy['NAME'] = names
geo_data = [{'name': 'states',
             'url': state_topo,
             'feature': 'us_states.geo'}]
vis = Map(data=yoy, geo_data=geo_data, scale=1000,
          projection='albersUsa', data_bind='AUG_2012', data_key='NAME',
          map_key={'states': 'properties.NAME'}, brew='YlGnBu')
#Custom threshold scale
vis.scales[0].type='threshold'
vis.scales[0].domain = [0, 2, 4, 6, 8, 10, 12]
vis.legend(title='Unemployment (%)')
vis.to_json('vega.json')

#Rebind and set our scale again
vis.rebind(column='AUG_2013', brew='YlGnBu')
vis.scales[0].type='threshold'
vis.scales[0].domain = [0, 2, 4, 6, 8, 10, 12]
vis.to_json('vega.json')

vis.rebind(column='CHANGE', brew='YlGnBu')
vis.scales[0].type='threshold'
vis.scales[0].domain = [-1.5, -1.3, -1.1, 0, 0.1, 0.3, 0.5, 0.8]
vis.legends[0].title = "YoY Change in Unemployment (%)"
vis.to_json('vega.json')

#Oregon County-level population data
or_data = pd.read_table('data/OR_County_Data.txt', delim_whitespace=True)
or_data['July_2012_Pop']= or_data['July_2012_Pop'].astype(int)
#Standardize keys
with open('or_counties.topo.json', 'r') as f:
    counties = json.load(f)

def split_county(name):
    parts = name.split(' ')
    parts.pop(-1)
    return ''.join(parts).upper()

#A little FIPS code munging
new_geoms = []
for geom in counties['objects']['or_counties.geo']['geometries']:
    geom['properties']['COUNTY'] = split_county(geom['properties']['COUNTY'])
    new_geoms.append(geom)

counties['objects']['or_counties.geo']['geometries'] = new_geoms

with open('or_counties.topo.json', 'w') as f:
    json.dump(counties, f)

geo_data = [{'name': 'states',
             'url': state_topo,
             'feature': 'us_states.geo'},
            {'name': 'or_counties',
             'url': or_topo,
             'feature': 'or_counties.geo'}]

vis = Map(data=or_data, geo_data=geo_data, scale=3700,
          translate=[1480, 830],
          projection='albersUsa', data_bind='July_2012_Pop', data_key='NAME',
          map_key={'or_counties': 'properties.COUNTY'})
vis.marks[0].properties.update.fill.value = '#c2c2c2'
vis.to_json('vega.json')


from . import helpers

import pandas as pd
import numpy as np
from pandas import DataFrame, Series

from pandas_datareader import data
from datetime import datetime, timedelta

import re
import os
import requests
import time

class StockInfo():
	def __init__(self, StockNumber):
		if isinstance(StockNumber, str) is False:
			print('StockNumber must be string')
			self.__StockNumber = '2330.TW'

		else:
			self.__StockNumber = StockNumber+'.TW'

	def get_StockNumber(self):
		return self.__StockNumber

	def fetch_StockPrice(self, StartTime, EndTime):
# 		self.__StockPrice = data.DataReader(self.__StockNumber,
# 											'yahoo',StartTime, EndTime)
		self.__StockPrice = data.DataReader(self.__StockNumber,
											'yahoo',StartTime, EndTime)

	def get_StockPrice(self):
		return self.__StockPrice

	def fetch_StockActions(self, StartTime, EndTime):
		self.__StockActions = data.DataReader(self.__StockNumber,
											'yahoo-actions',StartTime, EndTime)
	def get_StockActions(self):
		return self.__StockActions

class Crawler():
	def __init__(self, prefix='data'):
		if not os.path.isdir(prefix):
			os.mkdir(prefix)
		self.prefix = prefix
		# pass

	def get_tse_one_day(self, spec_date):
		date_str = '{0}{1:02d}{2:02d}'.format(spec_date.year, spec_date.month, spec_date.day)
		url = 'http://www.twse.com.tw/exchangeReport/MI_INDEX'

		query_params = {
			'date': date_str,
			'response': 'json',
			'type': 'ALL',
			'_': str(round(time.time() * 1000) - 500)
		}

		# Get json data
		page = requests.get(url, params=query_params)

		if not page.ok:
			logging.error("Can not get TSE data at {}".format(date_str))

		content = page.json()
		# print(content)
		# key = 'Nodata'
		isoffday = True
		for key in content.keys():
			if isinstance(content[key], list):
				if len(content[key][0]) == 16:
					isoffday = False
					break
		if isoffday:
			print('No data at this day %4d/%02d/%02d'%
					(spec_date.year,spec_date.month, spec_date.day))
			return -1

		# For compatible with original data
		# date_str_mingguo = '{0}/{1:02d}/{2:02d}'.format(spec_date.year - 1911,\
		# 	spec_date.month, spec_date.day)

		data_df = DataFrame(data=content[key],
							columns=['code','name','volume','transaction','turnover',
									 'open','high','low','close','UD','difference',
									 'last_buy', 'last_buy_volume',
									 'last_sell','last_sell_volume','PE_ratio'])

		data_df = data_df.applymap(lambda x: re.sub(",","",x))# clear comma
		data_df.replace({'UD':{'
+
':'+',
							   '
-
':'-'}},
						inplace=True)

		return data_df

	def get_otc_one_day(self, spec_date):
		date_str = '{0}/{1:02d}/{2:02d}'.format(spec_date.year-1911, spec_date.month, spec_date.day)

		ttime = str(int(time.time()*100))
		url = 'http://www.tpex.org.tw/web/stock/aftertrading/daily_close_quotes/stk_quote_result.php?l=zh-tw&d={}&_={}'.format(date_str, ttime)
		page = requests.get(url)

		if not page.ok:
			logging.error("Can not get OTC data at {}".format(date_str))
		# print(page.content)
		content = page.json()

		# print(content)
		# key = 'Nodata'
		if (len(content['aaData']) + len(content['mmData'])) == 0:
			print('No data at this day ' + date_str)
			return -1
		data_df = DataFrame(data=content['aaData'] + content['mmData'],
							columns=['code','name','close','difference','open',
									 'high','low','avg','volume','turnover',
									 'transaction','last_buy',
									 'last_sell','NumOfShare','NextRefPrice',
									 'NextUpperPrice', 'NextLowerPrice'])

		data_df = data_df.applymap(lambda x: re.sub(",","",x))# clear comma

		return data_df

	def check_all_tse_data(self):
		Filelist = os.listdir(self.prefix)
		if 'offday.xlsx' in Filelist:
			offday_ser = pd.read_excel(self.prefix + '/offday.xlsx')
			offday_ser = offday_ser['date'].copy()
		else:
			offday_ser = Series(name='date', data='First')

		offday_update = False
		lastday_update = False

		Now = datetime.now()
		Nowdate = datetime(Now.year, Now.month, Now.day)

		if 'lastday.txt' in Filelist:
			with open(self.prefix + '/lastday.txt', 'r') as f:
				read_data = f.read()
				f.close()
				Startdate = datetime(int(read_data[0:4]),
									int(read_data[4:6]),
									int(read_data[6:8]))
		else:
			#Start from 2004(093)/02/11
			Startdate = datetime(2004, 2, 11)

		datediff = timedelta(days=1)

		while Startdate <= Nowdate:
			date_str = '{0}{1:02d}{2:02d}'.\
					format(Startdate.year-1911,Startdate.month, Startdate.day)
			print('Read ' + date_str)
			if ('%s.xlsx' %(date_str)) not in Filelist:# not in FileList
				if (offday_ser != date_str).all():# not a offday
					lastday_update = True
					data_df = self.get_tse_one_day(Startdate) # collect data
					if isinstance(data_df, DataFrame):# success
						data_df.to_excel('{0}/{1}.xlsx'.format(self.prefix,date_str))# save data
					else:# is an offday, update offday series
						offday_ser.set_value( len(offday_ser), date_str)
						offday_update = True
						print(date_str + 'is an offday')
				else:
					print(date_str + ' is known as an offday')
			else:
				print(date_str + ' is in FileList')
			Startdate = Startdate + datediff

		if offday_update:
			offday_ser.to_excel(self.prefix + '/offday.xlsx')

		if lastday_update:
			with open(self.prefix + '/lastday.txt', 'w') as f:
				# Nowdate += timedelta(days=-1)
				date_str = '{0}{1:02d}{2:02d}'.\
					format(Nowdate.year,Nowdate.month, Nowdate.day)
				f.write(date_str)
				f.close()

	def check_all_otc_data(self):
		Filelist = os.listdir(self.prefix)
		if 'offdayOTC.xlsx' in Filelist:
			offday_ser = pd.read_excel(self.prefix + '/offdayOTC.xlsx')
			offday_ser = offday_ser['date'].copy()
		else:
			offday_ser = Series(name='date', data='First')

		offday_update = False
		lastday_update = False

		Now = datetime.now()
		Nowdate = datetime(Now.year, Now.month, Now.day)

		if 'lastdayOTC.txt' in Filelist:
			with open(self.prefix + '/lastdayOTC.txt', 'r') as f:
				read_data = f.read()
				f.close()
				Startdate = datetime(int(read_data[0:4]),
									int(read_data[4:6]),
									int(read_data[6:8]))
		else:
			#Start from 2007(096)/04/23
			Startdate = datetime(2007, 4, 23)
			# Startdate = datetime(2008, 2, 28)

		datediff = timedelta(days=1)

		while Startdate <= Nowdate:
			date_str = '{0}{1:02d}{2:02d}'.\
					format(Startdate.year-1911,Startdate.month, Startdate.day)
			print('Read ' + date_str + ' OTC')
			if ('%sOTC.xlsx' %(date_str)) not in Filelist:# not in FileList
				if (offday_ser != date_str).all():# not a offday
					lastday_update = True

					time.sleep(np.random.random())

					data_df = self.get_otc_one_day(Startdate) # collect data
					if isinstance(data_df, DataFrame):# success
						data_df.to_excel('{0}/{1}OTC.xlsx'.format(self.prefix,date_str))# save data
					else:# is an offday, update offday series
						offday_ser.set_value( len(offday_ser), date_str)
						offday_update = True
						print(date_str + 'is an offday')
				else:
					print(date_str + ' is known as an offday')
			else:
				print(date_str + ' is in FileList')
			Startdate = Startdate + datediff

		if offday_update:
			offday_ser.to_excel(self.prefix + '/offdayOTC.xlsx')

		if lastday_update:
			with open(self.prefix + '/lastdayOTC.txt', 'w') as f:
				# Nowdate += timedelta(days=-1)
				date_str = '{0}{1:02d}{2:02d}'.\
					format(Nowdate.year,Nowdate.month, Nowdate.day)
				f.write(date_str)
				f.close()


"""
============================================================
Sparse recovery: feature selection for sparse linear models
============================================================

Given a small number of observations, we want to recover which features
of X are relevant to explain y. For this :ref:`sparse linear models
<l1_feature_selection>` can outperform standard statistical tests if the
true model is sparse, i.e. if a small fraction of the features are
relevant.

As detailed in :ref:`the compressive sensing notes
<compressive_sensing>`, the ability of L1-based approach to identify the
relevant variables depends on the sparsity of the ground truth, the
number of samples, the number of features, the conditioning of the
design matrix on the signal subspace, the amount of noise, and the
absolute value of the smallest non-zero coefficient [Wainwright2006]
(http://statistics.berkeley.edu/tech-reports/709.pdf).

Here we keep all parameters constant and vary the conditioning of the
design matrix. For a well-conditioned design matrix (small mutual
incoherence) we are exactly in compressive sensing conditions (i.i.d
Gaussian sensing matrix), and L1-recovery with the Lasso performs very
well. For an ill-conditioned matrix (high mutual incoherence),
regressors are very correlated, and the Lasso randomly selects one.
However, randomized-Lasso can recover the ground truth well.

In each situation, we first vary the alpha parameter setting the sparsity
of the estimated model and look at the stability scores of the randomized
Lasso. This analysis, knowing the ground truth, shows an optimal regime
in which relevant features stand out from the irrelevant ones. If alpha
is chosen too small, non-relevant variables enter the model. On the
opposite, if alpha is selected too large, the Lasso is equivalent to
stepwise regression, and thus brings no advantage over a univariate
F-test.

In a second time, we set alpha and compare the performance of different
feature selection methods, using the area under curve (AUC) of the
precision-recall.
"""
print(__doc__)

# Author: Alexandre Gramfort and Gael Varoquaux
# License: BSD 3 clause

import warnings

import matplotlib.pyplot as plt
import numpy as np
from scipy import linalg

from sklearn.linear_model import (RandomizedLasso, lasso_stability_path,
                                  LassoLarsCV)
from sklearn.feature_selection import f_regression
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import auc, precision_recall_curve
from sklearn.ensemble import ExtraTreesRegressor
from sklearn.utils.extmath import pinvh
from sklearn.utils import ConvergenceWarning

def mutual_incoherence(X_relevant, X_irelevant):
    """Mutual incoherence, as defined by formula (26a) of [Wainwright2006].
    """
    projector = np.dot(np.dot(X_irelevant.T, X_relevant),
                       pinvh(np.dot(X_relevant.T, X_relevant)))
    return np.max(np.abs(projector).sum(axis=1))

for conditioning in (1, 1e-4):
    ###########################################################################
    # Simulate regression data with a correlated design
    n_features = 501
    n_relevant_features = 3
    noise_level = .2
    coef_min = .2
    # The Donoho-Tanner phase transition is around n_samples=25: below we
    # will completely fail to recover in the well-conditioned case
    n_samples = 25
    block_size = n_relevant_features

    rng = np.random.RandomState(42)

    # The coefficients of our model
    coef = np.zeros(n_features)
    coef[:n_relevant_features] = coef_min + rng.rand(n_relevant_features)

    # The correlation of our design: variables correlated by blocs of 3
    corr = np.zeros((n_features, n_features))
    for i in range(0, n_features, block_size):
        corr[i:i + block_size, i:i + block_size] = 1 - conditioning
    corr.flat[::n_features + 1] = 1
    corr = linalg.cholesky(corr)

    # Our design
    X = rng.normal(size=(n_samples, n_features))
    X = np.dot(X, corr)
    # Keep [Wainwright2006] (26c) constant
    X[:n_relevant_features] /= np.abs(
        linalg.svdvals(X[:n_relevant_features])).max()
    X = StandardScaler().fit_transform(X.copy())

    # The output variable
    y = np.dot(X, coef)
    y /= np.std(y)
    # We scale the added noise as a function of the average correlation
    # between the design and the output variable
    y += noise_level * rng.normal(size=n_samples)
    mi = mutual_incoherence(X[:, :n_relevant_features],
                            X[:, n_relevant_features:])

    ###########################################################################
    # Plot stability selection path, using a high eps for early stopping
    # of the path, to save computation time
    alpha_grid, scores_path = lasso_stability_path(X, y, random_state=42,
                                                   eps=0.05)

    plt.figure()
    # We plot the path as a function of alpha/alpha_max to the power 1/3: the
    # power 1/3 scales the path less brutally than the log, and enables to
    # see the progression along the path
    hg = plt.plot(alpha_grid[1:] ** .333, scores_path[coef != 0].T[1:], 'r')
    hb = plt.plot(alpha_grid[1:] ** .333, scores_path[coef == 0].T[1:], 'k')
    ymin, ymax = plt.ylim()
    plt.xlabel(r'$(\alpha / \alpha_{max})^{1/3}$')
    plt.ylabel('Stability score: proportion of times selected')
    plt.title('Stability Scores Path - Mutual incoherence: %.1f' % mi)
    plt.axis('tight')
    plt.legend((hg[0], hb[0]), ('relevant features', 'irrelevant features'),
               loc='best')

    ###########################################################################
    # Plot the estimated stability scores for a given alpha

    # Use 6-fold cross-validation rather than the default 3-fold: it leads to
    # a better choice of alpha:
    # Stop the user warnings outputs- they are not necessary for the example
    # as it is specifically set up to be challenging.
    with warnings.catch_warnings():
        warnings.simplefilter('ignore', UserWarning)
        warnings.simplefilter('ignore', ConvergenceWarning)
        lars_cv = LassoLarsCV(cv=6).fit(X, y)

    # Run the RandomizedLasso: we use a paths going down to .1*alpha_max
    # to avoid exploring the regime in which very noisy variables enter
    # the model
    alphas = np.linspace(lars_cv.alphas_[0], .1 * lars_cv.alphas_[0], 6)
    clf = RandomizedLasso(alpha=alphas, random_state=42).fit(X, y)
    trees = ExtraTreesRegressor(100).fit(X, y)
    # Compare with F-score
    F, _ = f_regression(X, y)

    plt.figure()
    for name, score in [('F-test', F),
                        ('Stability selection', clf.scores_),
                        ('Lasso coefs', np.abs(lars_cv.coef_)),
                        ('Trees', trees.feature_importances_),
                        ]:
        precision, recall, thresholds = precision_recall_curve(coef != 0,
                                                               score)
        plt.semilogy(np.maximum(score / np.max(score), 1e-4),
                     label="%s. AUC: %.3f" % (name, auc(recall, precision)))

    plt.plot(np.where(coef != 0)[0], [2e-4] * n_relevant_features, 'mo',
             label="Ground truth")
    plt.xlabel("Features")
    plt.ylabel("Score")
    # Plot only the 100 first coefficients
    plt.xlim(0, 100)
    plt.legend(loc='best')
    plt.title('Feature selection scores - Mutual incoherence: %.1f'
              % mi)

plt.show()


"""
Tests for the birch clustering algorithm.
"""

from scipy import sparse
import numpy as np

from sklearn.cluster.tests.common import generate_clustered_data
from sklearn.cluster.birch import Birch
from sklearn.cluster.hierarchical import AgglomerativeClustering
from sklearn.datasets import make_blobs
from sklearn.linear_model import ElasticNet
from sklearn.metrics import pairwise_distances_argmin, v_measure_score

from sklearn.utils.testing import assert_greater_equal
from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_greater
from sklearn.utils.testing import assert_almost_equal
from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import assert_raises
from sklearn.utils.testing import assert_warns

def test_n_samples_leaves_roots():
    # Sanity check for the number of samples in leaves and roots
    X, y = make_blobs(n_samples=10)
    brc = Birch()
    brc.fit(X)
    n_samples_root = sum([sc.n_samples_ for sc in brc.root_.subclusters_])
    n_samples_leaves = sum([sc.n_samples_ for leaf in brc._get_leaves()
                            for sc in leaf.subclusters_])
    assert_equal(n_samples_leaves, X.shape[0])
    assert_equal(n_samples_root, X.shape[0])

def test_partial_fit():
    # Test that fit is equivalent to calling partial_fit multiple times
    X, y = make_blobs(n_samples=100)
    brc = Birch(n_clusters=3)
    brc.fit(X)
    brc_partial = Birch(n_clusters=None)
    brc_partial.partial_fit(X[:50])
    brc_partial.partial_fit(X[50:])
    assert_array_equal(brc_partial.subcluster_centers_,
                       brc.subcluster_centers_)

    # Test that same global labels are obtained after calling partial_fit
    # with None
    brc_partial.set_params(n_clusters=3)
    brc_partial.partial_fit(None)
    assert_array_equal(brc_partial.subcluster_labels_, brc.subcluster_labels_)

def test_birch_predict():
    # Test the predict method predicts the nearest centroid.
    rng = np.random.RandomState(0)
    X = generate_clustered_data(n_clusters=3, n_features=3,
                                n_samples_per_cluster=10)

    # n_samples * n_samples_per_cluster
    shuffle_indices = np.arange(30)
    rng.shuffle(shuffle_indices)
    X_shuffle = X[shuffle_indices, :]
    brc = Birch(n_clusters=4, threshold=1.)
    brc.fit(X_shuffle)
    centroids = brc.subcluster_centers_
    assert_array_equal(brc.labels_, brc.predict(X_shuffle))
    nearest_centroid = pairwise_distances_argmin(X_shuffle, centroids)
    assert_almost_equal(v_measure_score(nearest_centroid, brc.labels_), 1.0)

def test_n_clusters():
    # Test that n_clusters param works properly
    X, y = make_blobs(n_samples=100, centers=10)
    brc1 = Birch(n_clusters=10)
    brc1.fit(X)
    assert_greater(len(brc1.subcluster_centers_), 10)
    assert_equal(len(np.unique(brc1.labels_)), 10)

    # Test that n_clusters = Agglomerative Clustering gives
    # the same results.
    gc = AgglomerativeClustering(n_clusters=10)
    brc2 = Birch(n_clusters=gc)
    brc2.fit(X)
    assert_array_equal(brc1.subcluster_labels_, brc2.subcluster_labels_)
    assert_array_equal(brc1.labels_, brc2.labels_)

    # Test that the wrong global clustering step raises an Error.
    clf = ElasticNet()
    brc3 = Birch(n_clusters=clf)
    assert_raises(ValueError, brc3.fit, X)

    # Test that a small number of clusters raises a warning.
    brc4 = Birch(threshold=10000.)
    assert_warns(UserWarning, brc4.fit, X)

def test_sparse_X():
    # Test that sparse and dense data give same results
    X, y = make_blobs(n_samples=100, centers=10)
    brc = Birch(n_clusters=10)
    brc.fit(X)

    csr = sparse.csr_matrix(X)
    brc_sparse = Birch(n_clusters=10)
    brc_sparse.fit(csr)

    assert_array_equal(brc.labels_, brc_sparse.labels_)
    assert_array_equal(brc.subcluster_centers_,
                       brc_sparse.subcluster_centers_)

def check_branching_factor(node, branching_factor):
    subclusters = node.subclusters_
    assert_greater_equal(branching_factor, len(subclusters))
    for cluster in subclusters:
        if cluster.child_:
            check_branching_factor(cluster.child_, branching_factor)

def test_branching_factor():
    # Test that nodes have at max branching_factor number of subclusters
    X, y = make_blobs()
    branching_factor = 9

    # Purposefully set a low threshold to maximize the subclusters.
    brc = Birch(n_clusters=None, branching_factor=branching_factor,
                threshold=0.01)
    brc.fit(X)
    check_branching_factor(brc.root_, branching_factor)
    brc = Birch(n_clusters=3, branching_factor=branching_factor,
                threshold=0.01)
    brc.fit(X)
    check_branching_factor(brc.root_, branching_factor)

    # Raises error when branching_factor is set to one.
    brc = Birch(n_clusters=None, branching_factor=1, threshold=0.01)
    assert_raises(ValueError, brc.fit, X)

def check_threshold(birch_instance, threshold):
    """Use the leaf linked list for traversal"""
    current_leaf = birch_instance.dummy_leaf_.next_leaf_
    while current_leaf:
        subclusters = current_leaf.subclusters_
        for sc in subclusters:
            assert_greater_equal(threshold, sc.radius)
        current_leaf = current_leaf.next_leaf_

def test_threshold():
    # Test that the leaf subclusters have a threshold lesser than radius
    X, y = make_blobs(n_samples=80, centers=4)
    brc = Birch(threshold=0.5, n_clusters=None)
    brc.fit(X)
    check_threshold(brc, 0.5)

    brc = Birch(threshold=5.0, n_clusters=None)
    brc.fit(X)
    check_threshold(brc, 5.)


""" Principal Component Analysis
"""

# Author: Alexandre Gramfort <alexandre.gramfort@inria.fr>
#         Olivier Grisel <olivier.grisel@ensta.org>
#         Mathieu Blondel <mathieu@mblondel.org>
#         Denis A. Engemann <d.engemann@fz-juelich.de>
#         Michael Eickenberg <michael.eickenberg@inria.fr>
#
# License: BSD 3 clause

from math import log, sqrt

import numpy as np
from scipy import linalg
from scipy.special import gammaln

from ..base import BaseEstimator, TransformerMixin
from ..utils import check_random_state, as_float_array
from ..utils import check_array
from ..utils.extmath import fast_dot, fast_logdet, randomized_svd
from ..utils.validation import check_is_fitted

def _assess_dimension_(spectrum, rank, n_samples, n_features):
    """Compute the likelihood of a rank ``rank`` dataset

    The dataset is assumed to be embedded in gaussian noise of shape(n,
    dimf) having spectrum ``spectrum``.

    Parameters
    ----------
    spectrum: array of shape (n)
        Data spectrum.
    rank: int
        Tested rank value.
    n_samples: int
        Number of samples.
    n_features: int
        Number of features.

    Returns
    -------
    ll: float,
        The log-likelihood

    Notes
    -----
    This implements the method of `Thomas P. Minka:
    Automatic Choice of Dimensionality for PCA. NIPS 2000: 598-604`
    """
    if rank > len(spectrum):
        raise ValueError("The tested rank cannot exceed the rank of the"
                         " dataset")

    pu = -rank * log(2.)
    for i in range(rank):
        pu += (gammaln((n_features - i) / 2.)
               - log(np.pi) * (n_features - i) / 2.)

    pl = np.sum(np.log(spectrum[:rank]))
    pl = -pl * n_samples / 2.

    if rank == n_features:
        pv = 0
        v = 1
    else:
        v = np.sum(spectrum[rank:]) / (n_features - rank)
        pv = -np.log(v) * n_samples * (n_features - rank) / 2.

    m = n_features * rank - rank * (rank + 1.) / 2.
    pp = log(2. * np.pi) * (m + rank + 1.) / 2.

    pa = 0.
    spectrum_ = spectrum.copy()
    spectrum_[rank:n_features] = v
    for i in range(rank):
        for j in range(i + 1, len(spectrum)):
            pa += log((spectrum[i] - spectrum[j]) *
                      (1. / spectrum_[j] - 1. / spectrum_[i])) + log(n_samples)

    ll = pu + pl + pv + pp - pa / 2. - rank * log(n_samples) / 2.

    return ll

def _infer_dimension_(spectrum, n_samples, n_features):
    """Infers the dimension of a dataset of shape (n_samples, n_features)

    The dataset is described by its spectrum `spectrum`.
    """
    n_spectrum = len(spectrum)
    ll = np.empty(n_spectrum)
    for rank in range(n_spectrum):
        ll[rank] = _assess_dimension_(spectrum, rank, n_samples, n_features)
    return ll.argmax()

class PCA(BaseEstimator, TransformerMixin):
    """Principal component analysis (PCA)

    Linear dimensionality reduction using Singular Value Decomposition of the
    data and keeping only the most significant singular vectors to project the
    data to a lower dimensional space.

    This implementation uses the scipy.linalg implementation of the singular
    value decomposition. It only works for dense arrays and is not scalable to
    large dimensional data.

    The time complexity of this implementation is ``O(n ** 3)`` assuming
    n ~ n_samples ~ n_features.

    Read more in the :ref:`User Guide <PCA>`.

    Parameters
    ----------
    n_components : int, None or string
        Number of components to keep.
        if n_components is not set all components are kept::

            n_components == min(n_samples, n_features)

        if n_components == 'mle', Minka\'s MLE is used to guess the dimension
        if ``0 < n_components < 1``, select the number of components such that
        the amount of variance that needs to be explained is greater than the
        percentage specified by n_components

    copy : bool
        If False, data passed to fit are overwritten and running
        fit(X).transform(X) will not yield the expected results,
        use fit_transform(X) instead.

    whiten : bool, optional
        When True (False by default) the `components_` vectors are divided
        by n_samples times singular values to ensure uncorrelated outputs
        with unit component-wise variances.

        Whitening will remove some information from the transformed signal
        (the relative variance scales of the components) but can sometime
        improve the predictive accuracy of the downstream estimators by
        making there data respect some hard-wired assumptions.

    Attributes
    ----------
    components_ : array, [n_components, n_features]
        Principal axes in feature space, representing the directions of
        maximum variance in the data.

    explained_variance_ratio_ : array, [n_components]
        Percentage of variance explained by each of the selected components.
        If ``n_components`` is not set then all components are stored and the
        sum of explained variances is equal to 1.0

    mean_ : array, [n_features]
        Per-feature empirical mean, estimated from the training set.

    n_components_ : int
        The estimated number of components. Relevant when n_components is set
        to 'mle' or a number between 0 and 1 to select using explained
        variance.

    noise_variance_ : float
        The estimated noise covariance following the Probabilistic PCA model
        from Tipping and Bishop 1999. See "Pattern Recognition and
        Machine Learning" by C. Bishop, 12.2.1 p. 574 or
        http://www.miketipping.com/papers/met-mppca.pdf. It is required to
        computed the estimated data covariance and score samples.

    Notes
    -----
    For n_components='mle', this class uses the method of `Thomas P. Minka:
    Automatic Choice of Dimensionality for PCA. NIPS 2000: 598-604`

    Implements the probabilistic PCA model from:
    M. Tipping and C. Bishop, Probabilistic Principal Component Analysis,
    Journal of the Royal Statistical Society, Series B, 61, Part 3, pp. 611-622
    via the score and score_samples methods.
    See http://www.miketipping.com/papers/met-mppca.pdf

    Due to implementation subtleties of the Singular Value Decomposition (SVD),
    which is used in this implementation, running fit twice on the same matrix
    can lead to principal components with signs flipped (change in direction).
    For this reason, it is important to always use the same estimator object to
    transform data in a consistent fashion.

    Examples
    --------

    >>> import numpy as np
    >>> from sklearn.decomposition import PCA
    >>> X = np.array([[-1, -1], [-2, -1], [-3, -2], [1, 1], [2, 1], [3, 2]])
    >>> pca = PCA(n_components=2)
    >>> pca.fit(X)
    PCA(copy=True, n_components=2, whiten=False)
    >>> print(pca.explained_variance_ratio_) # doctest: +ELLIPSIS
    [ 0.99244...  0.00755...]

    See also
    --------
    RandomizedPCA
    KernelPCA
    SparsePCA
    TruncatedSVD
    """
    def __init__(self, n_components=None, copy=True, whiten=False):
        self.n_components = n_components
        self.copy = copy
        self.whiten = whiten

    def fit(self, X, y=None):
        """Fit the model with X.

        Parameters
        ----------
        X: array-like, shape (n_samples, n_features)
            Training data, where n_samples in the number of samples
            and n_features is the number of features.

        Returns
        -------
        self : object
            Returns the instance itself.
        """
        self._fit(X)
        return self

    def fit_transform(self, X, y=None):
        """Fit the model with X and apply the dimensionality reduction on X.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            Training data, where n_samples is the number of samples
            and n_features is the number of features.

        Returns
        -------
        X_new : array-like, shape (n_samples, n_components)

        """
        U, S, V = self._fit(X)
        U = U[:, :self.n_components_]

        if self.whiten:
            # X_new = X * V / S * sqrt(n_samples) = U * sqrt(n_samples)
            U *= sqrt(X.shape[0])
        else:
            # X_new = X * V = U * S * V^T * V = U * S
            U *= S[:self.n_components_]

        return U

    def _fit(self, X):
        """Fit the model on X

        Parameters
        ----------
        X: array-like, shape (n_samples, n_features)
            Training vector, where n_samples in the number of samples and
            n_features is the number of features.

        Returns
        -------
        U, s, V : ndarrays
            The SVD of the input data, copied and centered when
            requested.
        """
        X = check_array(X)
        n_samples, n_features = X.shape
        X = as_float_array(X, copy=self.copy)
        # Center data
        self.mean_ = np.mean(X, axis=0)
        X -= self.mean_
        U, S, V = linalg.svd(X, full_matrices=False)
        explained_variance_ = (S ** 2) / n_samples
        explained_variance_ratio_ = (explained_variance_ /
                                     explained_variance_.sum())

        components_ = V

        n_components = self.n_components
        if n_components is None:
            n_components = n_features
        elif n_components == 'mle':
            if n_samples < n_features:
                raise ValueError("n_components='mle' is only supported "
                                 "if n_samples >= n_features")

            n_components = _infer_dimension_(explained_variance_,
                                             n_samples, n_features)
        elif not 0 <= n_components <= n_features:
            raise ValueError("n_components=%r invalid for n_features=%d"
                             % (n_components, n_features))

        if 0 < n_components < 1.0:
            # number of components for which the cumulated explained variance
            # percentage is superior to the desired threshold
            ratio_cumsum = explained_variance_ratio_.cumsum()
            n_components = np.sum(ratio_cumsum < n_components) + 1

        # Compute noise covariance using Probabilistic PCA model
        # The sigma2 maximum likelihood (cf. eq. 12.46)
        if n_components < n_features:
            self.noise_variance_ = explained_variance_[n_components:].mean()
        else:
            self.noise_variance_ = 0.

        # store n_samples to revert whitening when getting covariance
        self.n_samples_ = n_samples

        self.components_ = components_[:n_components]
        self.explained_variance_ = explained_variance_[:n_components]
        explained_variance_ratio_ = explained_variance_ratio_[:n_components]
        self.explained_variance_ratio_ = explained_variance_ratio_
        self.n_components_ = n_components

        return (U, S, V)

    def get_covariance(self):
        """Compute data covariance with the generative model.

        ``cov = components_.T * S**2 * components_ + sigma2 * eye(n_features)``
        where  S**2 contains the explained variances.

        Returns
        -------
        cov : array, shape=(n_features, n_features)
            Estimated covariance of data.
        """
        components_ = self.components_
        exp_var = self.explained_variance_
        if self.whiten:
            components_ = components_ * np.sqrt(exp_var[:, np.newaxis])
        exp_var_diff = np.maximum(exp_var - self.noise_variance_, 0.)
        cov = np.dot(components_.T * exp_var_diff, components_)
        cov.flat[::len(cov) + 1] += self.noise_variance_  # modify diag inplace
        return cov

    def get_precision(self):
        """Compute data precision matrix with the generative model.

        Equals the inverse of the covariance but computed with
        the matrix inversion lemma for efficiency.

        Returns
        -------
        precision : array, shape=(n_features, n_features)
            Estimated precision of data.
        """
        n_features = self.components_.shape[1]

        # handle corner cases first
        if self.n_components_ == 0:
            return np.eye(n_features) / self.noise_variance_
        if self.n_components_ == n_features:
            return linalg.inv(self.get_covariance())

        # Get precision using matrix inversion lemma
        components_ = self.components_
        exp_var = self.explained_variance_
        exp_var_diff = np.maximum(exp_var - self.noise_variance_, 0.)
        precision = np.dot(components_, components_.T) / self.noise_variance_
        precision.flat[::len(precision) + 1] += 1. / exp_var_diff
        precision = np.dot(components_.T,
                           np.dot(linalg.inv(precision), components_))
        precision /= -(self.noise_variance_ ** 2)
        precision.flat[::len(precision) + 1] += 1. / self.noise_variance_
        return precision

    def transform(self, X):
        """Apply the dimensionality reduction on X.

        X is projected on the first principal components previous extracted
        from a training set.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            New data, where n_samples is the number of samples
            and n_features is the number of features.

        Returns
        -------
        X_new : array-like, shape (n_samples, n_components)

        """
        check_is_fitted(self, 'mean_')

        X = check_array(X)
        if self.mean_ is not None:
            X = X - self.mean_
        X_transformed = fast_dot(X, self.components_.T)
        if self.whiten:
            X_transformed /= np.sqrt(self.explained_variance_)
        return X_transformed

    def inverse_transform(self, X):
        """Transform data back to its original space, i.e.,
        return an input X_original whose transform would be X

        Parameters
        ----------
        X : array-like, shape (n_samples, n_components)
            New data, where n_samples is the number of samples
            and n_components is the number of components.

        Returns
        -------
        X_original array-like, shape (n_samples, n_features)
        """
        check_is_fitted(self, 'mean_')

        if self.whiten:
            return fast_dot(
                X,
                np.sqrt(self.explained_variance_[:, np.newaxis]) *
                self.components_) + self.mean_
        else:
            return fast_dot(X, self.components_) + self.mean_

    def score_samples(self, X):
        """Return the log-likelihood of each sample

        See. "Pattern Recognition and Machine Learning"
        by C. Bishop, 12.2.1 p. 574
        or http://www.miketipping.com/papers/met-mppca.pdf

        Parameters
        ----------
        X: array, shape(n_samples, n_features)
            The data.

        Returns
        -------
        ll: array, shape (n_samples,)
            Log-likelihood of each sample under the current model
        """
        check_is_fitted(self, 'mean_')

        X = check_array(X)
        Xr = X - self.mean_
        n_features = X.shape[1]
        log_like = np.zeros(X.shape[0])
        precision = self.get_precision()
        log_like = -.5 * (Xr * (np.dot(Xr, precision))).sum(axis=1)
        log_like -= .5 * (n_features * log(2. * np.pi)
                          - fast_logdet(precision))
        return log_like

    def score(self, X, y=None):
        """Return the average log-likelihood of all samples

        See. "Pattern Recognition and Machine Learning"
        by C. Bishop, 12.2.1 p. 574
        or http://www.miketipping.com/papers/met-mppca.pdf

        Parameters
        ----------
        X: array, shape(n_samples, n_features)
            The data.

        Returns
        -------
        ll: float
            Average log-likelihood of the samples under the current model
        """
        return np.mean(self.score_samples(X))

class RandomizedPCA(BaseEstimator, TransformerMixin):
    """Principal component analysis (PCA) using randomized SVD

    Linear dimensionality reduction using approximated Singular Value
    Decomposition of the data and keeping only the most significant
    singular vectors to project the data to a lower dimensional space.

    Read more in the :ref:`User Guide <RandomizedPCA>`.

    Parameters
    ----------
    n_components : int, optional
        Maximum number of components to keep. When not given or None, this
        is set to n_features (the second dimension of the training data).

    copy : bool
        If False, data passed to fit are overwritten and running
        fit(X).transform(X) will not yield the expected results,
        use fit_transform(X) instead.

    iterated_power : int, optional
        Number of iterations for the power method. 3 by default.

    whiten : bool, optional
        When True (False by default) the `components_` vectors are divided
        by the singular values to ensure uncorrelated outputs with unit
        component-wise variances.

        Whitening will remove some information from the transformed signal
        (the relative variance scales of the components) but can sometime
        improve the predictive accuracy of the downstream estimators by
        making their data respect some hard-wired assumptions.

    random_state : int or RandomState instance or None (default)
        Pseudo Random Number generator seed control. If None, use the
        numpy.random singleton.

    Attributes
    ----------
    components_ : array, [n_components, n_features]
        Components with maximum variance.

    explained_variance_ratio_ : array, [n_components]
        Percentage of variance explained by each of the selected components. \
        k is not set then all components are stored and the sum of explained \
        variances is equal to 1.0

    mean_ : array, [n_features]
        Per-feature empirical mean, estimated from the training set.

    Examples
    --------
    >>> import numpy as np
    >>> from sklearn.decomposition import RandomizedPCA
    >>> X = np.array([[-1, -1], [-2, -1], [-3, -2], [1, 1], [2, 1], [3, 2]])
    >>> pca = RandomizedPCA(n_components=2)
    >>> pca.fit(X)                 # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
    RandomizedPCA(copy=True, iterated_power=3, n_components=2,
           random_state=None, whiten=False)
    >>> print(pca.explained_variance_ratio_) # doctest: +ELLIPSIS
    [ 0.99244...  0.00755...]

    See also
    --------
    PCA
    TruncatedSVD

    References
    ----------

    .. [Halko2009] `Finding structure with randomness: Stochastic algorithms
      for constructing approximate matrix decompositions Halko, et al., 2009
      (arXiv:909)`

    .. [MRT] `A randomized algorithm for the decomposition of matrices
      Per-Gunnar Martinsson, Vladimir Rokhlin and Mark Tygert`

    """

    def __init__(self, n_components=None, copy=True, iterated_power=3,
                 whiten=False, random_state=None):
        self.n_components = n_components
        self.copy = copy
        self.iterated_power = iterated_power
        self.whiten = whiten
        self.random_state = random_state

    def fit(self, X, y=None):
        """Fit the model with X by extracting the first principal components.

        Parameters
        ----------
        X: array-like, shape (n_samples, n_features)
            Training data, where n_samples in the number of samples
            and n_features is the number of features.

        Returns
        -------
        self : object
            Returns the instance itself.
        """
        self._fit(check_array(X))
        return self

    def _fit(self, X):
        """Fit the model to the data X.

        Parameters
        ----------
        X: array-like, shape (n_samples, n_features)
            Training vector, where n_samples in the number of samples and
            n_features is the number of features.

        Returns
        -------
        X : ndarray, shape (n_samples, n_features)
            The input data, copied, centered and whitened when requested.
        """
        random_state = check_random_state(self.random_state)
        X = np.atleast_2d(as_float_array(X, copy=self.copy))

        n_samples = X.shape[0]

        # Center data
        self.mean_ = np.mean(X, axis=0)
        X -= self.mean_
        if self.n_components is None:
            n_components = X.shape[1]
        else:
            n_components = self.n_components

        U, S, V = randomized_svd(X, n_components,
                                 n_iter=self.iterated_power,
                                 random_state=random_state)

        self.explained_variance_ = exp_var = (S ** 2) / n_samples
        full_var = np.var(X, axis=0).sum()
        self.explained_variance_ratio_ = exp_var / full_var

        if self.whiten:
            self.components_ = V / S[:, np.newaxis] * sqrt(n_samples)
        else:
            self.components_ = V

        return X

    def transform(self, X, y=None):
        """Apply dimensionality reduction on X.

        X is projected on the first principal components previous extracted
        from a training set.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            New data, where n_samples in the number of samples
            and n_features is the number of features.

        Returns
        -------
        X_new : array-like, shape (n_samples, n_components)

        """
        check_is_fitted(self, 'mean_')

        X = check_array(X)
        if self.mean_ is not None:
            X = X - self.mean_

        X = fast_dot(X, self.components_.T)
        return X

    def fit_transform(self, X, y=None):
        """Fit the model with X and apply the dimensionality reduction on X.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            New data, where n_samples in the number of samples
            and n_features is the number of features.

        Returns
        -------
        X_new : array-like, shape (n_samples, n_components)

        """
        X = check_array(X)
        X = self._fit(X)
        return fast_dot(X, self.components_.T)

    def inverse_transform(self, X, y=None):
        """Transform data back to its original space.

        Returns an array X_original whose transform would be X.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_components)
            New data, where n_samples in the number of samples
            and n_components is the number of components.

        Returns
        -------
        X_original array-like, shape (n_samples, n_features)

        Notes
        -----
        If whitening is enabled, inverse_transform does not compute the
        exact inverse operation of transform.
        """
        check_is_fitted(self, 'mean_')

        X_original = fast_dot(X, self.components_)
        if self.mean_ is not None:
            X_original = X_original + self.mean_
        return X_original


import numpy as np
import matplotlib.pyplot as plt
from errsim import *

def label(d, pe, pb, n):
    if pb is None:
        pb = pe
        label = 'd={} pe={} n={} BSC'.format(d, pe, n)
    else:
        label = 'd={} pe={} n={} pb={}'.format(d, pe, n, pb)
    return label

def plot(pe, fpath=None):
    fig, ax = plt.subplots(nrows=1, ncols=1, figsize=plt.figaspect(1/2))

    r = np.arange(8, 65)

    pWL = jointpmf5(pe, pe, 128)
    ax.plot(r, r_vs_pegd(pWL, 3, r) , 'g--', lw=2, label=label(3, pe, None, 128))
    ax.plot(r, r_vs_pegd(pWL, 6, r) , 'g-', lw=2, label=label(6, pe, None, 128))

    pWL = jointpmf5(pe, .1, 128)
    ax.plot(r, r_vs_pegd(pWL, 3, r) , 'b--', lw=2, label=label(3, pe, .1, 128))
    ax.plot(r, r_vs_pegd(pWL, 6, r) , 'b-', lw=2, label=label(6, pe, .1, 128))

    pWL = jointpmf5(pe, .5, 128)
    ax.plot(r, r_vs_pegd(pWL, 3, r) , 'r--', lw=2, label=label(3, pe, .5, 128))
    ax.plot(r, r_vs_pegd(pWL, 6, r) , 'r-', lw=2, label=label(6, pe, .5, 128))

    ax.set_yscale('log')
    ax.set_xticks(r[::8])
    ax.set_xlim(r[0], r[-1])
    #ax.set_ylim(1e-30, 1e-1)
    ax.set_xlabel('Burst error correction capability, $r$')
    ax.set_ylabel('$P_{egd}$')
    ax.set_title('Probability of Exceeding Guarenteed Error Detection Capability')

    ax.legend(loc='lower right')
    ax.grid(True)

    #plt.tight_layout()
    if fpath:
        fig.savefig(fpath)
    plt.show()

plt.close('all')
plot(1e-15, 'plots/pegd-pe=1e15.png')
plot(1e-6, 'plots/pegd-pe=1e6.png')


"""
An experimental module for reading dvi files output by TeX. Several
limitations make this not (currently) useful as a general-purpose dvi
preprocessor.

Interface::

  dvi = Dvi(filename, 72)
  for page in dvi:          # iterate over pages
      w, h, d = page.width, page.height, page.descent
      for x,y,font,glyph,width in page.text:
          fontname = font.texname
          pointsize = font.size
          ...
      for x,y,height,width in page.boxes:
          ...

"""

import errno
import matplotlib
import matplotlib.cbook as mpl_cbook
import numpy as np
import struct
import subprocess

_dvistate = mpl_cbook.Bunch(pre=0, outer=1, inpage=2, post_post=3, finale=4)

class Dvi(object):
    """
    A dvi ("device-independent") file, as produced by TeX.
    The current implementation only reads the first page and does not
    even attempt to verify the postamble.
    """

    def __init__(self, filename, dpi):
        """
        Initialize the object. This takes the filename as input and
        opens the file; actually reading the file happens when
        iterating through the pages of the file.
        """
        matplotlib.verbose.report('Dvi: ' + filename, 'debug')
        self.file = open(filename, 'rb')
        self.dpi = dpi
        self.fonts = {}
        self.state = _dvistate.pre

    def __iter__(self):
        """
        Iterate through the pages of the file.

        Returns (text, pages) pairs, where:
          text is a list of (x, y, fontnum, glyphnum, width) tuples
          boxes is a list of (x, y, height, width) tuples

        The coordinates are transformed into a standard Cartesian
        coordinate system at the dpi value given when initializing.
        The coordinates are floating point numbers, but otherwise
        precision is not lost and coordinate values are not clipped to
        integers.
        """
        while True:
            have_page = self._read()
            if have_page:
                yield self._output()
            else:
                break

    def close(self):
        """
        Close the underlying file if it is open.
        """
        if not self.file.closed:
            self.file.close()

    def _output(self):
        """
        Output the text and boxes belonging to the most recent page.
        page = dvi._output()
        """
        minx, miny, maxx, maxy = np.inf, np.inf, -np.inf, -np.inf
        maxy_pure = -np.inf
        for elt in self.text + self.boxes:
            if len(elt) == 4:   # box
                x,y,h,w = elt
                e = 0           # zero depth
            else:               # glyph
                x,y,font,g,w = elt
                h = _mul2012(font._scale, font._tfm.height[g])
                e = _mul2012(font._scale, font._tfm.depth[g])
            minx = min(minx, x)
            miny = min(miny, y - h)
            maxx = max(maxx, x + w)
            maxy = max(maxy, y + e)
            maxy_pure = max(maxy_pure, y)

        if self.dpi is None:
            # special case for ease of debugging: output raw dvi coordinates
            return mpl_cbook.Bunch(text=self.text, boxes=self.boxes,
                                   width=maxx-minx, height=maxy_pure-miny,
                                   descent=maxy-maxy_pure)

        d = self.dpi / (72.27 * 2**16) # from TeX's "scaled points" to dpi units
        text =  [ ((x-minx)*d, (maxy-y)*d, f, g, w*d)
                  for (x,y,f,g,w) in self.text ]
        boxes = [ ((x-minx)*d, (maxy-y)*d, h*d, w*d) for (x,y,h,w) in self.boxes ]

        return mpl_cbook.Bunch(text=text, boxes=boxes,
                               width=(maxx-minx)*d,
                               height=(maxy_pure-miny)*d,
                               descent=(maxy-maxy_pure)*d)

    def _read(self):
        """
        Read one page from the file. Return True if successful,
        False if there were no more pages.
        """
        while True:
            byte = ord(self.file.read(1))
            self._dispatch(byte)
#             if self.state == _dvistate.inpage:
#                 matplotlib.verbose.report(
#                     'Dvi._read: after %d at %f,%f' %
#                     (byte, self.h, self.v),
#                     'debug-annoying')
            if byte == 140: # end of page
                return True
            if self.state == _dvistate.post_post: # end of file
                self.close()
                return False

    def _arg(self, nbytes, signed=False):
        """
        Read and return an integer argument "nbytes" long.
        Signedness is determined by the "signed" keyword.
        """
        str = self.file.read(nbytes)
        value = ord(str[0])
        if signed and value >= 0x80:
            value = value - 0x100
        for i in range(1, nbytes):
            value = 0x100*value + ord(str[i])
        return value

    def _dispatch(self, byte):
        """
        Based on the opcode "byte", read the correct kinds of
        arguments from the dvi file and call the method implementing
        that opcode with those arguments.
        """
        if 0 <= byte <= 127: self._set_char(byte)
        elif byte == 128: self._set_char(self._arg(1))
        elif byte == 129: self._set_char(self._arg(2))
        elif byte == 130: self._set_char(self._arg(3))
        elif byte == 131: self._set_char(self._arg(4, True))
        elif byte == 132: self._set_rule(self._arg(4, True), self._arg(4, True))
        elif byte == 133: self._put_char(self._arg(1))
        elif byte == 134: self._put_char(self._arg(2))
        elif byte == 135: self._put_char(self._arg(3))
        elif byte == 136: self._put_char(self._arg(4, True))
        elif byte == 137: self._put_rule(self._arg(4, True), self._arg(4, True))
        elif byte == 138: self._nop()
        elif byte == 139: self._bop(*[self._arg(4, True) for i in range(11)])
        elif byte == 140: self._eop()
        elif byte == 141: self._push()
        elif byte == 142: self._pop()
        elif byte == 143: self._right(self._arg(1, True))
        elif byte == 144: self._right(self._arg(2, True))
        elif byte == 145: self._right(self._arg(3, True))
        elif byte == 146: self._right(self._arg(4, True))
        elif byte == 147: self._right_w(None)
        elif byte == 148: self._right_w(self._arg(1, True))
        elif byte == 149: self._right_w(self._arg(2, True))
        elif byte == 150: self._right_w(self._arg(3, True))
        elif byte == 151: self._right_w(self._arg(4, True))
        elif byte == 152: self._right_x(None)
        elif byte == 153: self._right_x(self._arg(1, True))
        elif byte == 154: self._right_x(self._arg(2, True))
        elif byte == 155: self._right_x(self._arg(3, True))
        elif byte == 156: self._right_x(self._arg(4, True))
        elif byte == 157: self._down(self._arg(1, True))
        elif byte == 158: self._down(self._arg(2, True))
        elif byte == 159: self._down(self._arg(3, True))
        elif byte == 160: self._down(self._arg(4, True))
        elif byte == 161: self._down_y(None)
        elif byte == 162: self._down_y(self._arg(1, True))
        elif byte == 163: self._down_y(self._arg(2, True))
        elif byte == 164: self._down_y(self._arg(3, True))
        elif byte == 165: self._down_y(self._arg(4, True))
        elif byte == 166: self._down_z(None)
        elif byte == 167: self._down_z(self._arg(1, True))
        elif byte == 168: self._down_z(self._arg(2, True))
        elif byte == 169: self._down_z(self._arg(3, True))
        elif byte == 170: self._down_z(self._arg(4, True))
        elif 171 <= byte <= 234: self._fnt_num(byte-171)
        elif byte == 235: self._fnt_num(self._arg(1))
        elif byte == 236: self._fnt_num(self._arg(2))
        elif byte == 237: self._fnt_num(self._arg(3))
        elif byte == 238: self._fnt_num(self._arg(4, True))
        elif 239 <= byte <= 242:
            len = self._arg(byte-238)
            special = self.file.read(len)
            self._xxx(special)
        elif 243 <= byte <= 246:
            k = self._arg(byte-242, byte==246)
            c, s, d, a, l = [ self._arg(x) for x in (4, 4, 4, 1, 1) ]
            n = self.file.read(a+l)
            self._fnt_def(k, c, s, d, a, l, n)
        elif byte == 247:
            i, num, den, mag, k = [ self._arg(x) for x in (1, 4, 4, 4, 1) ]
            x = self.file.read(k)
            self._pre(i, num, den, mag, x)
        elif byte == 248: self._post()
        elif byte == 249: self._post_post()
        else:
            raise ValueError, "unknown command: byte %d"%byte

    def _pre(self, i, num, den, mag, comment):
        if self.state != _dvistate.pre:
            raise ValueError, "pre command in middle of dvi file"
        if i != 2:
            raise ValueError, "Unknown dvi format %d"%i
        if num != 25400000 or den != 7227 * 2**16:
            raise ValueError, "nonstandard units in dvi file"
            # meaning: TeX always uses those exact values, so it
            # should be enough for us to support those
            # (There are 72.27 pt to an inch so 7227 pt =
            # 7227 * 2**16 sp to 100 in. The numerator is multiplied
            # by 10^5 to get units of 10**-7 meters.)
        if mag != 1000:
            raise ValueError, "nonstandard magnification in dvi file"
            # meaning: LaTeX seems to frown on setting \mag, so
            # I think we can assume this is constant
        self.state = _dvistate.outer

    def _set_char(self, char):
        if self.state != _dvistate.inpage:
            raise ValueError, "misplaced set_char in dvi file"
        self._put_char(char)
        self.h += self.fonts[self.f]._width_of(char)

    def _set_rule(self, a, b):
        if self.state != _dvistate.inpage:
            raise ValueError, "misplaced set_rule in dvi file"
        self._put_rule(a, b)
        self.h += b

    def _put_char(self, char):
        if self.state != _dvistate.inpage:
            raise ValueError, "misplaced put_char in dvi file"
        font = self.fonts[self.f]
        if font._vf is None:
            self.text.append((self.h, self.v, font, char,
                              font._width_of(char)))
#             matplotlib.verbose.report(
#                 'Dvi._put_char: %d,%d %d' %(self.h, self.v, char),
#                 'debug-annoying')
        else:
            scale = font._scale
            for x, y, f, g, w in font._vf[char].text:
                newf = DviFont(scale=_mul2012(scale, f._scale),
                               tfm=f._tfm, texname=f.texname, vf=f._vf)
                self.text.append((self.h + _mul2012(x, scale),
                                  self.v + _mul2012(y, scale),
                                  newf, g, newf._width_of(g)))
            self.boxes.extend([(self.h + _mul2012(x, scale),
                                self.v + _mul2012(y, scale),
                                _mul2012(a, scale), _mul2012(b, scale))
                               for x, y, a, b in font._vf[char].boxes])

    def _put_rule(self, a, b):
        if self.state != _dvistate.inpage:
            raise ValueError, "misplaced put_rule in dvi file"
        if a > 0 and b > 0:
            self.boxes.append((self.h, self.v, a, b))
#             matplotlib.verbose.report(
#                 'Dvi._put_rule: %d,%d %d,%d' % (self.h, self.v, a, b),
#                 'debug-annoying')

    def _nop(self):
        pass

    def _bop(self, c0, c1, c2, c3, c4, c5, c6, c7, c8, c9, p):
        if self.state != _dvistate.outer:
            raise ValueError, \
                "misplaced bop in dvi file (state %d)" % self.state
        self.state = _dvistate.inpage
        self.h, self.v, self.w, self.x, self.y, self.z = 0, 0, 0, 0, 0, 0
        self.stack = []
        self.text = []          # list of (x,y,fontnum,glyphnum)
        self.boxes = []         # list of (x,y,width,height)

    def _eop(self):
        if self.state != _dvistate.inpage:
            raise ValueError, "misplaced eop in dvi file"
        self.state = _dvistate.outer
        del self.h, self.v, self.w, self.x, self.y, self.z, self.stack

    def _push(self):
        if self.state != _dvistate.inpage:
            raise ValueError, "misplaced push in dvi file"
        self.stack.append((self.h, self.v, self.w, self.x, self.y, self.z))

    def _pop(self):
        if self.state != _dvistate.inpage:
            raise ValueError, "misplaced pop in dvi file"
        self.h, self.v, self.w, self.x, self.y, self.z = self.stack.pop()

    def _right(self, b):
        if self.state != _dvistate.inpage:
            raise ValueError, "misplaced right in dvi file"
        self.h += b

    def _right_w(self, new_w):
        if self.state != _dvistate.inpage:
            raise ValueError, "misplaced w in dvi file"
        if new_w is not None:
            self.w = new_w
        self.h += self.w

    def _right_x(self, new_x):
        if self.state != _dvistate.inpage:
            raise ValueError, "misplaced x in dvi file"
        if new_x is not None:
            self.x = new_x
        self.h += self.x

    def _down(self, a):
        if self.state != _dvistate.inpage:
            raise ValueError, "misplaced down in dvi file"
        self.v += a

    def _down_y(self, new_y):
        if self.state != _dvistate.inpage:
            raise ValueError, "misplaced y in dvi file"
        if new_y is not None:
            self.y = new_y
        self.v += self.y

    def _down_z(self, new_z):
        if self.state != _dvistate.inpage:
            raise ValueError, "misplaced z in dvi file"
        if new_z is not None:
            self.z = new_z
        self.v += self.z

    def _fnt_num(self, k):
        if self.state != _dvistate.inpage:
            raise ValueError, "misplaced fnt_num in dvi file"
        self.f = k

    def _xxx(self, special):
        matplotlib.verbose.report(
            'Dvi._xxx: encountered special: %s'
            % ''.join([(32 <= ord(ch) < 127) and ch
                       or '<%02x>' % ord(ch)
                       for ch in special]),
            'debug')

    def _fnt_def(self, k, c, s, d, a, l, n):
        tfm = _tfmfile(n[-l:])
        if c != 0 and tfm.checksum != 0 and c != tfm.checksum:
            raise ValueError, 'tfm checksum mismatch: %s'%n
        # It seems that the assumption behind the following check is incorrect:
        #if d != tfm.design_size:
        #    raise ValueError, 'tfm design size mismatch: %d in dvi, %d in %s'%\
        #        (d, tfm.design_size, n)

        vf = _vffile(n[-l:])

        self.fonts[k] = DviFont(scale=s, tfm=tfm, texname=n, vf=vf)

    def _post(self):
        if self.state != _dvistate.outer:
            raise ValueError, "misplaced post in dvi file"
        self.state = _dvistate.post_post
        # TODO: actually read the postamble and finale?
        # currently post_post just triggers closing the file

    def _post_post(self):
        raise NotImplementedError

class DviFont(object):
    """
    Object that holds a font's texname and size, supports comparison,
    and knows the widths of glyphs in the same units as the AFM file.
    There are also internal attributes (for use by dviread.py) that
    are _not_ used for comparison.

    The size is in Adobe points (converted from TeX points).
    """
    __slots__ = ('texname', 'size', 'widths', '_scale', '_vf', '_tfm')

    def __init__(self, scale, tfm, texname, vf):
        self._scale, self._tfm, self.texname, self._vf = \
            scale, tfm, texname, vf
        self.size = scale * (72.0 / (72.27 * 2**16))
        try:
            nchars = max(tfm.width.iterkeys())
        except ValueError:
            nchars = 0
        self.widths = [ (1000*tfm.width.get(char, 0)) >> 20
                        for char in range(nchars) ]

    def __eq__(self, other):
        return self.__class__ == other.__class__ and \
            self.texname == other.texname and self.size == other.size

    def __ne__(self, other):
        return not self.__eq__(other)

    def _width_of(self, char):
        """
        Width of char in dvi units. For internal use by dviread.py.
        """

        width = self._tfm.width.get(char, None)
        if width is not None:
            return _mul2012(width, self._scale)

        matplotlib.verbose.report(
            'No width for char %d in font %s' % (char, self.texname),
            'debug')
        return 0

class Vf(Dvi):
    """
    A virtual font (\*.vf file) containing subroutines for dvi files.

    Usage::

      vf = Vf(filename)
      glyph = vf[code]
      glyph.text, glyph.boxes, glyph.width
    """

    def __init__(self, filename):
        Dvi.__init__(self, filename, 0)
        self._first_font = None
        self._chars = {}
        self._packet_ends = None
        self._read()
        self.close()

    def __getitem__(self, code):
        return self._chars[code]

    def _dispatch(self, byte):
        # If we are in a packet, execute the dvi instructions
        if self.state == _dvistate.inpage:
            byte_at = self.file.tell()-1
            if byte_at == self._packet_ends:
                self._finalize_packet()
                # fall through
            elif byte_at > self._packet_ends:
                raise ValueError, "Packet length mismatch in vf file"
            else:
                if byte in (139, 140) or byte >= 243:
                    raise ValueError, "Inappropriate opcode %d in vf file" % byte
                Dvi._dispatch(self, byte)
                return

        # We are outside a packet
        if byte < 242:          # a short packet (length given by byte)
            cc, tfm = self._arg(1), self._arg(3)
            self._init_packet(byte, cc, tfm)
        elif byte == 242:       # a long packet
            pl, cc, tfm = [ self._arg(x) for x in (4, 4, 4) ]
            self._init_packet(pl, cc, tfm)
        elif 243 <= byte <= 246:
            Dvi._dispatch(self, byte)
        elif byte == 247:       # preamble
            i, k = self._arg(1), self._arg(1)
            x = self.file.read(k)
            cs, ds = self._arg(4), self._arg(4)
            self._pre(i, x, cs, ds)
        elif byte == 248:       # postamble (just some number of 248s)
            self.state = _dvistate.post_post
        else:
            raise ValueError, "unknown vf opcode %d" % byte

    def _init_packet(self, pl, cc, tfm):
        if self.state != _dvistate.outer:
            raise ValueError, "Misplaced packet in vf file"
        self.state = _dvistate.inpage
        self._packet_ends = self.file.tell() + pl
        self._packet_char = cc
        self._packet_width = tfm
        self.h, self.v, self.w, self.x, self.y, self.z = 0, 0, 0, 0, 0, 0
        self.stack, self.text, self.boxes = [], [], []
        self.f = self._first_font

    def _finalize_packet(self):
        self._chars[self._packet_char] = mpl_cbook.Bunch(
            text=self.text, boxes=self.boxes, width = self._packet_width)
        self.state = _dvistate.outer

    def _pre(self, i, x, cs, ds):
        if self.state != _dvistate.pre:
            raise ValueError, "pre command in middle of vf file"
        if i != 202:
            raise ValueError, "Unknown vf format %d" % i
        if len(x):
            matplotlib.verbose.report('vf file comment: ' + x, 'debug')
        self.state = _dvistate.outer
        # cs = checksum, ds = design size

    def _fnt_def(self, k, *args):
        Dvi._fnt_def(self, k, *args)
        if self._first_font is None:
            self._first_font = k

def _fix2comp(num):
    """
    Convert from two's complement to negative.
    """
    assert 0 <= num < 2**32
    if num & 2**31:
        return num - 2**32
    else:
        return num

def _mul2012(num1, num2):
    """
    Multiply two numbers in 20.12 fixed point format.
    """
    # Separated into a function because >> has surprising precedence
    return (num1*num2) >> 20

class Tfm(object):
    """
    A TeX Font Metric file. This implementation covers only the bare
    minimum needed by the Dvi class.

    Attributes:

      checksum: for verifying against dvi file

      design_size: design size of the font (in what units?)

      width[i]: width of character \#i, needs to be scaled
        by the factor specified in the dvi file
        (this is a dict because indexing may not start from 0)

      height[i], depth[i]: height and depth of character \#i
    """
    __slots__ = ('checksum', 'design_size', 'width', 'height', 'depth')

    def __init__(self, filename):
        matplotlib.verbose.report('opening tfm file ' + filename, 'debug')
        file = open(filename, 'rb')

        try:
            header1 = file.read(24)
            lh, bc, ec, nw, nh, nd = \
                struct.unpack('!6H', header1[2:14])
            matplotlib.verbose.report(
                'lh=%d, bc=%d, ec=%d, nw=%d, nh=%d, nd=%d' % (
                    lh, bc, ec, nw, nh, nd), 'debug')
            header2 = file.read(4*lh)
            self.checksum, self.design_size = \
                struct.unpack('!2I', header2[:8])
            # there is also encoding information etc.
            char_info = file.read(4*(ec-bc+1))
            widths = file.read(4*nw)
            heights = file.read(4*nh)
            depths = file.read(4*nd)
        finally:
            file.close()

        self.width, self.height, self.depth = {}, {}, {}
        widths, heights, depths = \
            [ struct.unpack('!%dI' % (len(x)/4), x)
              for x in (widths, heights, depths) ]
        for i in range(ec-bc):
            self.width[bc+i] = _fix2comp(widths[ord(char_info[4*i])])
            self.height[bc+i] = _fix2comp(heights[ord(char_info[4*i+1]) >> 4])
            self.depth[bc+i] = _fix2comp(depths[ord(char_info[4*i+1]) & 0xf])

class PsfontsMap(object):
    """
    A psfonts.map formatted file, mapping TeX fonts to PS fonts.
    Usage: map = PsfontsMap('.../psfonts.map'); map['cmr10']

    For historical reasons, TeX knows many Type-1 fonts by different
    names than the outside world. (For one thing, the names have to
    fit in eight characters.) Also, TeX's native fonts are not Type-1
    but Metafont, which is nontrivial to convert to PostScript except
    as a bitmap. While high-quality conversions to Type-1 format exist
    and are shipped with modern TeX distributions, we need to know
    which Type-1 fonts are the counterparts of which native fonts. For
    these reasons a mapping is needed from internal font names to font
    file names.

    A texmf tree typically includes mapping files called e.g.
    psfonts.map, pdftex.map,  dvipdfm.map. psfonts.map is used by
    dvips, pdftex.map by pdfTeX, and dvipdfm.map by dvipdfm.
    psfonts.map might avoid embedding the 35 PostScript fonts, while
    the pdf-related files perhaps only avoid the "Base 14" pdf fonts.
    But the user may have configured these files differently.
    """
    __slots__ = ('_font',)

    def __init__(self, filename):
        self._font = {}
        file = open(filename, 'rt')
        try:
            self._parse(file)
        finally:
            file.close()

    def __getitem__(self, texname):
        result = self._font[texname]
        fn, enc = result.filename, result.encoding
        if fn is not None and not fn.startswith('/'):
            result.filename = find_tex_file(fn)
        if enc is not None and not enc.startswith('/'):
            result.encoding = find_tex_file(result.encoding)
        return result

    def _parse(self, file):
        """Parse each line into words."""
        for line in file:
            line = line.strip()
            if line == '' or line.startswith('%'):
                continue
            words, pos = [], 0
            while pos < len(line):
                if line[pos] == '"': # double quoted word
                    pos += 1
                    end = line.index('"', pos)
                    words.append(line[pos:end])
                    pos = end + 1
                else:                # ordinary word
                    end = line.find(' ', pos+1)
                    if end == -1: end = len(line)
                    words.append(line[pos:end])
                    pos = end
                while pos < len(line) and line[pos] == ' ':
                    pos += 1
            self._register(words)

    def _register(self, words):
        """Register a font described by "words".

        The format is, AFAIK: texname fontname [effects and filenames]
        Effects are PostScript snippets like ".177 SlantFont",
        filenames begin with one or two less-than signs. A filename
        ending in enc is an encoding file, other filenames are font
        files. This can be overridden with a left bracket: <[foobar
        indicates an encoding file named foobar.

        There is some difference between <foo.pfb and <<bar.pfb in
        subsetting, but I have no example of << in my TeX installation.
        """
        texname, psname = words[:2]
        effects, encoding, filename = [], None, None
        for word in words[2:]:
            if not word.startswith('<'):
                effects.append(word)
            else:
                word = word.lstrip('<')
                if word.startswith('['):
                    assert encoding is None
                    encoding = word[1:]
                elif word.endswith('.enc'):
                    assert encoding is None
                    encoding = word
                else:
                    assert filename is None
                    filename = word
        self._font[texname] = mpl_cbook.Bunch(
            texname=texname, psname=psname, effects=effects,
            encoding=encoding, filename=filename)

class Encoding(object):
    """
    Parses a \*.enc file referenced from a psfonts.map style file.
    The format this class understands is a very limited subset of
    PostScript.

    Usage (subject to change)::

      for name in Encoding(filename):
          whatever(name)
    """
    __slots__ = ('encoding',)

    def __init__(self, filename):
        file = open(filename, 'rt')
        try:
            matplotlib.verbose.report('Parsing TeX encoding ' + filename, 'debug-annoying')
            self.encoding = self._parse(file)
            matplotlib.verbose.report('Result: ' + `self.encoding`, 'debug-annoying')
        finally:
            file.close()

    def __iter__(self):
        for name in self.encoding:
            yield name

    def _parse(self, file):
        result = []

        state = 0
        for line in file:
            comment_start = line.find('%')
            if comment_start > -1:
                line = line[:comment_start]
            line = line.strip()

            if state == 0:
                # Expecting something like /FooEncoding [
                if '[' in line:
                    state = 1
                    line = line[line.index('[')+1:].strip()

            if state == 1:
                if ']' in line: # ] def
                    line = line[:line.index(']')]
                    state = 2
                words = line.split()
                for w in words:
                    if w.startswith('/'):
                        # Allow for /abc/def/ghi
                        subwords = w.split('/')
                        result.extend(subwords[1:])
                    else:
                        raise ValueError, "Broken name in encoding file: " + w

        return result

def find_tex_file(filename, format=None):
    """
    Call kpsewhich to find a file in the texmf tree.
    If format is not None, it is used as the value for the --format option.
    See the kpathsea documentation for more information.

    Apparently most existing TeX distributions on Unix-like systems
    use kpathsea. I hear MikTeX (a popular distribution on Windows)
    doesn't use kpathsea, so what do we do? (TODO)
    """

    cmd = ['kpsewhich']
    if format is not None:
        cmd += ['--format=' + format]
    cmd += [filename]

    matplotlib.verbose.report('find_tex_file(%s): %s' \
                                  % (filename,cmd), 'debug')
    pipe = subprocess.Popen(cmd, stdout=subprocess.PIPE)
    result = pipe.communicate()[0].rstrip()
    matplotlib.verbose.report('find_tex_file result: %s' % result,
                              'debug')
    return result

def _read_nointr(pipe, bufsize=-1):
    while True:
        try:
            return pipe.read(bufsize)
        except OSError, e:
            if e.errno == errno.EINTR:
                continue
            else:
                raise

# With multiple text objects per figure (e.g. tick labels) we may end
# up reading the same tfm and vf files many times, so we implement a
# simple cache. TODO: is this worth making persistent?

_tfmcache = {}
_vfcache = {}

def _fontfile(texname, class_, suffix, cache):
    try:
        return cache[texname]
    except KeyError:
        pass

    filename = find_tex_file(texname + suffix)
    if filename:
        result = class_(filename)
    else:
        result = None

    cache[texname] = result
    return result

def _tfmfile(texname):
    return _fontfile(texname, Tfm, '.tfm', _tfmcache)

def _vffile(texname):
    return _fontfile(texname, Vf, '.vf', _vfcache)

if __name__ == '__main__':
    import sys
    matplotlib.verbose.set_level('debug-annoying')
    fname = sys.argv[1]
    try: dpi = float(sys.argv[2])
    except IndexError: dpi = None
    dvi = Dvi(fname, dpi)
    fontmap = PsfontsMap(find_tex_file('pdftex.map'))
    for page in dvi:
        print '=== new page ==='
        fPrev = None
        for x,y,f,c,w in page.text:
            if f != fPrev:
                print 'font', f.texname, 'scaled', f._scale/pow(2.0,20)
                fPrev = f
            print x,y,c, 32 <= c < 128 and chr(c) or '.', w
        for x,y,w,h in page.boxes:
            print x,y,'BOX',w,h


"""
=========================
Bayesian Ridge Regression
=========================

Computes a Bayesian Ridge Regression on a synthetic dataset.

See :ref:`bayesian_ridge_regression` for more information on the regressor.

Compared to the OLS (ordinary least squares) estimator, the coefficient
weights are slightly shifted toward zeros, which stabilises them.

As the prior on the weights is a Gaussian prior, the histogram of the
estimated weights is Gaussian.

The estimation of the model is done by iteratively maximizing the
marginal log-likelihood of the observations.
"""
print(__doc__)

import numpy as np
import matplotlib.pyplot as plt
from scipy import stats

from sklearn.linear_model import BayesianRidge, LinearRegression

###############################################################################
# Generating simulated data with Gaussian weigthts
np.random.seed(0)
n_samples, n_features = 100, 100
X = np.random.randn(n_samples, n_features)  # Create Gaussian data
# Create weigts with a precision lambda_ of 4.
lambda_ = 4.
w = np.zeros(n_features)
# Only keep 10 weights of interest
relevant_features = np.random.randint(0, n_features, 10)
for i in relevant_features:
    w[i] = stats.norm.rvs(loc=0, scale=1. / np.sqrt(lambda_))
# Create noise with a precision alpha of 50.
alpha_ = 50.
noise = stats.norm.rvs(loc=0, scale=1. / np.sqrt(alpha_), size=n_samples)
# Create the target
y = np.dot(X, w) + noise

###############################################################################
# Fit the Bayesian Ridge Regression and an OLS for comparison
clf = BayesianRidge(compute_score=True)
clf.fit(X, y)

ols = LinearRegression()
ols.fit(X, y)

###############################################################################
# Plot true weights, estimated weights and histogram of the weights
plt.figure(figsize=(6, 5))
plt.title("Weights of the model")
plt.plot(clf.coef_, 'b-', label="Bayesian Ridge estimate")
plt.plot(w, 'g-', label="Ground truth")
plt.plot(ols.coef_, 'r--', label="OLS estimate")
plt.xlabel("Features")
plt.ylabel("Values of the weights")
plt.legend(loc="best", prop=dict(size=12))

plt.figure(figsize=(6, 5))
plt.title("Histogram of the weights")
plt.hist(clf.coef_, bins=n_features, log=True)
plt.plot(clf.coef_[relevant_features], 5 * np.ones(len(relevant_features)),
         'ro', label="Relevant features")
plt.ylabel("Features")
plt.xlabel("Values of the weights")
plt.legend(loc="lower left")

plt.figure(figsize=(6, 5))
plt.title("Marginal log-likelihood")
plt.plot(clf.scores_)
plt.ylabel("Score")
plt.xlabel("Iterations")
plt.show()


import numpy as np
import matplotlib.pyplot as plt
from scipy.signal import hamming, hanning, triang, blackmanharris, resample
import math
import sys, os, functools, time

sys.path.append(os.path.join(os.path.dirname(os.path.realpath(__file__)), '../../../software/models/'))
sys.path.append(os.path.join(os.path.dirname(os.path.realpath(__file__)), '../../../software/transformations/'))

import sineModel as SM
import stft as STFT
import utilFunctions as UF
import sineTransformations as SMT

(fs, x) = UF.wavread('../../../sounds/orchestra.wav')
w = np.hamming(801)
N = 2048
t = -90
minSineDur = .005
maxnSines = 150
freqDevOffset = 20
freqDevSlope = 0.02
Ns = 512
H = Ns/4
mX, pX = STFT.stftAnal(x, fs, w, N, H)
tfreq, tmag, tphase = SM.sineModelAnal(x, fs, w, N, H, t, maxnSines, minSineDur, freqDevOffset, freqDevSlope)
freqScaling = np.array([0, .8, 1, 1.2])
ytfreq = SMT.sineFreqScaling(tfreq, freqScaling)
y = SM.sineModelSynth(ytfreq, tmag, np.array([]), Ns, H, fs)
mY, pY = STFT.stftAnal(y, fs, w, N, H)
UF.wavwrite(y,fs, 'sineModelFreqScale-orchestra.wav')

maxplotfreq = 4000.0

plt.figure(1, figsize=(9.5, 7))
plt.subplot(4,1,1)
plt.plot(np.arange(x.size)/float(fs), x, 'b')
plt.axis([0,x.size/float(fs),min(x),max(x)])
plt.title('x (orchestra.wav)')

plt.subplot(4,1,2)
numFrames = int(tfreq[:,0].size)
frmTime = H*np.arange(numFrames)/float(fs)
tracks = tfreq*np.less(tfreq, maxplotfreq)
tracks[tracks<=0] = np.nan
plt.plot(frmTime, tracks, color='k', lw=1)
plt.autoscale(tight=True)
plt.title('sine frequencies')

maxplotbin = int(N*maxplotfreq/fs)
numFrames = int(mX[:,0].size)
frmTime = H*np.arange(numFrames)/float(fs)
binFreq = np.arange(maxplotbin+1)*float(fs)/N
plt.pcolormesh(frmTime, binFreq, np.transpose(mX[:,:maxplotbin+1]))
plt.autoscale(tight=True)

plt.subplot(4,1,3)
numFrames = int(ytfreq[:,0].size)
frmTime = H*np.arange(numFrames)/float(fs)
tracks = ytfreq*np.less(ytfreq, maxplotfreq)
tracks[tracks<=0] = np.nan
plt.plot(frmTime, tracks, color='k', lw=1)
plt.autoscale(tight=True)
plt.title('freq-scaled sine frequencies')

maxplotbin = int(N*maxplotfreq/fs)
numFrames = int(mY[:,0].size)
frmTime = H*np.arange(numFrames)/float(fs)
binFreq = np.arange(maxplotbin+1)*float(fs)/N
plt.pcolormesh(frmTime, binFreq, np.transpose(mY[:,:maxplotbin+1]))
plt.autoscale(tight=True)

plt.subplot(4,1,4)
plt.plot(np.arange(y.size)/float(fs), y, 'b')
plt.axis([0,y.size/float(fs),min(y),max(y)])
plt.title('y')

plt.tight_layout()
plt.savefig('sineModelFreqScale-orchestra.png')
plt.show()