#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
===================================================================
BugController.py - Bug Algorithm Motion Controller
===================================================================

This motion controller uses the Bug 2 algorithm developed by V. Lumelsky and A. Stepanov. The algorithm assumes the robot is a
point operating in the plane with a contact sensor or a zero range sensor to
detect obstacles. When the robot has a ﬁnite range (non-zero range) sensor
"""

#import  BugControllerHelper
from numpy import *
from __is_inside import is_inside
import Polygon,Polygon.IO
import Polygon.Utils as PolyUtils
import Polygon.Shapes as PolyShapes
import time, math
import sys,os
import matplotlib.pyplot as plt
import copy
from scipy.linalg import norm
from math import *
import random
import matplotlib.animation as animation
import thread
import threading

import lib.handlers.handlerTemplates as handlerTemplates

class BugControllerHandler(handlerTemplates.MotionControlHandler):
    def __init__(self, executor, shared_data,robot_type):
        """
        Bug alogorithm motion planning controller

        robot_type (int): Which robot is used for execution. pioneer is 1, ODE is 2 (default=1)
        """
        self.velocity_count_thres = 100;
        self.velocity_count = 0;

        #operate_system (int): Which operating system is used for execution. Ubuntu and Mac is 1, Windows is 2
        if sys.platform in ['win32', 'cygwin']:
            self.operate_system = 2
        else:
            self.operate_system = 1

        # Information about the robot
        ## 1: Pioneer ; 2: 0DE
        if robot_type not in [1,2]:
            robot_type = 1
        self.system = robot_type

        #settings for Ubuntu or Mac plotting (only when you set operate_system = 1)
        self.PLOT              = False    # plot with matplot
        self.PLOT_M_LINE       = False    # plot m-line
        self.PLOT_EXIT         = False    # plot exit point of a region
        self.PLOT_OVERLAP      = False    # plot overlap area with the obstacle

        #settings for Windows (only when you set operate_system = 2)
        self.PLOT_WINDOWS     = True

        # Get references to handlers we'll need to communicate with
        self.drive_handler = executor.hsub.getHandlerInstanceByType(handlerTemplates.DriveHandler)
        self.pose_handler = executor.hsub.getHandlerInstanceByType(handlerTemplates.PoseHandler)
        if self.system == 1:
            self.robocomm = shared_data['robocomm']

        # Get information about regions
        self.proj = executor.proj
        self.coordmap_map2lab = self.hsub.coordmap_map2lab
        self.coordmap_lab2map = self.hsub.coordmap_lab2map
        self.last_warning = 0

        ###################################
        ########used by Bug algorithm######
        ###################################

        # PARAMETERS

        #for plotting
        self.time = time.clock()
        self.time_thres = 1;

        # Pioneer related parameters
        self.PioneerWidthHalf  = 0.20     #0.25 # (m) width of Pioneer    #0.20
        self.PioneerLengthHalf = 0.25     #0.30 (m) lenght of Pioneer   #0.25

        # Real Robot polygon related parameters
        self.boxRealVertical         = self.PioneerLengthHalf*2
        self.boxRealHorizontal       = self.PioneerWidthHalf*2.5
        self.boxRealVertical_shift   = self.boxRealVertical/2
        self.boxRealHorizontal_shift = self.boxRealHorizontal/2

        # Pioneer Range related parameters
        self.range             = 2*self.PioneerLengthHalf+0.40     # (m) specify the range of the robot (when the normal circle range cannot detect obstacle)   #0.85
        self.obsRange          = self.range*0.7                                 # (m) range that says the robot detects obstacles    #0.25
        self.shift             = 0.20                                           # How far the range is shifted to ensure it is sensing region in front is bigger    0.20
        self.boxVertical       = self.obsRange*2                                # box cutting from range of Pioneer
        self.boxHorizontal     = self.obsRange*2                                 # box cutting from range of Pioneer
        self.boxVertical_shift = self.boxVertical + self.boxRealVertical/2*1.5    # vertical shifting of box
        self.boxHorizontal_shift = self.boxHorizontal/2                          # horizontal shifting of the box

        ## 2: 0DE
        self.factorODE = 30    # 30 works better than 50
        if  self.system == 2:
            self.PioneerWidthHalf       = self.PioneerWidthHalf*self.factorODE
            self.PioneerLengthHalf      = self.PioneerLengthHalf*self.factorODE
            self.range                  = self.range*self.factorODE
            self.obsRange               = self.obsRange*self.factorODE
            self.shift                  = self.shift*self.factorODE
            self.boxVertical            = self.boxVertical*self.factorODE
            self.boxHorizontal          = self.boxHorizontal*self.factorODE
            self.boxVertical_shift      = self.boxVertical_shift*self.factorODE
            self.boxHorizontal_shift    = self.boxHorizontal_shift*self.factorODE
            self.boxRealVertical        = self.boxRealVertical*self.factorODE
            self.boxRealHorizontal      = self.boxRealHorizontal*self.factorODE
            self.boxRealVertical_shift  = self.boxRealVertical_shift*self.factorODE
            self.boxRealHorizontal_shift= self.boxRealHorizontal_shift*self.factorODE

        self.map = {}                             # dictionary for all the regions
        self.all = Polygon.Polygon()              # Polygon with all the regions
        self.map_work = Polygon.Polygon()         # Polygon of the current region and next region considered
        self.ogr = Polygon.Polygon()              #Polygon built from occupancy grid data points

        self.previous_current_reg = None    # previous current region
        self.currentRegionPoly  = None      # current region's polygon
        self.nextRegionPoly    = None       # next region's polygon
        self.overlap           = None
        self.q_g               = [0,0]      # goal point of the robot heading to
        self.q_hit             = [0,0]      # location where the robot first detect an obstacle
        self.boundary_following= False      # tracking whether it is in boundary following mode
        self.m_line            = None       # m-line polygon
        self.trans_matrix      = mat([[0,1],[-1,0]])   # transformation matrix for find the normal to the vector connecting a point on the obstacle to the robot
        self.q_hit_count       = 0
        self.q_hit_Thres       = 1000
        self.prev_follow       = [[],[]]

        ## Construct robot polygon (for checking overlap)
        pose = self.pose_handler.getPose()
        self.prev_pose = pose
        self.robot = PolyShapes.Rectangle(self.boxHorizontal,self.boxVertical)
        self.robot.shift(pose[0]-self.boxHorizontal_shift,pose[1]-self.boxVertical_shift)
        self.robot = PolyShapes.Circle(self.obsRange,(pose[0],pose[1])) - self.robot
        self.robot.rotate(pose[2]-pi/2,pose[0],pose[1])
        self.robot.shift(self.shift*cos(pose[2]),self.shift*sin(pose[2]))

        #construct real robot polygon( see if there is overlaping with path to goal
        self.realRobot = PolyShapes.Rectangle(self.boxRealHorizontal,self.boxRealVertical )
        self.realRobot.shift(pose[0]-self.boxRealHorizontal_shift,pose[1]-self.boxRealVertical_shift)
        self.realRobot.rotate(pose[2]-pi/2,pose[0],pose[1])

        #constructing polygon of different regions (holes being taken care)

        for region in self.proj.rfi.regions:
            self.map[region.name] = self.createRegionPolygon(region)
            for n in range(len(region.holeList)): # no of holes
                self.map[region.name] -= self.createRegionPolygon(region,n)

        #construct a polygon that included all the regions
        for regionName,regionPoly in self.map.iteritems():
            self.all += regionPoly

        #setting for plotting
        if self.operate_system == 1:
            if self.PLOT or self.PLOT_OVERLAP == True:
                self.original_figure = 1
                plt.figure(self.original_figure)

            if self.PLOT_EXIT or self.PLOT_M_LINE == True:
                self.overlap_figure  = 2
                plt.figure(self.overlap_figure)

        else:
            if self.PLOT_WINDOWS == True:
                # start using anmination to plot Pioneer
                self.fig = plt.figure()
                self.ax = self.fig.add_subplot(111)
                self.scope = _Scope(self.ax,self)
                thread.start_new_thread(self.jplot,())

        #Plot the robot on the map in figure 1
        if self.PLOT == True:
            plt.figure(self.original_figure)
            plt.clf()
            self.plotPoly(self.realRobot, 'r')
            self.plotPoly(self.robot, 'b')
            plt.plot(pose[0],pose[1],'bo')
            self.plotPioneer(self.original_figure)

    def gotoRegion(self, current_reg, next_reg, last=False):

        """
        If ``last`` is True, we will move to the center of the destination region.
        Returns ``True`` if we've reached the destination region.
        """
        q_gBundle              = [[],[]]          # goal bundle
        q_overlap              = [[],[]]          # overlapping points with robot range
        pose = self.pose_handler.getPose()        # Find our current configuration

        #Plot the robot on the map in figure 1
        if self.PLOT == True:
            plt.figure(self.original_figure)
            plt.clf()
            self.plotPoly(self.realRobot, 'r')
            self.plotPoly(self.robot, 'b')
            plt.plot(pose[0],pose[1],'bo')
            self.plotPioneer(self.original_figure)

        if current_reg == next_reg and not last:
            # No need to move!
            self.drive_handler.setVelocity(0, 0)  # So let's stop
            return False

        # Check if Vicon has cut out
        if math.isnan(pose[2]):
            print "no vicon pose"
            print "WARNING: No Vicon data! Pausing."
            #self.drive_handler.setVelocity(0, 0)  # So let's stop
            time.sleep(1)
            return False

        ###This part is run when the robot goes to a new region, otherwise, the original map will be used.
        if not self.previous_current_reg == current_reg:
            #print 'getting into bug alogorithm'

            #clean up the previous self.map_work
            self.map_work =  Polygon.Polygon()

            # NOTE: Information about region geometry can be found in self.proj.rfi.regions
            # create polygon list for regions other than the current_reg and the next_reg
            self.map_work += self.map[self.proj.rfi.regions[current_reg].name]
            self.map_work += self.map[self.proj.rfi.regions[next_reg].name]

            # building current polygon and destination polygon
            self.nextRegionPoly    = self.map[self.proj.rfi.regions[next_reg].name]
            self.currentRegionPoly = self.map[self.proj.rfi.regions[current_reg].name]

            #set to zero velocity before finding the tranFace
            self.drive_handler.setVelocity(0, 0)
            if last:
                transFace = None
            else:
                print "Current reg is " + str(self.proj.rfi.regions[current_reg].name.lower())
                print "Next reg is "+ str(self.proj.rfi.regions[next_reg].name.lower())

                for i in range(len(self.proj.rfi.transitions[current_reg][next_reg])):
                    pointArray_transface = [x for x in self.proj.rfi.transitions[current_reg][next_reg][i]]
                    transFace = asarray(map(self.coordmap_map2lab,pointArray_transface))
                    bundle_x = (transFace[0,0] +transFace[1,0])/2    #mid-point coordinate x
                    bundle_y = (transFace[0,1] +transFace[1,1])/2    #mid-point coordinate y
                    q_gBundle = hstack((q_gBundle,vstack((bundle_x,bundle_y))))
                q_gBundle = q_gBundle.transpose()

                # Find the closest face to the current position
                max_magsq = 1000000
                for tf in q_gBundle:
                    magsq = (tf[0] - pose[0])**2 + (tf[1] - pose[1])**2
                    if magsq < max_magsq:
                        connection = 0
                        tf = tf+(tf-asarray(self.currentRegionPoly.center()))/norm(tf-asarray(self.currentRegionPoly.center()))*2.1*self.PioneerLengthHalf
                        if not self.nextRegionPoly.covers(PolyShapes.Circle(self.PioneerLengthHalf*2,(tf[0],tf[1]))):
                            tf = tf-(tf-asarray(self.currentRegionPoly.center()))/norm(tf-asarray(self.currentRegionPoly.center()))*4.2*self.PioneerLengthHalf
                            if self.nextRegionPoly.covers(PolyShapes.Circle(self.PioneerLengthHalf*2,(tf[0],tf[1]))):
                                connection = 1
                        else:
                            connection = 1
                        if connection == 1:
                            pt1  = tf
                            max_magsq = magsq
                            transFace = 1
                            self.q_g[0] = pt1[0]
                            self.q_g[1] = pt1[1]
                        else:
                            sample = False
                            while not sample:
                                self.q_g[0],self.q_g[1] = self.nextRegionPoly.sample(random.random)
                                robo = PolyShapes.Circle(self.PioneerLengthHalf,(self.q_g[0],self.q_g[1]))
                                if not bool(robo - self.nextRegionPoly):
                                    sample = True

                """
                # Push the goal point to somewhere inside the next region to ensure the robot will get there.(CHECK!!)
                self.q_g = self.q_g+(self.q_g-asarray(self.currentRegionPoly.center()))/norm(self.q_g-asarray(self.currentRegionPoly.center()))*3*self.PioneerLengthHalf
                if not self.nextRegionPoly.isInside(self.q_g[0],self.q_g[1]):
                    self.q_g = self.q_g-(self.q_g-asarray(self.currentRegionPoly.center()))/norm(self.q_g-asarray(self.currentRegionPoly.center()))*6*self.PioneerLengthHalf
                """

                #plot exiting point
                if self.PLOT_EXIT == True:
                    plt.figure(self.overlap_figure)
                    plt.clf()
                    plt.plot(q_gBundle[:,0],q_gBundle[:,1],'ko' )
                    plt.plot(self.q_g[0],self.q_g[1],'ro')
                    plt.plot(pose[0],pose[1],'bo')
                    self.plotPioneer(self.overlap_figure,0)

                if transFace is None:
                    print "ERROR: Unable to find transition face between regions %s and %s.  Please check the decomposition (try viewing projectname_decomposed.regions in RegionEditor or a text editor)." % (self.proj.rfi.regions[current_reg].name, self.proj.rfi.regions[next_reg].name)

        ##################################################
        #######check whether obstacle is detected#########
        ##################################################

        #Update pose,update self.robot, self.realRobot orientation
        self.robot.shift(pose[0]-self.prev_pose[0],pose[1]-self.prev_pose[1])
        self.realRobot.shift(pose[0]-self.prev_pose[0],pose[1]-self.prev_pose[1])
        self.robot.rotate(pose[2]-self.prev_pose[2],pose[0],pose[1])
        self.realRobot.rotate(pose[2]-self.prev_pose[2],pose[0],pose[1])
        self.prev_pose = pose

        ############################
        ########### STEP 1##########
        ############################
        ##Check whether obsRange overlaps with obstacle or the boundary (overlap returns the part of robot not covered by the region

        # for real Pioneer robot
        if self.system == 1:
            # motion controller is not in boundary following mode
            if self.boundary_following == False:

                if self.robocomm.getReceiveObs() == False:
                    overlap = self.robot - ( self.map_work)
                else:
                    overlap = self.robot  - ( self.map_work - self.robocomm.getObsPoly())

            else: #TRUE
                # use a robot with full range all around it
                Robot = PolyShapes.Circle(self.obsRange,(pose[0],pose[1]))
                Robot.shift(self.shift*cos(pose[2]),self.shift*sin(pose[2]))

                if self.robocomm.getReceiveObs() == False:
                    overlap = Robot - ( self.map_work)
                else:
                    overlap = Robot  - ( self.map_work - self.robocomm.getObsPoly())
        # for ODE
        else:
            if self.boundary_following == False:
                overlap = self.robot  - (self.map_work)
            else:#TRUE
                overlap = self.robot  - (self.map_work)

        if self.boundary_following == False:
            if bool(overlap):    ## overlap of obstacles
                #print "There MAYBE overlap~~ check connection to goal"

                # check whether the real robot or and path to goal overlap with the obstacle
                QGoalPoly= PolyShapes.Circle(self.PioneerLengthHalf,(self.q_g[0],self.q_g[1]))
                path  = PolyUtils.convexHull(self.realRobot + QGoalPoly)
                if self.system == 1:
                    if self.robocomm.getReceiveObs() == False:
                        pathOverlap = path - ( self.map_work)
                    else:
                        pathOverlap = path  - ( self.map_work - self.robocomm.getObsPoly())
                else:
                    pathOverlap = path - ( self.map_work)

                if bool(pathOverlap):   # there is overlapping, go into bounding following mode
                    #print "There IS overlap"
                    self.q_hit  = mat([pose[0],pose[1]]).T
                    self.boundary_following = True

                    #Generate m-line polygon
                    QHitPoly = PolyShapes.Circle(self.PioneerLengthHalf/4,(pose[0],pose[1]))
                    QGoalPoly= PolyShapes.Circle(self.PioneerLengthHalf/4,(self.q_g[0],self.q_g[1]))
                    self.m_line  = PolyUtils.convexHull(QHitPoly + QGoalPoly)

                    #plot the first overlap
                    if self.PLOT_M_LINE == True:
                        plt.figure(self.overlap_figure)
                        plt.clf()
                        self.plotPoly(QHitPoly,'k')
                        self.plotPoly(QGoalPoly,'k')
                        self.plotPoly(overlap,'g')
                        self.plotPoly(self.m_line,'b')
                        plt.plot(pose[0],pose[1],'bo')
                        self.plotPioneer(self.overlap_figure,0)

                else:                ##head towards the q_goal
                    if self.system == 1:
                        if self.robocomm.getReceiveObs() == False:   # wait for obstacles from Pioneer
                            vx = 0
                            vy = 0
                        else:
                            dis_cur  = vstack((self.q_g[0],self.q_g[1]))- mat([pose[0],pose[1]]).T
                            vx = (dis_cur/norm(dis_cur)/3)[0,0]
                            vy = (dis_cur/norm(dis_cur)/3)[1,0]

                    else:
                        dis_cur  = vstack((self.q_g[0],self.q_g[1]))- mat([pose[0],pose[1]]).T
                        vx = (dis_cur/norm(dis_cur)/3)[0,0]
                        vy = (dis_cur/norm(dis_cur)/3)[1,0]
                        #print "no obstacles 2-ODE true"

            else:                ##head towards the q_goal
                    if self.system == 1:
                        if self.robocomm.getReceiveObs() == False:   # wait for obstacles from Pioneer
                            vx = 0
                            vy = 0
                        else:
                            dis_cur  = vstack((self.q_g[0],self.q_g[1]))- mat([pose[0],pose[1]]).T
                            vx = (dis_cur/norm(dis_cur)/3)[0,0]
                            vy = (dis_cur/norm(dis_cur)/3)[1,0]

                    else:
                        dis_cur  = vstack((self.q_g[0],self.q_g[1]))- mat([pose[0],pose[1]]).T
                        vx = (dis_cur/norm(dis_cur)/3)[0,0]
                        vy = (dis_cur/norm(dis_cur)/3)[1,0]
                        #print "no obstacles 1-ODE true"

        if self.boundary_following == True:
            self.q_hit_count       += 1
            # finding the point to go normal to (closest overlapping point)

            j = 0
            recheck = 0
            while not bool(overlap):
                # cannot see the obstacle. Based on the location of the previous point blow up the range of the robot on the left or on the right
                j += 1

                # finding whether the previous obstacle point is on the left side or the right side of the robot
                # angle = angle of the previous point from the x-axis of the field
                # omega = angle of the current Pioneer orientation from the x-axis of the field
                # cc    = differnece between angle and omega ( < pi = previous point on the left of robot, else on the right of robot)
                x = self.prev_follow[0] -pose[0]
                y = self.prev_follow[1] -pose[1]
                angle = atan(y/x)

                # convert angle to 2pi
                if x > 0 and y > 0:
                    angle = angle
                elif x < 0 and y > 0:
                    angle = pi + angle
                elif x <0 and y < 0:
                    angle = pi + angle
                else:
                    angle = 2*pi + angle

                # convert pose to 2pi
                if pose[2] < 0:
                    omega = (2*pi + pose[2])
                else:
                    omega = pose[2]

                if omega > angle:
                    cc = 2*pi - (omega - angle)
                else:
                    cc = angle - omega

                # on the left
                #if angle - omega > 0 and angle - omega < pi:
                if cc < pi:
                    #print "on the left, angle: "+ str(angle) + " omega: "+ str(omega)+ " angle-omega: "+ str(angle-omega)
                    Robot = PolyShapes.Rectangle(self.range*2*j,self.range*2*j)
                    Robot.shift(pose[0]-self.range*j*2,pose[1]-self.range*j)
                    Robot.rotate(pose[2]-pi/2,pose[0],pose[1])

                # on the right
                else:
                    #print "on the right, angle: "+ str(angle) + " omega: "+ str(omega)+ " angle-omega: "+ str(angle-omega)
                    Robot = PolyShapes.Rectangle(self.range*2*j,self.range*2*j)
                    Robot.shift(pose[0],pose[1]-self.range*j)
                    Robot.rotate(pose[2]-pi/2,pose[0],pose[1])

                if self.system == 1:
                    overlap = Robot - ( self.map_work - self.robocomm.getObsPoly())
                else:
                    overlap = Robot - ( self.map_work)

                #self.plotPoly(Robot, 'm',2)
                #determines as dynamic obstacles and can be go striaight to the goal point
                if j >= 2:
                    dis_cur  = vstack((self.q_g[0],self.q_g[1]))- mat([pose[0],pose[1]]).T
                    vx = (dis_cur/norm(dis_cur)/3)[0,0]
                    vy = (dis_cur/norm(dis_cur)/3)[1,0]
                    overlap = None
                    self.overlap = overlap
                    self.q_hit_count        = 0
                    self.boundary_following = False
                    self.m_line             = None
                    self.drive_handler.setVelocity(vx,vy, pose[2])
                    RobotPoly = PolyShapes.Circle(self.PioneerLengthHalf+0.06,(pose[0],pose[1]))     ###0.05
                    departed = not self.currentRegionPoly.overlaps(self.realRobot)
                    #departed = not self.currentRegionPoly.overlaps(self.realRobot) and (not (self.nextRegionPoly.overlaps(self.realRobot) and not self.nextRegionPoly.covers(self.realRobot)))
                    arrived  = self.nextRegionPoly.covers(self.realRobot)
                    return arrived

            ##extra box plotting in figure 1#
            if self.PLOT_OVERLAP == True:
                plt.figure(self.original_figure)
                plt.clf()
                self.plotPoly(self.realRobot, 'r')
                self.plotPoly(self.robot, 'b')
                self.plotPoly(overlap,'g',3)
                plt.plot(pose[0],pose[1],'bo')
                self.plotPioneer(self.original_figure)

            # find the closest point on the obstacle to the robot
            overlap_len = len(overlap)
            for j in range(overlap_len):
                BoundPolyPoints = asarray(overlap[j])
                for i in range(len(BoundPolyPoints)-1):
                    bundle_x = (BoundPolyPoints[i,0] +BoundPolyPoints[1+i,0])/2    #mid-point coordinate x
                    bundle_y = (BoundPolyPoints[i,1] +BoundPolyPoints[1+i,1])/2    #mid-point coordinate y
                    q_overlap = hstack((q_overlap,vstack((bundle_x,bundle_y))))
                bundle_x = (BoundPolyPoints[len(BoundPolyPoints)-1,0] +BoundPolyPoints[0,0])/2    #mid-point coordinate x
                bundle_y = (BoundPolyPoints[len(BoundPolyPoints)-1,1] +BoundPolyPoints[0,1])/2    #mid-point coordinate y
                q_overlap = hstack((q_overlap,vstack((bundle_x,bundle_y))))
            q_overlap = q_overlap.transpose()
            pt =  self.closest_pt([pose[0],pose[1]], vstack((q_overlap,asarray(PolyUtils.pointList(overlap)))))
            self.prev_follow = pt

            #calculate the vector to follow the obstacle
            normal = mat([pose[0],pose[1]] - pt)

            #find the distance from the closest point
            distance = norm(normal)
            velocity = normal * self.trans_matrix
            vx = (velocity/norm(velocity)/3)[0,0]
            vy = (velocity/norm(velocity)/3)[0,1]

            # push or pull the robot towards the obstacle depending on whether the robot is close or far from the obstacle.
            turn = pi/4*(distance-0.5*self.obsRange)/(self.obsRange)    ### change to 0.6 from 0.5 for more allowance in following
            corr_matrix       = mat([[cos(turn),-sin(turn)],[sin(turn),cos(turn)]])
            v =  corr_matrix*mat([[vx],[vy]])
            vx = v[0,0]
            vy = v[1,0]

            ##plotting overlap on figure 2
            if self.PLOT_OVERLAP == True:
                plt.figure(self.overlap_figure)
                plt.clf()
                self.plotPoly(self.m_line,'b');
                self.plotPoly(overlap,'r');
                plt.plot(pt[0],pt[1],'ro')
                plt.plot(pose[0],pose[1],'bo')
                self.plotPioneer(self.overlap_figure,0)

            ## conditions that the loop will end
            #for 11111
            RobotPoly = PolyShapes.Circle(self.PioneerLengthHalf+0.06,(pose[0],pose[1]))   ####0.05
            departed = not self.currentRegionPoly.overlaps(self.realRobot)
            #departed = not self.currentRegionPoly.overlaps(self.realRobot) and (not (self.nextRegionPoly.overlaps(self.realRobot) and not self.nextRegionPoly.covers(self.realRobot)))
            arrived  = self.nextRegionPoly.covers(self.realRobot)

            #for 33333
            reachMLine= self.m_line.overlaps(RobotPoly)

            # 1.reached the next region
            if arrived:
                self.boundary_following = False
                self.m_line             = None
                self.q_hit_count       = 0
                print "arriving at the next region. Exit boundary following mode"
                vx = 0
                vy = 0

                """
                # 2.q_hit is reencountered
                elif norm(self.q_hit-mat([pose[0],pose[1]]).T) < 0.05 and self.q_hit_count > self.q_hit_Thres:
                    print "reencounter q_hit. cannot reach q_goal"
                    vx = 0
                    vy = 0
                """

            # 3.m-line reencountered
            elif reachMLine:
                #print >>sys.__stdout__, "m-line overlaps RoboPoly, m-line" + str(norm(self.q_g-self.q_hit)-2*self.obsRange) + " distance: " + str(norm(self.q_g-mat([pose[0],pose[1]]).T))
                if norm(self.q_g-mat([pose[0],pose[1]]).T) < norm(self.q_g-self.q_hit)-2*self.obsRange:
                    #print "m-line overlaps RoboPoly, m-line" + str(norm(self.q_g-self.q_hit)-2*self.obsRange) + " distance: " + str(norm(self.q_g-mat([pose[0],pose[1]]).T))
                    #print "leaving boundary following mode"
                    self.boundary_following = False
                    self.m_line             = None
                    self.q_hit_count       = 0
                    leaving                = False

                    # turn the robot till it is facing the goal
                    while not leaving:
                        x = self.q_g[0] -self.pose_handler.getPose()[0]
                        y = self.q_g[1] -self.pose_handler.getPose()[1]
                        angle = atan(y/x)
                        if x > 0 and y > 0:
                            angle = angle
                        elif x < 0 and y > 0:
                            angle = pi + angle
                        elif x <0 and y < 0:
                            angle = pi + angle
                        else:
                            angle = 2*pi + angle

                        if self.pose_handler.getPose()[2] < 0:
                            omega = (2*pi + self.pose_handler.getPose()[2])
                            #print >>sys.__stdout__,"omega<0: "+ str(omega)
                        else:
                            omega = self.pose_handler.getPose()[2]
                            #print >>sys.__stdout__,"omega: "+ str(omega)

                        if omega > angle:
                            cc = 2*pi - (omega - angle)
                        else:
                            cc = angle - omega

                        # angle(goal point orientation) on the left of omega(robot orientation)
                        #if angle - omega > 0 and angle - omega < pi:
                        if cc < pi:
                            #print>>sys.__stdout__, "turn left"
                            vx,vy = self.turnLeft(cc)

                        # on the right
                        else:
                            #print>>sys.__stdout__, "turn right"
                            vx, vy = self.turnRight(2*pi-cc)

                        #print>>sys.__stdout__, "omega: "+ str(omega) + " angle: "+ str(angle) + " (omega-angle): " + str(omega-angle)
                        self.drive_handler.setVelocity(vx,vy, self.pose_handler.getPose()[2])

                        if omega - angle < pi/6 and omega - angle > -pi/6:
                            leaving = True

            #Check whether the robot can leave now (the robot has to be closer to the goal than when it is at q_hit to leave)
            QGoalPoly= PolyShapes.Circle(self.PioneerLengthHalf,(self.q_g[0],self.q_g[1]))
            path  = PolyUtils.convexHull(self.realRobot + QGoalPoly)
            if self.system == 1:
                if self.robocomm.getReceiveObs() == False:
                    pathOverlap = path - ( self.map_work)
                else:
                    pathOverlap = path  - ( self.map_work - self.robocomm.getObsPoly())
            else:
                pathOverlap = path - ( self.map_work)

            if not bool(pathOverlap):
                #print "There is NO MORE obstacles in front for now."

                # check if the robot is closer to the goal compared with q_hit
                if norm(self.q_hit-mat(self.q_g).T) > norm(mat([pose[0],pose[1]]).T-mat(self.q_g).T) :
                    #print "The robot is closer than the leaving point. The robot can leave"
                    self.boundary_following = False
                    self.m_line             = None
                    self.q_hit_count       = 0
                    dis_cur  = vstack((self.q_g[0],self.q_g[1]))- mat([pose[0],pose[1]]).T
                    vx = (dis_cur/norm(dis_cur)/3)[0,0]
                    vy = (dis_cur/norm(dis_cur)/3)[1,0]
                else:
                    lala = 1
                    #print "not leaving bug algorithm. difference(-farther) =" + str(norm(self.q_hit-mat(self.q_g).T) - norm(mat([pose[0],pose[1]]).T-mat(self.q_g).T))

        """
        # Pass this desired velocity on to the drive handler
        # Check if there are obstacles within 0.35m of the robot, if so, stop the robot
        if self.system == 1:
            if self.robocomm.getSTOP() == True:
                vx = 0
                vy = 0
        """
        #vx = 0
        #vy = 0
        self.overlap = overlap
        self.drive_handler.setVelocity(vx,vy, pose[2])

        # Set the current region as the previous current region(for checking whether the robot has arrived at the next region)
        self.previous_current_reg = current_reg

        # check whether robot has arrived at the next region
        RobotPoly = PolyShapes.Circle(self.PioneerLengthHalf+0.06,(pose[0],pose[1]))     ###0.05
        #departed = not self.currentRegionPoly.overlaps(self.realRobot) and (not (self.nextRegionPoly.overlaps(self.realRobot) and not self.nextRegionPoly.covers(self.realRobot)))
        departed = not self.currentRegionPoly.overlaps(self.realRobot)
        arrived  = self.nextRegionPoly.covers(self.realRobot)
        if arrived:
            self.q_hit_count        = 0
            self.boundary_following = False
            self.m_line             = None

        if departed and (not arrived) and (time.time()-self.last_warning) > 0.5:
            print "WARNING: Left current region but not in expected destination region"
            # Figure out what region we think we stumbled into
            for r in self.proj.rfi.regions:
                pointArray = [self.coordmap_map2lab(x) for x in r.getPoints()]
                vertices = mat(pointArray).T

                if is_inside([pose[0], pose[1]], vertices):
                    #print "I think I'm in " + r.name
                    #print pose
                    break
            self.last_warning = time.time()

        return arrived

    def plotPioneer(self,number,y = 1):
        """
        Plotting regions and obstacles with matplotlib.pyplot

        number: figure number (see on top)
        y = 0 : plot self.map_work instead of self.map
        """
        if self.operate_system == 1:
            if not plt.isinteractive():
                plt.ion()
            plt.hold(True)

        for regionName,regionPoly in self.map.iteritems():
            self.plotPoly(regionPoly,'k')

        if self.system == 1:
            if bool(self.robocomm.getObsPoly()):
                self.plotPoly(self.robocomm.getObsPoly(),'k')

        if self.operate_system == 1:
            plt.figure(number).canvas.draw()

    def turnLeft(self,a):
        """
        Turn left with angle a
        """
        vx = cos(self.pose_handler.getPose()[2]+a)* self.PioneerLengthHalf;
        vy = sin(self.pose_handler.getPose()[2]+a)* self.PioneerLengthHalf;
        return vx,vy

    def turnRight(self,a):
        """
        Turn right with angle a
        """
        vx = cos(self.pose_handler.getPose()[2]-a)* self.PioneerLengthHalf;
        vy = sin(self.pose_handler.getPose()[2]-a)* self.PioneerLengthHalf;
        return vx,vy

    def euclid(self,pt1, pt2):
        """
        for calculating the minimum distance from a point on the obstacle
        """
        pairs = zip(pt1, pt2)                            # Form pairs in corresponding dimensions
        sum_sq_diffs = sum((a - b)**2 for a, b in pairs) # Find sum of squared diff
        return (sum_sq_diffs)**(float(1)/2)              # Take sqrt to get euclidean distance

    def closest_pt(self,pt, vec):
        """
        Returns the point in vec with minimum euclidean distance to pt
        """
        return min(vec, key=lambda x: self.euclid(pt, x))

    def plotPoly(self,c,string,w = 1):
        """
        Plot polygons inside the boundary
        c = polygon to be plotted with matlabplot
        string = string that specify color
        w      = width of the line plotting
        """
        if bool(c):
            for i in range(len(c)):
                #toPlot = Polygon.Polygon(c.contour(i))
                toPlot = Polygon.Polygon(c.contour(i)) & self.all
                if bool(toPlot):
                    for j in range(len(toPlot)):
                        #BoundPolyPoints = asarray(PolyUtils.pointList(toPlot.contour(j)))
                        BoundPolyPoints = asarray(PolyUtils.pointList(Polygon.Polygon(toPlot.contour(j))))
                        if self.operate_system == 2:
                            self.ax.plot(BoundPolyPoints[:,0],BoundPolyPoints[:,1],string,linewidth=w)
                            self.ax.plot([BoundPolyPoints[-1,0],BoundPolyPoints[0,0]],[BoundPolyPoints[-1,1],BoundPolyPoints[0,1]],string,linewidth=w)
                        else:
                            plt.plot(BoundPolyPoints[:,0],BoundPolyPoints[:,1],string,linewidth=w)
                            plt.plot([BoundPolyPoints[-1,0],BoundPolyPoints[0,0]],[BoundPolyPoints[-1,1],BoundPolyPoints[0,1]],string,linewidth=w)

    def getOldRegionName(self, regionName):
        """
        This function returns the old region name (eg: r1, r2, other etc) when given the new region name (eg: p1, p2..)
        """
        for oldRegionName,newRegionNames in self.proj.regionMapping.iteritems():
            if regionName in newRegionNames:
                return oldRegionName
        print 'Cannot find region with sub-region %s' % regionName
        return None

    def createRegionPolygon(self,region,hole = None):
        """
        This function takes in the region points and make it a Polygon.
        """
        if hole == None:
            pointArray = [x for x in region.getPoints()]
        else:
            pointArray = [x for x in region.getPoints(hole_id = hole)]
        pointArray = map(self.coordmap_map2lab, pointArray)
        regionPoints = [(pt[0],pt[1]) for pt in pointArray]
        formedPolygon= Polygon.Polygon(regionPoints)
        return formedPolygon

    def data_gen(self):
        self.ax.cla()
        self.plotPioneer(1)
        self.plotPoly(self.realRobot, 'r')
        self.plotPoly(self.robot, 'b')
        pose = self.pose_handler.getPose()
        self.ax.plot(pose[0],pose[1],'bo')
        self.ax.plot(self.q_g[0],self.q_g[1],'ro')
        self.plotPoly(self.overlap,'g')
        self.plotPoly(self.m_line,'b')
        yield(pose[0],pose[1])
        self.ax.plot(self.prev_follow[0],self.prev_follow[1],'ko')

    def jplot(self):
        ani = animation.FuncAnimation(self.fig, self.scope.update, self.data_gen)
        plt.show()

class _Scope:
    def __init__(self, ax, motion, maxt=2, dt=0.02):
        self.i = 0
        self.ax = ax
        self.line, = self.ax.plot(1)
        self.ax.set_ylim(0, 1)
        self.motion = motion

    def update(self,data):
        (data1) = self.motion.data_gen()
        a = data1.next()
        self.line.set_data(a)
        self.ax.relim()
        self.ax.autoscale()
        return self.line,


#  Copyright 2015-present The Scikit Flow Authors. All Rights Reserved.
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import random

import numpy as np

from sklearn import datasets
from sklearn.metrics import accuracy_score, mean_squared_error

import tensorflow as tf
from tensorflow.contrib.skflow.python import skflow

class MultiOutputTest(tf.test.TestCase):

    def testMultiRegression(self):
        random.seed(42)
        rng = np.random.RandomState(1)
        X = np.sort(200 * rng.rand(100, 1) - 100, axis=0)
        y = np.array([np.pi * np.sin(X).ravel(), np.pi * np.cos(X).ravel()]).T
        regressor = skflow.TensorFlowLinearRegressor(learning_rate=0.01)
        regressor.fit(X, y)
        score = mean_squared_error(regressor.predict(X), y)
        self.assertLess(score, 10, "Failed with score = {0}".format(score))

if __name__ == "__main__":
    tf.test.main()


import re
import matplotlib.pyplot as plt
import seaborn as sns

MORSE_CODE = {
    ".-": "A",
    "-...": "B",
    "-.-.": "C",
    "-..": "D",
    ".": "E",
    "..-.": "F",
    "--.": "G",
    "....": "H",
    "..": "I",
    ".---": "J",
    "-.-": "K",
    ".-..": "L",
    "--": "M",
    "-.": "N",
    "---": "O",
    ".--.": "P",
    "--.-": "Q",
    ".-.": "R",
    "...": "S",
    "-": "T",
    "..-": "U",
    "...-": "V",
    ".--": "W",
    "-..-": "X",
    "-.--": "Y",
    "--..": "Z",
    "-----": "0",
    ".----": "1",
    "..---": "2",
    "...--": "3",
    "....-": "4",
    ".....": "5",
    "-....": "6",
    "--...": "7",
    "---..": "8",
    "----.": "9",
    "...---...": "SOS"
}

heyJude = ".... . -.--   .--- ..- -.. ."
JudeBits = "00011001100110011000000110000001111110011001111110011111100000000000000110011111100111111001111110000001100110011111100000011111100110011000000110000"
fuzzyBits = "0000000011011010011100000110000001111110100111110011111100000000000111011111111011111011111000000101100011111100000111110011101100000100000"
fuzzyTest = "00000000000000011111111000000011111111111100000000000111111111000001111111110100000000111111111111011000011111111011111111111000000000000000000011111111110000110001111111111111000111000000000001111111111110000111111111100001100111111111110000000000111111111111011100001110000000000000000001111111111010111111110110000000000000001111111111100001111111111110000100001111111111111100000000000111111111000000011000000111000000000000000000000000000011110001111100000111100000000111111111100111111111100111111111111100000000011110011111011111110000000000000000000000111111111110000000011111000000011111000000001111111111110000000001111100011111111000000000111111111110000011000000000111110000000111000000000011111111111111000111001111111111001111110000000000000000000001111000111111111100001111111111111100100000000001111111100111111110111111110000000011101111111000111000000001001111111000000001111111111000000000111100001111111000000000000011111111100111111110111111111100000000000111111110000001100000000000000000000111111101010000010000001111111100000000011111000111111111000000111111111110011111111001111111110000000011000111111110000111011111111111100001111100001111111100000000000011110011101110001000111111110000000001111000011111110010110001111111111000000000000000000111111111110000000100000000000000000011110111110000001000011101110000000000011111111100000011111111111100111111111111000111111111000001111111100000000000001110111111111111000000110011111111111101110001111111111100000000111100000111100000111111111100000111111111111000000011111111000000000001000000111100000001000001111100111111111110000000000000000000010001111111100000011111111100000000000000100001111111111110111001111111111100000111111100001111111111000000000000000000000000011100000111111111111011110000000010000000011111111100011111111111100001110000111111111111100000000000000111110000011111001111111100000000000011100011100000000000011111000001111111111101000000001110000000000000000000000000000111110010000000000111111111000011111111110000000000111111111111101111111111100000000010000000000000011111111100100001100000000000000111100111100000000001100000001111111111110000000011111111111000000000111100000000000000000000111101111111111111000000000001111000011111000011110000000001100111111100111000000000100111000000000000111110000010000011111000000000000001111111111100000000110111111111100000000000000111111111111100000111000000000111111110001111000000111111110111111000000001111000000000010000111111111000011110001111111110111110000111111111111000000000000000000000000111111111110000000111011111111100011111110000000001111111110000011111111100111111110000000001111111111100111111111110000000000110000000000000000001000011111111110000000001111111110000000000000000000000011111111111111000000111111111000001111111110000000000111111110000010000000011111111000011111001111111100000001110000000011110000000001011111111000011111011111111110011011111111111000000000000000000100011111111111101111111100000000000000001100000000000000000011110010111110000000011111111100000000001111100011111111111101100000000111110000011110000111111111111000000001111111111100001110111111111110111000000000011111111101111100011111111110000000000000000000000000010000111111111100000000001111111110111110000000000000000000000110000011110000000000001111111111100110001111111100000011100000000000111110000000011111111110000011111000001111000110000000011100000000000000111100001111111111100000111000000001111111111000000111111111100110000000001111000001111111100011100001111111110000010011111111110000000000000000000111100000011111000001111000000000111111001110000000011111111000100000000000011111111000011001111111100000000000110111000000000000111111111111000100000000111111111110000001111111111011100000000000000000000000000"

class Cluster(object):
    def __init__(self, loc):
        self.currentPoints = []
        self.centroid = None
        self.previousPoints = []
        self.location = loc

    ##  Methods for claiming currentPoints and calculating centroid.
    def addPoint(self, point):
        self.currentPoints.append(point)

    def didChange(self):
        if len(self.currentPoints) != len(self.previousPoints):
            return True
        else:
            return not (self.currentPoints == self.previousPoints)

    def clearPoints(self):
        self.previousPoints = self.currentPoints[:]
        del self.currentPoints[:]

    def update(self):
        '''
        After new points have been assigned to this cluster, this method
        calculates the new centroid of the cluster and moves the cluster
        to that location.
        '''
        if len(self.currentPoints) > 0:
            s = 0.0
            for p in self.currentPoints:
                s += p
            self.centroid = s / len(self.currentPoints)
            self.location = self.centroid

    ##  Getter methods.
    def getLocation(self):
        return self.location

    def getDistance(self, point):
        return abs(self.location - point)

    ##  Printer methods.
    def printCentroid(self):
        print(self.centroid)

    def printLocation(self):
        print(self.location)

    def printPoints(self):
        result = ""
        for point in self.currentPoints:
            result += str(point) + " "
        print(result[:-1])

    def printPreviousPoints(self):
        result = ""
        for point in self.previousPoints:
            result += str(point) + " "
        print(result[:-1])

class KMeans(object):
    def __init__(self, stream, numClusters):
        self.clusters = []
        self.bitCollection = []
        self.timeUnits = [0,0,0]
        self.dist = {}
        self.keys = []
        self.converged = False

        stream = stream.strip("0")

        ##  Populate this.bitCollection.
        if len(stream) == 0:
            self.bitCollection.append("")
        else:
            ones = re.split("0+", stream)
            zeros = re.split("1+", stream)
            if len(zeros) == 0:
                self.bitCollection.append(ones[0])
            else:
                for i in range(len(ones) - 1):
                    self.bitCollection.append(ones[i])
                    self.bitCollection.append(zeros[i + 1])
                self.bitCollection.append(ones[-1])

        ##  Populate this.dist.
        for bit in self.bitCollection:
            l = len(bit)
            if l in self.dist:
                self.dist[l] += 1
            else:
                self.dist[l] = 1
        self.keys = sorted(self.dist.keys())

        ##  Handle short inputs (i.e. displays fewer than three timing units)
        if len(self.keys) == 1 or len(self.keys) == 2:
            self.timeUnits[0] = self.keys[0]
            self.timeUnits[1] = self.keys[0] * 3
            self.timeUnits[2] = self.keys[0] * 7
            self.converged = True

        ##  Handle long inputs via KMeans
        else:
            self.initializeClusters()

    def initializeClusters(self):
        '''
        Populates this.clusters with this.numClusters Cluster objects,
        whose initial locations are from this.keys (the minimum, the
        maximum, and the middle between the two).
        '''
        self.clusters.append(Cluster(float(self.keys[0])))
        self.clusters.append(Cluster((float(self.keys[0]) + float(self.keys[-1])) / 2))
        self.clusters.append(Cluster(float(self.keys[-1])))

    def assignToClosestCluster(self):
        '''
        Assigns cluster-labels to each length-point from the fuzzy input,
        which is subsequently used by the clusters to re-calculate their
        centroids and move accordingly.
        '''
        self.clear()
        for key in self.keys:
            bestCluster = Cluster(5000)
            closest = 10000000.0
            for c in self.clusters:
                d = c.getDistance(key)
                if d < closest:
                    closest = d
                    bestCluster = c
            for i in range(self.dist[key]):
                bestCluster.addPoint(key)

    def calculateTimeUnits(self):
        for i in range(3):
            self.timeUnits[i] = self.clusters[i].getLocation()

    def clear(self):
        for c in self.clusters:
            c.clearPoints()

    def converge(self):
        '''
        Assigns the closest Cluster to each point, calculates the centroid
        for those Clusters based off of those points, moves the Clusters
        to their respective centroids, and repeats until assignment on the next
        iteration is the same.
        '''
        if not self.converged:
            self.assignToClosestCluster()
            while not self.converged:
                self.update()
                self.assignToClosestCluster()
                if not self.didChange():
                    self.converged = True
            self.calculateTimeUnits()

    def didChange(self):
        for c in self.clusters:
            if c.didChange():
                return True
        return False

    def update(self):
        for c in self.clusters:
            c.update()

    ##  Getter methods.
    def getTimeUnit(self, index):
        return self.timeUnits[index]

    ##  Printer methods.
    def printBitCollection(self):
        for bit in self.bitCollection:
            print(bit)

    def printClusterPoints(self):
        for c in self.clusters:
            print("Points for cluster at " + str(c.getLocation()))
            c.printPoints()

    def printClusters(self):
        for c in self.clusters:
            print(c.getLocation())

    def printDidChange(self):
        print(self.didChange)

    def printDistances(self):
        for key in self.keys:
            best = -1.0
            closest = 10000000.0
            for c in self.clusters:
                d = c.getDistance(key)
                print("From cluster at " + str(c.getLocation()) + \
                "to point at " + str(key) + " is: " + str(d))
                if d < closest:
                    closest = d
                    best = c.getLocation()
            print("Closest to: " + str(best))

    def printDistribution(self):
        for key in self.keys:
            print("Length: " + str(key) + " occurred " + str(self.dist[key]) + " times")

    def printKeys(self):
        for key in self.keys:
            print(key)

    def printTimeUnits(self):
        for t in self.timeUnits:
            print(t)

    ##  Plotter methods.
    def plotDistribution(self):
        xmax = max(self.keys)
        ymax = max(self.dist.values())
        plt.figure()
        plt.bar(self.dist.keys(), self.dist.values())
        plt.title("Bit Length Frequencies")
        plt.xlabel("Number of Characters")
        plt.ylabel("Frequency")
        plt.axis([0, xmax, 0, ymax])
        plt.axvline((self.getTimeUnit(0) + self.getTimeUnit(1)) / 2, color='b', linestyle='dashed', linewidth=2)
        plt.axvline((self.getTimeUnit(1) + self.getTimeUnit(2)) / 2, color='b', linestyle='dashed', linewidth=2)
        plt.show()

# Required as per problem API
def decodeBitsAdvanced(fuzzyBits):
    '''
    input bits, a string of 0s and 1s with variable timing
    returns string, a morse code message
    '''
    morse = ""
    fuzzyBits = fuzzyBits.strip("0")
    km = KMeans(fuzzyBits, 3)
    km.converge()
    thresh13 = (km.getTimeUnit(0) + km.getTimeUnit(1)) / 2
    thresh37 = (km.getTimeUnit(1) + km.getTimeUnit(2)) / 2
    ones = re.split("0+", fuzzyBits)
    zeros = re.split("1+", fuzzyBits)
    for i in range(len(zeros) - 1):
        morse += nextTelePairFuzzy(ones[i], zeros[i + 1], thresh13, thresh37)
    return morse

# Required as per problem API
def decodeBits(bits):
    '''
    input bits, a string of 0s and 1s with fixed timing
    returns string, a morse code message
    '''
    morse = ""
    bits = bits.strip("0")
    tu = getTimeUnit(bits)
    ones = re.split("0+", bits)
    zeros = re.split("1+",bits)
    for i in range(len(zeros) - 1):
        morse += nextTelePair(ones[i], zeros[i + 1], tu)
    return morse

# Required as per problem API
def decodeMorse(morseCode):
    '''
    input morseCode, a string of dots, dashes, and spaces
    returns string, a human-readable message
    '''
    result = ""
    morseCode = morseCode.replace("   ", " SPACE ")
    morses = morseCode.split()
    for morse in morses:
        if morse == "SPACE":
            result += " "
        else:
            try:
                result += MORSE_CODE[morse]
            except KeyError:
                result += "(KEYERR: "+morse+")"
    return result

# Helper function for 3 of 3 in the series
def nextTelePairFuzzy(one, zero, thresh13, thresh37):
    tele = nextTeleSingleFuzzy(one, thresh13)
    if len(zero) >= thresh13 and len(zero) < thresh37:
        tele += " "
    elif len(zero) >= thresh37:
        tele += "   "
    return tele

# Helper function for 3 of 3 in the series
def nextTeleSingleFuzzy(one, thresh13):
    tele = ""
    if len(one) <= thresh13:
        tele += "."
    else:
        tele += "-"
    return tele

# Helper function for 2 of 3 in the series
def nextTelePair(one, zero, tu):
    tele = nextTeleSingle(one, tu)
    if len(zero) == 3 * tu:
        tele += " "
    elif len(zero) == 7 * tu:
        tele += "   "
    return tele

# Helper function for 2 of 3 in the series
def nextTeleSingle(one, tu):
    tele = ""
    if len(one) == tu:
        tele += "."
    elif len(one) == 3 * tu:
        tele += "-"
    return tele

# Helper function for 2 of 3 in the series
def getTimeUnit(bits):
    '''
    input bits, a string of 0s and 1s with fixed timing
    returns int, the single timing unit (i.e. dot or shortest pause)
    '''
    if bits == "":
        return 0
    o = re.split("0+", bits)
    if len(o) == 1:
        return len(bits)
    if o == ['','']:
        return 0
    os = len(o[0])
    for elem in o:
        if len(elem) != os:
            os = min(os,len(elem))
            break
    z = re.split("1+", bits)
    zs = len(z[1])
    for i in range(1, len(z) - 1):
        if zs != len(z[i]):
            zs = min(zs, len(z[i]))
            break
    return min(os, zs)

# Function to rapidly iterate through possible thresholds;
# not a valid way of solving 3 of 3.
def bruteThreshholds(fuzzyBits):
    fuzzyBits = fuzzyBits.strip("0")
    f = open('BruteForceDump', 'w')
    ones = re.split("0+", fuzzyBits)
    zeros = re.split("1+", fuzzyBits)
    lowerStart = 7
    lowerStop = 8       # exclusive
    lowerStep = 0.25
    upperStart = 15
    upperStop = 16      # exclusive
    upperStep = 0.25
    for lower in range(0, int((lowerStop - lowerStart) / lowerStep), 1):
        for upper in range(0, int((upperStop - upperStart) / upperStep), 1):
            morse = ""
            for i in range(len(zeros) - 1):
                morse += nextTelePairFuzzy(ones[i], zeros[i + 1],
                    lowerStart + lower * lowerStep, upperStart + upper * upperStep)
            f.write(str(lowerStart + lower * lowerStep) + " " + str(upperStart + upper * upperStep) + '\n')
            f.write(decodeMorse(morse))
            f.write('\n\n')


import numpy as np
import pytest

import pandas as pd
from pandas import Int64Index, TimedeltaIndex, timedelta_range
import pandas.util.testing as tm

from pandas.tseries.offsets import Hour

class TestTimedeltaIndex:
    def test_union(self):

        i1 = timedelta_range("1day", periods=5)
        i2 = timedelta_range("3day", periods=5)
        result = i1.union(i2)
        expected = timedelta_range("1day", periods=7)
        tm.assert_index_equal(result, expected)

        i1 = Int64Index(np.arange(0, 20, 2))
        i2 = timedelta_range(start="1 day", periods=10, freq="D")
        i1.union(i2)  # Works
        i2.union(i1)  # Fails with "AttributeError: can't set attribute"

    def test_union_coverage(self):

        idx = TimedeltaIndex(["3d", "1d", "2d"])
        ordered = TimedeltaIndex(idx.sort_values(), freq="infer")
        result = ordered.union(idx)
        tm.assert_index_equal(result, ordered)

        result = ordered[:0].union(ordered)
        tm.assert_index_equal(result, ordered)
        assert result.freq == ordered.freq

    def test_union_bug_1730(self):

        rng_a = timedelta_range("1 day", periods=4, freq="3H")
        rng_b = timedelta_range("1 day", periods=4, freq="4H")

        result = rng_a.union(rng_b)
        exp = TimedeltaIndex(sorted(set(list(rng_a)) | set(list(rng_b))))
        tm.assert_index_equal(result, exp)

    def test_union_bug_1745(self):

        left = TimedeltaIndex(["1 day 15:19:49.695000"])
        right = TimedeltaIndex(
            ["2 day 13:04:21.322000", "1 day 15:27:24.873000", "1 day 15:31:05.350000"]
        )

        result = left.union(right)
        exp = TimedeltaIndex(sorted(set(list(left)) | set(list(right))))
        tm.assert_index_equal(result, exp)

    def test_union_bug_4564(self):

        left = timedelta_range("1 day", "30d")
        right = left + pd.offsets.Minute(15)

        result = left.union(right)
        exp = TimedeltaIndex(sorted(set(list(left)) | set(list(right))))
        tm.assert_index_equal(result, exp)

    def test_intersection_bug_1708(self):
        index_1 = timedelta_range("1 day", periods=4, freq="h")
        index_2 = index_1 + pd.offsets.Hour(5)

        result = index_1 & index_2
        assert len(result) == 0

        index_1 = timedelta_range("1 day", periods=4, freq="h")
        index_2 = index_1 + pd.offsets.Hour(1)

        result = index_1 & index_2
        expected = timedelta_range("1 day 01:00:00", periods=3, freq="h")
        tm.assert_index_equal(result, expected)

    @pytest.mark.parametrize("sort", [None, False])
    def test_intersection_equal(self, sort):
        # GH 24471 Test intersection outcome given the sort keyword
        # for equal indicies intersection should return the original index
        first = timedelta_range("1 day", periods=4, freq="h")
        second = timedelta_range("1 day", periods=4, freq="h")
        intersect = first.intersection(second, sort=sort)
        if sort is None:
            tm.assert_index_equal(intersect, second.sort_values())
        assert tm.equalContents(intersect, second)

        # Corner cases
        inter = first.intersection(first, sort=sort)
        assert inter is first

    @pytest.mark.parametrize("period_1, period_2", [(0, 4), (4, 0)])
    @pytest.mark.parametrize("sort", [None, False])
    def test_intersection_zero_length(self, period_1, period_2, sort):
        # GH 24471 test for non overlap the intersection should be zero length
        index_1 = timedelta_range("1 day", periods=period_1, freq="h")
        index_2 = timedelta_range("1 day", periods=period_2, freq="h")
        expected = timedelta_range("1 day", periods=0, freq="h")
        result = index_1.intersection(index_2, sort=sort)
        tm.assert_index_equal(result, expected)

    @pytest.mark.parametrize("sort", [None, False])
    def test_zero_length_input_index(self, sort):
        # GH 24966 test for 0-len intersections are copied
        index_1 = timedelta_range("1 day", periods=0, freq="h")
        index_2 = timedelta_range("1 day", periods=3, freq="h")
        result = index_1.intersection(index_2, sort=sort)
        assert index_1 is not result
        assert index_2 is not result
        tm.assert_copy(result, index_1)

    @pytest.mark.parametrize(
        "rng, expected",
        # if target has the same name, it is preserved
        [
            (
                timedelta_range("1 day", periods=5, freq="h", name="idx"),
                timedelta_range("1 day", periods=4, freq="h", name="idx"),
            ),
            # if target name is different, it will be reset
            (
                timedelta_range("1 day", periods=5, freq="h", name="other"),
                timedelta_range("1 day", periods=4, freq="h", name=None),
            ),
            # if no overlap exists return empty index
            (
                timedelta_range("1 day", periods=10, freq="h", name="idx")[5:],
                TimedeltaIndex([], name="idx"),
            ),
        ],
    )
    @pytest.mark.parametrize("sort", [None, False])
    def test_intersection(self, rng, expected, sort):
        # GH 4690 (with tz)
        base = timedelta_range("1 day", periods=4, freq="h", name="idx")
        result = base.intersection(rng, sort=sort)
        if sort is None:
            expected = expected.sort_values()
        tm.assert_index_equal(result, expected)
        assert result.name == expected.name
        assert result.freq == expected.freq

    @pytest.mark.parametrize(
        "rng, expected",
        # part intersection works
        [
            (
                TimedeltaIndex(["5 hour", "2 hour", "4 hour", "9 hour"], name="idx"),
                TimedeltaIndex(["2 hour", "4 hour"], name="idx"),
            ),
            # reordered part intersection
            (
                TimedeltaIndex(["2 hour", "5 hour", "5 hour", "1 hour"], name="other"),
                TimedeltaIndex(["1 hour", "2 hour"], name=None),
            ),
            # reveresed index
            (
                TimedeltaIndex(["1 hour", "2 hour", "4 hour", "3 hour"], name="idx")[
                    ::-1
                ],
                TimedeltaIndex(["1 hour", "2 hour", "4 hour", "3 hour"], name="idx"),
            ),
        ],
    )
    @pytest.mark.parametrize("sort", [None, False])
    def test_intersection_non_monotonic(self, rng, expected, sort):
        # 24471 non-monotonic
        base = TimedeltaIndex(["1 hour", "2 hour", "4 hour", "3 hour"], name="idx")
        result = base.intersection(rng, sort=sort)
        if sort is None:
            expected = expected.sort_values()
        tm.assert_index_equal(result, expected)
        assert result.name == expected.name

        # if reveresed order, frequency is still the same
        if all(base == rng[::-1]) and sort is None:
            assert isinstance(result.freq, Hour)
        else:
            assert result.freq is None


"""
Testing for Theil-Sen module (sklearn.linear_model.theil_sen)
"""

# Author: Florian Wilhelm <florian.wilhelm@gmail.com>
# License: BSD 3 clause

from __future__ import division, print_function, absolute_import

import os
import sys
from contextlib import contextmanager
import numpy as np
from numpy.testing import assert_array_equal, assert_array_less
from numpy.testing import assert_array_almost_equal, assert_warns
from scipy.linalg import norm
from scipy.optimize import fmin_bfgs
from nose.tools import raises, assert_almost_equal
from sklearn.utils import ConvergenceWarning
from sklearn.linear_model import LinearRegression, TheilSenRegressor
from sklearn.linear_model.theil_sen import _spatial_median, _breakdown_point
from sklearn.linear_model.theil_sen import _modified_weiszfeld_step
from sklearn.utils.testing import assert_greater, assert_less

@contextmanager
def no_stdout_stderr():
    old_stdout = sys.stdout
    old_stderr = sys.stderr
    sys.stdout = open(os.devnull, 'w')
    sys.stderr = open(os.devnull, 'w')
    yield
    sys.stdout.flush()
    sys.stderr.flush()
    sys.stdout = old_stdout
    sys.stderr = old_stderr

def gen_toy_problem_1d(intercept=True):
    random_state = np.random.RandomState(0)
    # Linear model y = 3*x + N(2, 0.1**2)
    w = 3.
    if intercept:
        c = 2.
        n_samples = 50
    else:
        c = 0.1
        n_samples = 100
    x = random_state.normal(size=n_samples)
    noise = 0.1 * random_state.normal(size=n_samples)
    y = w * x + c + noise
    # Add some outliers
    if intercept:
        x[42], y[42] = (-2, 4)
        x[43], y[43] = (-2.5, 8)
        x[33], y[33] = (2.5, 1)
        x[49], y[49] = (2.1, 2)
    else:
        x[42], y[42] = (-2, 4)
        x[43], y[43] = (-2.5, 8)
        x[53], y[53] = (2.5, 1)
        x[60], y[60] = (2.1, 2)
        x[72], y[72] = (1.8, -7)
    return x[:, np.newaxis], y, w, c

def gen_toy_problem_2d():
    random_state = np.random.RandomState(0)
    n_samples = 100
    # Linear model y = 5*x_1 + 10*x_2 + N(1, 0.1**2)
    X = random_state.normal(size=(n_samples, 2))
    w = np.array([5., 10.])
    c = 1.
    noise = 0.1 * random_state.normal(size=n_samples)
    y = np.dot(X, w) + c + noise
    # Add some outliers
    n_outliers = n_samples // 10
    ix = random_state.randint(0, n_samples, size=n_outliers)
    y[ix] = 50 * random_state.normal(size=n_outliers)
    return X, y, w, c

def gen_toy_problem_4d():
    random_state = np.random.RandomState(0)
    n_samples = 10000
    # Linear model y = 5*x_1 + 10*x_2  + 42*x_3 + 7*x_4 + N(1, 0.1**2)
    X = random_state.normal(size=(n_samples, 4))
    w = np.array([5., 10., 42., 7.])
    c = 1.
    noise = 0.1 * random_state.normal(size=n_samples)
    y = np.dot(X, w) + c + noise
    # Add some outliers
    n_outliers = n_samples // 10
    ix = random_state.randint(0, n_samples, size=n_outliers)
    y[ix] = 50 * random_state.normal(size=n_outliers)
    return X, y, w, c

def test_modweiszfeld_step_1d():
    X = np.array([1., 2., 3.]).reshape(3, 1)
    # Check startvalue is element of X and solution
    median = 2.
    new_y = _modified_weiszfeld_step(X, median)
    assert_array_almost_equal(new_y, median)
    # Check startvalue is not the solution
    y = 2.5
    new_y = _modified_weiszfeld_step(X, y)
    assert_array_less(median, new_y)
    assert_array_less(new_y, y)
    # Check startvalue is not the solution but element of X
    y = 3.
    new_y = _modified_weiszfeld_step(X, y)
    assert_array_less(median, new_y)
    assert_array_less(new_y, y)
    # Check that a single vector is identity
    X = np.array([1., 2., 3.]).reshape(1, 3)
    y = X[0, ]
    new_y = _modified_weiszfeld_step(X, y)
    assert_array_equal(y, new_y)

def test_modweiszfeld_step_2d():
    X = np.array([0., 0., 1., 1., 0., 1.]).reshape(3, 2)
    y = np.array([0.5, 0.5])
    # Check first two iterations
    new_y = _modified_weiszfeld_step(X, y)
    assert_array_almost_equal(new_y, np.array([1 / 3, 2 / 3]))
    new_y = _modified_weiszfeld_step(X, new_y)
    assert_array_almost_equal(new_y, np.array([0.2792408, 0.7207592]))
    # Check fix point
    y = np.array([0.21132505, 0.78867497])
    new_y = _modified_weiszfeld_step(X, y)
    assert_array_almost_equal(new_y, y)

def test_spatial_median_1d():
    X = np.array([1., 2., 3.]).reshape(3, 1)
    true_median = 2.
    _, median = _spatial_median(X)
    assert_array_almost_equal(median, true_median)
    # Test larger problem and for exact solution in 1d case
    random_state = np.random.RandomState(0)
    X = random_state.randint(100, size=(1000, 1))
    true_median = np.median(X.ravel())
    _, median = _spatial_median(X)
    assert_array_equal(median, true_median)

def test_spatial_median_2d():
    X = np.array([0., 0., 1., 1., 0., 1.]).reshape(3, 2)
    _, median = _spatial_median(X, max_iter=100, tol=1.e-6)

    def cost_func(y):
        dists = np.array([norm(x - y) for x in X])
        return np.sum(dists)

    # Check if median is solution of the Fermat-Weber location problem
    fermat_weber = fmin_bfgs(cost_func, median, disp=False)
    assert_array_almost_equal(median, fermat_weber)
    # Check when maximum iteration is exceeded a warning is emitted
    assert_warns(ConvergenceWarning, _spatial_median, X, max_iter=30, tol=0.)

def test_theil_sen_1d():
    X, y, w, c = gen_toy_problem_1d()
    # Check that Least Squares fails
    lstq = LinearRegression().fit(X, y)
    assert_greater(np.abs(lstq.coef_ - w), 0.9)
    # Check that Theil-Sen works
    theil_sen = TheilSenRegressor(random_state=0).fit(X, y)
    assert_array_almost_equal(theil_sen.coef_, w, 1)
    assert_array_almost_equal(theil_sen.intercept_, c, 1)

def test_theil_sen_1d_no_intercept():
    X, y, w, c = gen_toy_problem_1d(intercept=False)
    # Check that Least Squares fails
    lstq = LinearRegression(fit_intercept=False).fit(X, y)
    assert_greater(np.abs(lstq.coef_ - w - c), 0.5)
    # Check that Theil-Sen works
    theil_sen = TheilSenRegressor(fit_intercept=False,
                                  random_state=0).fit(X, y)
    assert_array_almost_equal(theil_sen.coef_, w + c, 1)
    assert_almost_equal(theil_sen.intercept_, 0.)

def test_theil_sen_2d():
    X, y, w, c = gen_toy_problem_2d()
    # Check that Least Squares fails
    lstq = LinearRegression().fit(X, y)
    assert_greater(norm(lstq.coef_ - w), 1.0)
    # Check that Theil-Sen works
    theil_sen = TheilSenRegressor(max_subpopulation=1e3,
                                  random_state=0).fit(X, y)
    assert_array_almost_equal(theil_sen.coef_, w, 1)
    assert_array_almost_equal(theil_sen.intercept_, c, 1)

def test_calc_breakdown_point():
    bp = _breakdown_point(1e10, 2)
    assert_less(np.abs(bp - 1 + 1/(np.sqrt(2))), 1.e-6)

@raises(ValueError)
def test_checksubparams_negative_subpopulation():
    X, y, w, c = gen_toy_problem_1d()
    TheilSenRegressor(max_subpopulation=-1, random_state=0).fit(X, y)

@raises(ValueError)
def test_checksubparams_too_few_subsamples():
    X, y, w, c = gen_toy_problem_1d()
    TheilSenRegressor(n_subsamples=1, random_state=0).fit(X, y)

@raises(ValueError)
def test_checksubparams_too_many_subsamples():
    X, y, w, c = gen_toy_problem_1d()
    TheilSenRegressor(n_subsamples=101, random_state=0).fit(X, y)

@raises(ValueError)
def test_checksubparams_n_subsamples_if_less_samples_than_features():
    random_state = np.random.RandomState(0)
    n_samples, n_features = 10, 20
    X = random_state.normal(size=(n_samples, n_features))
    y = random_state.normal(size=n_samples)
    TheilSenRegressor(n_subsamples=9, random_state=0).fit(X, y)

def test_subpopulation():
    X, y, w, c = gen_toy_problem_4d()
    theil_sen = TheilSenRegressor(max_subpopulation=250,
                                  random_state=0).fit(X, y)
    assert_array_almost_equal(theil_sen.coef_, w, 1)
    assert_array_almost_equal(theil_sen.intercept_, c, 1)

def test_subsamples():
    X, y, w, c = gen_toy_problem_4d()
    theil_sen = TheilSenRegressor(n_subsamples=X.shape[0],
                                  random_state=0).fit(X, y)
    lstq = LinearRegression().fit(X, y)
    # Check for exact the same results as Least Squares
    assert_array_almost_equal(theil_sen.coef_, lstq.coef_, 9)

def test_verbosity():
    X, y, w, c = gen_toy_problem_1d()
    # Check that Theil-Sen can be verbose
    with no_stdout_stderr():
        TheilSenRegressor(verbose=True, random_state=0).fit(X, y)
        TheilSenRegressor(verbose=True,
                          max_subpopulation=10,
                          random_state=0).fit(X, y)

def test_theil_sen_parallel():
    X, y, w, c = gen_toy_problem_2d()
    # Check that Least Squares fails
    lstq = LinearRegression().fit(X, y)
    assert_greater(norm(lstq.coef_ - w), 1.0)
    # Check that Theil-Sen works
    theil_sen = TheilSenRegressor(n_jobs=-1,
                                  random_state=0,
                                  max_subpopulation=2e3).fit(X, y)
    assert_array_almost_equal(theil_sen.coef_, w, 1)
    assert_array_almost_equal(theil_sen.intercept_, c, 1)

def test_less_samples_than_features():
    random_state = np.random.RandomState(0)
    n_samples, n_features = 10, 20
    X = random_state.normal(size=(n_samples, n_features))
    y = random_state.normal(size=n_samples)
    # Check that Theil-Sen falls back to Least Squares if fit_intercept=False
    theil_sen = TheilSenRegressor(fit_intercept=False,
                                  random_state=0).fit(X, y)
    lstq = LinearRegression(fit_intercept=False).fit(X, y)
    assert_array_almost_equal(theil_sen.coef_, lstq.coef_, 12)
    # Check fit_intercept=True case. This will not be equal to the Least
    # Squares solution since the intercept is calculated differently.
    theil_sen = TheilSenRegressor(fit_intercept=True, random_state=0).fit(X, y)
    y_pred = theil_sen.predict(X)
    assert_array_almost_equal(y_pred, y, 12)


#!/usr/bin/env python
# a bar plot with errorbars
import matplotlib
import numpy as np
import matplotlib.pyplot as plt
from matplotlib.patches import Ellipse, Polygon
from pylab import *

width = 0.28      # the width of the bars

font = {'family' : 'sans-serif',
		'variant' : 'normal',
        'weight' : 'light',
        'size'   : 14}

matplotlib.rc('font', **font)
# plot with various axes scales
plt.figure(1)
fig = gcf()

def plot_bars_FORMATION_STRONG_REL(PersistingMeans, PersistingStd, Means, Std, PERSreal, PERSstd):

	ind = np.arange(N)  # the x locations for the groups
	#width = 0.3       # the width of the bars

	#ax = plt.subplot(321)
	ax = plt.subplot2grid((1,2),(0, 0))
	#rects1 = ax.bar(ind-0.2, PersistingMeans, width, color='c', yerr=PersistingStd, align='center')
	#rects2 = ax.bar(ind+0.2, Means, width, color='cyan', yerr=Std, align='center')

	rects1 = ax.bar(ind-width, PersistingMeans, width, color='darkred', \
		align='center', yerr=PersistingStd, linewidth=0,\
		error_kw=dict(ecolor='gray', lw=1.5, capsize=2.7, capthick=1))

	rects2 = ax.bar(ind, Means, width, color='lightcoral', \
		yerr=Std, align='center', linewidth=0,\
		error_kw=dict(ecolor='gray', lw=1.5, capsize=2.7, capthick=1))

	rects3 = ax.bar(ind+width, PERSreal, width, color='r',\
		yerr=PERSstd, align='center',linewidth=0,\
		error_kw=dict(ecolor='gray', lw=1.5, capsize=2.7, capthick=1))

	ax.legend((rects1[0], rects2[0], rects3[0]), \
		('Formed and persisting', \
			'Formed and non-persisting', 'Persisting average'),\
		frameon=False)

	# add some text for labels, title and axes ticks
	#ax.set_title('Relative status (strong contacts)')
	ax.set_xticks(ind )
	ax.set_xticklabels(('Before', 'At formation', 'After'))

	ax.set_ylim([-0.5, 5])
	ax.set_yticks((0,5))

	def autolabel(rects):
	    # attach some text labels
	    for rect in rects:
	        height = rect.get_height()
	        ax.text(rect.get_x() + rect.get_width()/2., 1.05*height,
	                '%.2f' % float(height),
	                ha='center', va='bottom')

	autolabel(rects1)
	autolabel(rects2)
	autolabel(rects3)

	return plt

N = 3
##########################################################################
# NON PERSISTING LINKS
# STRONG contacts REL
formationDeletionMeans = (1.12747979427, 1.56808719079, 1.62160176341)
formationDeletionStd = (1.35650452374, 1.71205560699, 1.83913259462)

# PERSISTING LINKS
# STRONG contacts REL
formationNodeletionMeans = (0.964889222681, 1.44874202028, 1.68794592565)
formationNodeletionStd = (1.30256068643, 1.64860382968, 1.94388833634)

SRMeans = (0.856632, 0.906697, 0.995124, 1.010403, 1.031534)
SRStd = (1.114944, 1.194131, 1.283704, 1.245234, 1.317081)
SRMeansS = (0.96007799999999988,0.96007799999999988,0.96007799999999988)
SRStdS = (1.2310188,1.2310188,1.2310188)

plt1 = plot_bars_FORMATION_STRONG_REL(formationNodeletionMeans, formationNodeletionStd,\
formationDeletionMeans, formationDeletionStd, SRMeansS, SRStdS)

def plot_bars_DELETION_STRONG_REL(PersistingMeans, PersistingStd, Means, Std, PERSreal, PERSstd):

	ind = np.arange(N)  # the x locations for the groups
	#width = 0.3       # the width of the bars

	#ax = plt.subplot(321)
	ax = plt.subplot2grid((1,2),(0, 1))
	#rects1 = ax.bar(ind-0.2, PersistingMeans, width, color='c', yerr=PersistingStd, align='center')
	#rects2 = ax.bar(ind+0.2, Means, width, color='cyan', yerr=Std, align='center')

	rects1 = ax.bar(ind-width, PersistingMeans, width, color='c', \
		align='center', yerr=PersistingStd, linewidth=0,\
		error_kw=dict(ecolor='gray', lw=1.5, capsize=2.7, capthick=1))

	rects2 = ax.bar(ind, Means, width, color='cyan', \
		yerr=Std, align='center', linewidth=0,\
		error_kw=dict(ecolor='gray', lw=1.5, capsize=2.7, capthick=1))

	rects3 = ax.bar(ind+width, PERSreal, width, color='r',\
		yerr=PERSstd, align='center',linewidth=0,\
		error_kw=dict(ecolor='gray', lw=1.5, capsize=2.7, capthick=1))

	ax.legend((rects1[0], rects2[0], rects3[0]), \
		('Persisting decommissioned', \
			'Non-persisting decommissioned', 'Persisting average'),\
		loc='best',frameon=False)

	# add some text for labels, title and axes ticks
	#ax.set_title('Relative status (strong contacts)')
	ax.set_xticks(ind )
	ax.set_xticklabels(('Before', 'At decommission', 'After'))

	ax.set_ylim([-0.5, 5])
	ax.set_yticks((0,5))

	def autolabel(rects):
	    # attach some text labels
	    for rect in rects:
	        height = rect.get_height()
	        ax.text(rect.get_x() + rect.get_width()/2., 1.05*height,
	                '%.2f' % float(height),
	                ha='center', va='bottom')

	autolabel(rects1)
	autolabel(rects2)
	autolabel(rects3)

	return plt

##########################################################################
# NON PERSISTING LINKS
# STRONG contacts REL
#deletionFormationMeans = (1.35860783095, 1.40335612181, 1.38222498446)
#deletionFormationStd = (1.39698763227, 1.515042018, 1.6001731639)

deletionFormationMeans = (1.21614009307, 1.58645603723, 1.613397012)
deletionFormationStd = (1.39228801763, 1.73298601092, 1.84822380219)

# PERSISTING LINKS
#deletionNoformationMeans = (1.16101995042, 1.52591193484, 1.54066816196)
#deletionNoformationStd = (1.36105887603, 1.69996084625, 1.80123581372)

deletionNoformationMeans = (1.09195402299, 1.16457680251, 1.09717868339)
deletionNoformationStd = (1.25857893939, 1.33146910699, 1.31900439894)

SRMeans = (0.856632, 0.906697, 0.995124, 1.010403, 1.031534)
SRStd = (1.114944, 1.194131, 1.283704, 1.245234, 1.317081)
SRMeansS = (0.96007799999999988,0.96007799999999988,0.96007799999999988)
SRStdS = (1.2310188,1.2310188,1.2310188)

plt1 = plot_bars_DELETION_STRONG_REL(deletionNoformationMeans, deletionNoformationStd,\
	deletionFormationMeans, deletionFormationStd, SRMeansS, SRStdS)

##########################################################################

plt.tight_layout()
fig = plt.gcf()
fig.set_size_inches(12.4,4.5)
plt.tight_layout()
#plt.figtext(0.20, 0.49, 'Relative status of the pair: weak contacts')
#plt.figtext(0.27, 0.973, 'Relative status of the pair: strong contacts')
fig.suptitle('Relative status (strong contacts)', verticalalignment='center', horizontalalignment='center', size = 16)
#fig.suptitle('Sum including weak contacts', verticalalignment='center', y=0.5, horizontalalignment='center', size = 16)

plt.savefig("/home/sscepano/Projects7s/Twitter-workspace/DATA/General/explain_FORMATION_DELETION_REL.eps", dpi=710)


# -*- coding: utf-8 -*-
"""
Created on Mon Jun  1 17:07:33 2015

@author: malkaguillot
"""

from ipp_work.utils import survey_simulate, df_weighted_average_grouped
from ipp_work.simulations.ir_marg_rate import varying_survey_simulation
from ipp_work.example.quantiles_of_revimp import make_weighted_deciles_of_variable
import pandas

year = 2009
ind_variables = ['idmen', 'quimen', 'idfoy', 'salaire_imposable', 'salaire_net']
foy_variables = ['irpp', 'decile_rfr', 'weight_foyers', 'idfoy_original', 'rfr']
used_as_input_variables = ['salaire_imposable', 'cho', 'rst', 'age_en_mois', 'smic55']
df_by_entity_key_plural, simulation = survey_simulate(used_as_input_variables, year, ind_variables,
                                                      foy_variables = foy_variables)
df_individus = df_by_entity_key_plural['individus']
df_foyers = df_by_entity_key_plural['foyers']

tax_rates = varying_survey_simulation(year = 2009, increment = 10, target = 'irpp', varying = 'rni',
                                   used_as_input_variables = used_as_input_variables)
tax_rates = tax_rates[['idfoy_original', 'marginal_rate', 'average_rate']]
df_foyers = pandas.merge(df_foyers, tax_rates, on = 'idfoy_original')

make_weighted_deciles_of_variable(df_foyers, 'rfr', 'weight_foyers', 100)

Wconcat = df_weighted_average_grouped(
    dataframe = df_foyers,
    groupe = 'decile_of_rfr',
    varlist = [
        'marginal_rate', 'average_rate'
        ],
    )
print Wconcat
df_foyers['decile_rfr'].count()
df_foyers['rfr'].describe()
df_foyers['weight_foyers'].describe()


"""This file contains code used in "Think Stats",
by Allen B. Downey, available from greenteapress.com

Copyright 2010 Allen B. Downey
License: GNU GPLv3 http://www.gnu.org/licenses/gpl.html
"""

from __future__ import print_function, division

import nsfg
import nsfg2
import first

import thinkstats2
import thinkplot

import copy
import random
import numpy as np
import matplotlib.pyplot as pyplot

class CoinTest(thinkstats2.HypothesisTest):
    """Tests the hypothesis that a coin is fair."""

    def TestStatistic(self, data):
        """Computes the test statistic.

        data: data in whatever form is relevant
        """
        heads, tails = data
        test_stat = abs(heads - tails)
        return test_stat

    def RunModel(self):
        """Run the model of the null hypothesis.

        returns: simulated data
        """
        heads, tails = self.data
        n = heads + tails
        sample = [random.choice('HT') for _ in range(n)]
        hist = thinkstats2.Hist(sample)
        data = hist['H'], hist['T']
        return data

class DiffMeansPermute(thinkstats2.HypothesisTest):
    """Tests a difference in means by permutation."""

    def TestStatistic(self, data):
        """Computes the test statistic.

        data: data in whatever form is relevant
        """
        group1, group2 = data
        test_stat = abs(group1.mean() - group2.mean())
        return test_stat

    def MakeModel(self):
        """Build a model of the null hypothesis.
        """
        group1, group2 = self.data
        self.n, self.m = len(group1), len(group2)
        self.pool = np.hstack((group1, group2))

    def RunModel(self):
        """Run the model of the null hypothesis.

        returns: simulated data
        """
        np.random.shuffle(self.pool)
        data = self.pool[:self.n], self.pool[self.n:]
        return data

class DiffMeansOneSided(DiffMeansPermute):
    """Tests a one-sided difference in means by permutation."""

    def TestStatistic(self, data):
        """Computes the test statistic.

        data: data in whatever form is relevant
        """
        group1, group2 = data
        test_stat = group1.mean() - group2.mean()
        return test_stat

class DiffStdPermute(DiffMeansPermute):
    """Tests a one-sided difference in standard deviation by permutation."""

    def TestStatistic(self, data):
        """Computes the test statistic.

        data: data in whatever form is relevant
        """
        group1, group2 = data
        test_stat = group1.std() - group2.std()
        return test_stat

class CorrelationPermute(thinkstats2.HypothesisTest):
    """Tests correlations by permutation."""

    def TestStatistic(self, data):
        """Computes the test statistic.

        data: tuple of xs and ys
        """
        xs, ys = data
        test_stat = abs(thinkstats2.Corr(xs, ys))
        return test_stat

    def RunModel(self):
        """Run the model of the null hypothesis.

        returns: simulated data
        """
        xs, ys = self.data
        xs = np.random.permutation(xs)
        return xs, ys

class DiceTest(thinkstats2.HypothesisTest):
    """Tests whether a six-sided die is fair."""

    def TestStatistic(self, data):
        """Computes the test statistic.

        data: list of frequencies
        """
        observed = data
        n = sum(observed)
        expected = np.ones(6) * n / 6
        test_stat = sum(abs(observed - expected))
        return test_stat

    def RunModel(self):
        """Run the model of the null hypothesis.

        returns: simulated data
        """
        n = sum(self.data)
        values = [1,2,3,4,5,6]
        rolls = np.random.choice(values, n, replace=True)
        hist = thinkstats2.Hist(rolls)
        freqs = hist.Freqs(values)
        return freqs

class DiceChiTest(DiceTest):
    """Tests a six-sided die using a chi-squared statistic."""

    def TestStatistic(self, data):
        """Computes the test statistic.

        data: list of frequencies
        """
        observed = data
        n = sum(observed)
        expected = np.ones(6) * n / 6
        test_stat = sum((observed - expected)**2 / expected)
        return test_stat

class PregLengthTest(thinkstats2.HypothesisTest):
    """Tests difference in pregnancy length using a chi-squared statistic."""

    def TestStatistic(self, data):
        """Computes the test statistic.

        data: pair of lists of pregnancy lengths
        """
        firsts, others = data
        stat = self.ChiSquared(firsts) + self.ChiSquared(others)
        return stat

    def ChiSquared(self, lengths):
        """Computes the chi-squared statistic.

        lengths: sequence of lengths

        returns: float
        """
        hist = thinkstats2.Hist(lengths)
        observed = np.array(hist.Freqs(self.values))
        expected = self.expected_probs * len(lengths)
        stat = sum((observed - expected)**2 / expected)
        return stat

    def MakeModel(self):
        """Build a model of the null hypothesis.
        """
        firsts, others = self.data
        self.n = len(firsts)
        self.pool = np.hstack((firsts, others))

        pmf = thinkstats2.Pmf(self.pool)
        self.values = range(35, 44)
        self.expected_probs = np.array(pmf.Probs(self.values))

    def RunModel(self):
        """Run the model of the null hypothesis.

        returns: simulated data
        """
        np.random.shuffle(self.pool)
        data = self.pool[:self.n], self.pool[self.n:]
        return data

def RunDiceTest():
    """Tests whether a die is fair.
    """
    data = [8, 9, 19, 5, 8, 11]
    dt = DiceTest(data)
    print('dice test', dt.PValue(iters=10000))
    dt = DiceChiTest(data)
    print('dice chi test', dt.PValue(iters=10000))

def FalseNegRate(data, num_runs=1000):
    """Computes the chance of a false negative based on resampling.

    data: pair of sequences
    num_runs: how many experiments to simulate

    returns: float false negative rate
    """
    group1, group2 = data
    count = 0

    for i in range(num_runs):
        sample1 = thinkstats2.Resample(group1)
        sample2 = thinkstats2.Resample(group2)
        ht = DiffMeansPermute((sample1, sample2))
        p_value = ht.PValue(iters=101)
        if p_value > 0.05:
            count += 1

    return count / num_runs

def PrintTest(p_value, ht):
    """Prints results from a hypothesis test.

    p_value: float
    ht: HypothesisTest
    """
    print('p-value =', p_value)
    print('actual =', ht.actual)
    print('ts max =', ht.MaxTestStat())

def RunTests(data, iters=1000):
    """Runs several tests on the given data.

    data: pair of sequences
    iters: number of iterations to run
    """

    # test the difference in means
    ht = DiffMeansPermute(data)
    p_value = ht.PValue(iters=iters)
    print('\nmeans permute two-sided')
    PrintTest(p_value, ht)

    ht.PlotCdf()
    thinkplot.Save(root='hypothesis1',
                   title='Permutation test',
                   xlabel='difference in means (weeks)',
                   ylabel='CDF',
                   legend=False)

    # test the difference in means one-sided
    ht = DiffMeansOneSided(data)
    p_value = ht.PValue(iters=iters)
    print('\nmeans permute one-sided')
    PrintTest(p_value, ht)

    # test the difference in std
    ht = DiffStdPermute(data)
    p_value = ht.PValue(iters=iters)
    print('\nstd permute one-sided')
    PrintTest(p_value, ht)

def ReplicateTests():
    """Replicates tests with the new NSFG data."""

    live, firsts, others = nsfg2.MakeFrames()

    # compare pregnancy lengths
    print('\nprglngth2')
    data = firsts.prglngth.values, others.prglngth.values
    ht = DiffMeansPermute(data)
    p_value = ht.PValue(iters=1000)
    print('means permute two-sided')
    PrintTest(p_value, ht)

    print('\nbirth weight 2')
    data = (firsts.totalwgt_lb.dropna().values,
            others.totalwgt_lb.dropna().values)
    ht = DiffMeansPermute(data)
    p_value = ht.PValue(iters=1000)
    print('means permute two-sided')
    PrintTest(p_value, ht)

    # test correlation
    live2 = live.dropna(subset=['agepreg', 'totalwgt_lb'])
    data = live2.agepreg.values, live2.totalwgt_lb.values
    ht = CorrelationPermute(data)
    p_value = ht.PValue()
    print('\nage weight correlation 2')
    PrintTest(p_value, ht)

    # compare pregnancy lengths (chi-squared)
    data = firsts.prglngth.values, others.prglngth.values
    ht = PregLengthTest(data)
    p_value = ht.PValue()
    print('\npregnancy length chi-squared 2')
    PrintTest(p_value, ht)

def main():
    thinkstats2.RandomSeed(17)

    # run the coin test
    ct = CoinTest((140, 110))
    pvalue = ct.PValue()
    print('coin test p-value', pvalue)

    # compare pregnancy lengths
    print('\nprglngth')
    live, firsts, others = first.MakeFrames()
    data = firsts.prglngth.values, others.prglngth.values
    RunTests(data)

    # compare birth weights
    print('\nbirth weight')
    data = (firsts.totalwgt_lb.dropna().values,
            others.totalwgt_lb.dropna().values)
    ht = DiffMeansPermute(data)
    p_value = ht.PValue(iters=1000)
    print('means permute two-sided')
    PrintTest(p_value, ht)

    # test correlation
    live2 = live.dropna(subset=['agepreg', 'totalwgt_lb'])
    data = live2.agepreg.values, live2.totalwgt_lb.values
    ht = CorrelationPermute(data)
    p_value = ht.PValue()
    print('\nage weight correlation')
    print('n=', len(live2))
    PrintTest(p_value, ht)

    # run the dice test
    RunDiceTest()

    # compare pregnancy lengths (chi-squared)
    data = firsts.prglngth.values, others.prglngth.values
    ht = PregLengthTest(data)
    p_value = ht.PValue()
    print('\npregnancy length chi-squared')
    PrintTest(p_value, ht)

    # compute the false negative rate for difference in pregnancy length
    data = firsts.prglngth.values, others.prglngth.values
    neg_rate = FalseNegRate(data)
    print('false neg rate', neg_rate)

    # run the tests with new nsfg data
    ReplicateTests()

if __name__ == "__main__":
    main()


######################################################
# _untuned_modeling.py
# author: Gert Jacobusse, gert.jacobusse@rogatio.nl
# licence: FreeBSD

"""
Copyright (c) 2015, Gert Jacobusse
All rights reserved.

Redistribution and use in source and binary forms, with or without modification,
 are permitted provided that the following conditions are met:

1. Redistributions of source code must retain the above copyright notice, this
 list of conditions and the following disclaimer.

2. Redistributions in binary form must reproduce the above copyright notice,
 this list of conditions and the following disclaimer in the documentation
 and/or other materials provided with the distribution.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
 ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
 INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
 BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
 LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
 OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
 OF THE POSSIBILITY OF SUCH DAMAGE.
"""

#first run feature_extraction.py
#then run this file from the same directory

######################################################
# import dependencies

import csv
import numpy as np
from sklearn.cross_validation import KFold
from sklearn.ensemble import GradientBoostingClassifier,ExtraTreesClassifier
from sklearn.metrics import log_loss

######################################################
# list ids and labels

trainids=[]
labels=[]
with open('trainLabels.csv','r') as f:
    r=csv.reader(f)
    r.next() # skip header
    for row in r:
        trainids.append(row[0])
        labels.append(float(row[1]))

testids=[]
with open('sampleSubmission.csv','r') as f:
    r=csv.reader(f)
    r.next()
    for row in r:
        testids.append(row[0])

######################################################
# general functions

def readdata(fname,header=True,selectedcols=None):
    with open(fname,'r') as f:
        r=csv.reader(f)
        names = r.next() if header else None
        if selectedcols:
            assert header==True
            data = [[float(e) for i,e in enumerate(row) if names[i] in selectedcols] for row in r]
            names = [name for name in names if name in selectedcols]
        else:
            data = [[float(e) for e in row] for row in r]
    return data,names

def writedata(data,fname,header=None):
    with open(fname,'w') as f:
        w=csv.writer(f)
        if header:
            w.writerow(header)
        for row in data:
            w.writerow(row)

######################################################
# cross validation

"""
function docv
input: classifier, kfolds object, features, labels, number of data rows
output: holdout-set-predictions for all rows
* run cross validation
"""
def docv(clf,kf,x,y,nrow,nlab=9):
    pred = np.zeros((nrow,nlab))
    for trainidx, testidx in kf:
        clf.fit(x[trainidx],y[trainidx])
        pred[testidx] = clf.predict_proba(x[testidx])
    return pred

"""
function runcv
input: name of train/ test file, classifier 1 and 2 to be used
output: writes holdout-set-predictions for all rows to file
* run cross validation by calling docv for both classifiers, combine and save results
"""
def runcv(filename,c1,c2):
    y=np.array(labels)
    nrow=len(y)
    x,_=readdata('train_%s'%filename)
    x=np.array(x)
    kf = KFold(nrow,10,shuffle=True)
    p1=docv(c1,kf,x,y,nrow)
    p2=docv(c2,kf,x,y,nrow)
    pcombi=0.667*p1+0.333*p2
    print '%.4f %.4f %.4f'%(log_loss(y,p1),log_loss(y,p2),log_loss(y,pcombi))
    with open('pred_%s'%filename,'w') as f:
        w=csv.writer(f)
        for row in pcombi:
            w.writerow(row)

######################################################
# submit and print feature importance

"""
function writesubm
input: name of train/ test file, classifier 1 and 2 to be used
output: writes testset predictions to file
* train classifiers using all traindata, create testset predictions, combine and save results
"""
def writesubm(filename,c1,c2):
    xtrain,names=readdata('train_%s'%filename)
    xtest,_=readdata('test_%s'%filename)
    c1.fit(xtrain,labels)
    c2.fit(xtrain,labels)
    p1=c1.predict_proba(xtest)
    p2=c2.predict_proba(xtest)
    p=0.667*p1+0.333*p2
    with open('subm_%s'%filename,'w') as f:
        w=csv.writer(f)
        w.writerow(['Id']+['Prediction%d'%num for num in xrange(1,10)])
        for inum,i in enumerate(testids):
            w.writerow([i]+list(p[inum]))

######################################################
# go

if __name__ == '__main__':
    gbm=GradientBoostingClassifier(
                                n_estimators=400, max_features=5)
    xtr=ExtraTreesClassifier(
                                n_estimators=400,max_features=None,
                                min_samples_leaf=2,min_samples_split=3,
                                n_jobs=7)
    for filename in [
            '45c.csv',
            ]:
        print filename
        runcv(filename,gbm,xtr)
        writesubm(filename,gbm,xtr)
        print ''
"""

45c.csv
0.0117 0.0168 0.0101

public LB: 0.008071379
private LB: 0.007615772

"""


"""
=========================================
SVM: Maximum margin separating hyperplane
=========================================

Plot the maximum margin separating hyperplane within a two-class
separable dataset using a Support Vector Machines classifier with
linear kernel.
"""
print(__doc__)

import numpy as np
import matplotlib.pyplot as plt
from sklearn import svm

# we create 40 separable points
np.random.seed(0)
X = np.r_[np.random.randn(20, 2) - [2, 2], np.random.randn(20, 2) + [2, 2]]
Y = [0] * 20 + [1] * 20

# fit the model
clf = svm.SVC(kernel='linear')
clf.fit(X, Y)

# get the separating hyperplane
w = clf.coef_[0]
a = -w[0] / w[1]
xx = np.linspace(-5, 5)
yy = a * xx - (clf.intercept_[0]) / w[1]

# plot the parallels to the separating hyperplane that pass through the
# support vectors
b = clf.support_vectors_[0]
yy_down = a * xx + (b[1] - a * b[0])
b = clf.support_vectors_[-1]
yy_up = a * xx + (b[1] - a * b[0])

# plot the line, the points, and the nearest vectors to the plane
plt.plot(xx, yy, 'k-')
plt.plot(xx, yy_down, 'k--')
plt.plot(xx, yy_up, 'k--')

plt.scatter(clf.support_vectors_[:, 0], clf.support_vectors_[:, 1],
            s=80, facecolors='none')
plt.scatter(X[:, 0], X[:, 1], c=Y, cmap=plt.cm.Paired)

plt.axis('tight')
plt.show()