#!/usr/bin/env python
# -*- coding: utf-8 -*-
#https://github.com/paramaggarwal/CarND-Advanced-Lane-Lines/blob/master/Notebook.ipynb
from __future__ import print_function
from __future__ import division
import sys
import traceback
import rospy
import numpy as np
import cv2
import pickle
import glob
import time
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
from std_msgs.msg import String
from sensor_msgs.msg import Image
from cv_bridge import CvBridge, CvBridgeError

class camera_calibarion(object):
    def __init__(self):

      """ROS Subscriptions """
      self.image_pub = rospy.Publisher("/camera_calibation/image_corrected",Image, queue_size=10)
      self.image_sub = rospy.Subscriber("/cam/camera_/image_raw",Image,self.cvt_image)

      """ Variables """
      self.bridge = CvBridge()
      self.latestImage = None
      self.outputImage = None
      self.process = False
      self.calibrated = False
      self.correctedImage = None
      self.mtx = None
      self.dist = None

    def cvt_image(self,data):
      try:
        self.latestImage = self.bridge.imgmsg_to_cv2(data, "bgr8")
      except CvBridgeError as e:
        print(e)
      if self.process != True:
          self.process = True

    def camera_cal(self, image):

        # termination criteria
        criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 30, 0.001)

        nx = 8
        ny = 6

        dst = np.copy(image)

        # prepare object points, like (0,0,0), (1,0,0), (2,0,0) ....,(6,5,0)
        objp = np.zeros((ny * nx, 3), np.float32)
        objp[:,:2] = np.mgrid[0:nx, 0:ny].T.reshape(-1,2)

        # Arrays to store object points and image points from all the images.
        objpoints = [] # 3d points in real world space
        imgpoints = [] # 2d points in image plane.

        # Search for chessboard corners
        grey = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)

        #ret_thresh,  mask = cv2.threshold(grey, 30, 255, cv2.THRESH_BINARY)

        ret, corners = cv2.findChessboardCorners(image, (nx, ny), None)  #flags=(cv2.cv.CV_CALIB_CB_ADAPTIVE_THRESH + cv2.cv.CV_CALIB_CB_FILTER_QUADS))

        # If found, add object points, image points
        if ret == True:
            objpoints.append(objp)
            cv2.cornerSubPix(grey,corners, (11,11), (-1,-1), criteria)
            imgpoints.append(corners)
            self.calibrated = True
            print ("FOUND!")

            #Draw and display the corners
            cv2.drawChessboardCorners(image, (nx, ny), corners, ret)

            # Do camera calibration given object points and image points
            ret, self.mtx, self.dist, rvecs, tvecs = cv2.calibrateCamera(objpoints, imgpoints, grey.shape[::-1], None, None)

            # Save the camera calibration result for later use (we won't worry about rvecs / tvecs)
            dist_pickle = {}
            dist_pickle["mtx"] = self.mtx
            dist_pickle["dist"] = self.dist
            dist_pickle['objpoints'] = objpoints
            dist_pickle['imgpoints'] = imgpoints
            pickle.dump( dist_pickle, open( "/home/wil/ros/catkin_ws/src/av_sim/computer_vision/camera_calibration/data/camera_cal_pickle.p", "wb" ) )

         #else:
             #print("Searching...")

        return image

    def drawQuad(self, image, points, color=[255, 0, 0], thickness=4):
        p1, p2, p3, p4 = points
        cv2.line(image, tuple(p1), tuple(p2), color, thickness)
        cv2.line(image, tuple(p2), tuple(p3), color, thickness)
        cv2.line(image, tuple(p3), tuple(p4), color, thickness)
        cv2.line(image, tuple(p4), tuple(p1), color, thickness)

    def perspective_transform(self,  image, debug=True, size_top=70, size_bottom=370):
        height, width = image.shape[0:2]
        output_size = height/2

        #src = np.float32([[(width/2) - size_top, height*0.65], [(width/2) + size_top, height*0.65], [(width/2) + size_bottom, height-50], [(width/2) - size_bottom, height-50]])
        src = np.float32([[512, 450], [675, 454], [707, 560], [347, 568]])
        dst = np.float32([[347, height], [707, height], [707, 0], [347, 0]])
        #dst = np.float32([[(width/2) - output_size, (height/2) - output_size], [(width/2) + output_size, (height/2) - output_size], [(width/2) + output_size, (height/2) + output_size], [(width/2) - output_size, (height/2) + output_size]])

        M = cv2.getPerspectiveTransform(src, dst)
        print(M)
        warped = cv2.warpPerspective(image, M, (width, height), flags=cv2.INTER_LINEAR)

        if debug:
            self.drawQuad(image, src, [255, 0, 0])
            self.drawQuad(image, dst, [255, 255, 0])
            plt.imshow(image)
            plt.show()

        return warped

    def undistort_image(self, image):

        return cv2.undistort(image, self.mtx, self.dist, None, self.mtx)

    def run(self):

         while True:

             # Only run loop if we have an image
             if self.process:

                 filename = "/home/wil/ros/catkin_ws/src/av_sim/computer_vision/camera_calibration/data/check_test.png"
                 image = cv2.imread(filename, flags=cv2.IMREAD_COLOR)

                 if self.calibrated is not True:
                     #print("Calibrating...")
                     cornersImage = self.camera_cal(image)
                     cvImage = cornersImage

                 else:
                     correctedImage = self.undistort_image(self.latestImage)	# Distortion Correction Function
                     transformImage = self.perspective_transform(self.latestImage)
                     cvImage = transformImage

                 # Publish Undistorted Image
                 try:
                     imgmsg = self.bridge.cv2_to_imgmsg(cvImage, "bgr8") #"mono8" "bgr8"
                     self.image_pub.publish(imgmsg)
                 except CvBridgeError as e:
                     print(e)

def main(args):

  rospy.init_node('camera_calibarion', anonymous=True)

  cc = camera_calibarion()

  cc.run()

  try:
    rospy.spin()
  except KeyboardInterrupt:
    print("Shutting down")
  cv2.destroyAllWindows()

if __name__ == '__main__':
    main(sys.argv)


""" Principal Component Analysis
"""

# Author: Alexandre Gramfort <alexandre.gramfort@inria.fr>
#         Olivier Grisel <olivier.grisel@ensta.org>
#         Mathieu Blondel <mathieu@mblondel.org>
#         Denis A. Engemann <d.engemann@fz-juelich.de>
#         Michael Eickenberg <michael.eickenberg@inria.fr>
#
# License: BSD 3 clause

from math import log, sqrt

import numpy as np
from scipy import linalg
from scipy.special import gammaln

from ..base import BaseEstimator, TransformerMixin
from ..utils import check_random_state, as_float_array
from ..utils import check_array
from ..utils.extmath import fast_dot, fast_logdet, randomized_svd
from ..utils.validation import check_is_fitted

def _assess_dimension_(spectrum, rank, n_samples, n_features):
    """Compute the likelihood of a rank ``rank`` dataset

    The dataset is assumed to be embedded in gaussian noise of shape(n,
    dimf) having spectrum ``spectrum``.

    Parameters
    ----------
    spectrum: array of shape (n)
        Data spectrum.
    rank: int
        Tested rank value.
    n_samples: int
        Number of samples.
    n_features: int
        Number of features.

    Returns
    -------
    ll: float,
        The log-likelihood

    Notes
    -----
    This implements the method of `Thomas P. Minka:
    Automatic Choice of Dimensionality for PCA. NIPS 2000: 598-604`
    """
    if rank > len(spectrum):
        raise ValueError("The tested rank cannot exceed the rank of the"
                         " dataset")

    pu = -rank * log(2.)
    for i in range(rank):
        pu += (gammaln((n_features - i) / 2.)
               - log(np.pi) * (n_features - i) / 2.)

    pl = np.sum(np.log(spectrum[:rank]))
    pl = -pl * n_samples / 2.

    if rank == n_features:
        pv = 0
        v = 1
    else:
        v = np.sum(spectrum[rank:]) / (n_features - rank)
        pv = -np.log(v) * n_samples * (n_features - rank) / 2.

    m = n_features * rank - rank * (rank + 1.) / 2.
    pp = log(2. * np.pi) * (m + rank + 1.) / 2.

    pa = 0.
    spectrum_ = spectrum.copy()
    spectrum_[rank:n_features] = v
    for i in range(rank):
        for j in range(i + 1, len(spectrum)):
            pa += log((spectrum[i] - spectrum[j]) *
                      (1. / spectrum_[j] - 1. / spectrum_[i])) + log(n_samples)

    ll = pu + pl + pv + pp - pa / 2. - rank * log(n_samples) / 2.

    return ll

def _infer_dimension_(spectrum, n_samples, n_features):
    """Infers the dimension of a dataset of shape (n_samples, n_features)

    The dataset is described by its spectrum `spectrum`.
    """
    n_spectrum = len(spectrum)
    ll = np.empty(n_spectrum)
    for rank in range(n_spectrum):
        ll[rank] = _assess_dimension_(spectrum, rank, n_samples, n_features)
    return ll.argmax()

class PCA(BaseEstimator, TransformerMixin):
    """Principal component analysis (PCA)

    Linear dimensionality reduction using Singular Value Decomposition of the
    data and keeping only the most significant singular vectors to project the
    data to a lower dimensional space.

    This implementation uses the scipy.linalg implementation of the singular
    value decomposition. It only works for dense arrays and is not scalable to
    large dimensional data.

    The time complexity of this implementation is ``O(n ** 3)`` assuming
    n ~ n_samples ~ n_features.

    Read more in the :ref:`User Guide <PCA>`.

    Parameters
    ----------
    n_components : int, None or string
        Number of components to keep.
        if n_components is not set all components are kept::

            n_components == min(n_samples, n_features)

        if n_components == 'mle', Minka\'s MLE is used to guess the dimension
        if ``0 < n_components < 1``, select the number of components such that
        the amount of variance that needs to be explained is greater than the
        percentage specified by n_components

    copy : bool
        If False, data passed to fit are overwritten and running
        fit(X).transform(X) will not yield the expected results,
        use fit_transform(X) instead.

    whiten : bool, optional
        When True (False by default) the `components_` vectors are divided
        by n_samples times singular values to ensure uncorrelated outputs
        with unit component-wise variances.

        Whitening will remove some information from the transformed signal
        (the relative variance scales of the components) but can sometime
        improve the predictive accuracy of the downstream estimators by
        making there data respect some hard-wired assumptions.

    Attributes
    ----------
    components_ : array, [n_components, n_features]
        Principal axes in feature space, representing the directions of
        maximum variance in the data.

    explained_variance_ratio_ : array, [n_components]
        Percentage of variance explained by each of the selected components.
        If ``n_components`` is not set then all components are stored and the
        sum of explained variances is equal to 1.0

    mean_ : array, [n_features]
        Per-feature empirical mean, estimated from the training set.

    n_components_ : int
        The estimated number of components. Relevant when n_components is set
        to 'mle' or a number between 0 and 1 to select using explained
        variance.

    noise_variance_ : float
        The estimated noise covariance following the Probabilistic PCA model
        from Tipping and Bishop 1999. See "Pattern Recognition and
        Machine Learning" by C. Bishop, 12.2.1 p. 574 or
        http://www.miketipping.com/papers/met-mppca.pdf. It is required to
        computed the estimated data covariance and score samples.

    Notes
    -----
    For n_components='mle', this class uses the method of `Thomas P. Minka:
    Automatic Choice of Dimensionality for PCA. NIPS 2000: 598-604`

    Implements the probabilistic PCA model from:
    M. Tipping and C. Bishop, Probabilistic Principal Component Analysis,
    Journal of the Royal Statistical Society, Series B, 61, Part 3, pp. 611-622
    via the score and score_samples methods.
    See http://www.miketipping.com/papers/met-mppca.pdf

    Due to implementation subtleties of the Singular Value Decomposition (SVD),
    which is used in this implementation, running fit twice on the same matrix
    can lead to principal components with signs flipped (change in direction).
    For this reason, it is important to always use the same estimator object to
    transform data in a consistent fashion.

    Examples
    --------

    >>> import numpy as np
    >>> from sklearn.decomposition import PCA
    >>> X = np.array([[-1, -1], [-2, -1], [-3, -2], [1, 1], [2, 1], [3, 2]])
    >>> pca = PCA(n_components=2)
    >>> pca.fit(X)
    PCA(copy=True, n_components=2, whiten=False)
    >>> print(pca.explained_variance_ratio_) # doctest: +ELLIPSIS
    [ 0.99244...  0.00755...]

    See also
    --------
    RandomizedPCA
    KernelPCA
    SparsePCA
    TruncatedSVD
    """
    def __init__(self, n_components=None, copy=True, whiten=False):
        self.n_components = n_components
        self.copy = copy
        self.whiten = whiten

    def fit(self, X, y=None):
        """Fit the model with X.

        Parameters
        ----------
        X: array-like, shape (n_samples, n_features)
            Training data, where n_samples in the number of samples
            and n_features is the number of features.

        Returns
        -------
        self : object
            Returns the instance itself.
        """
        self._fit(X)
        return self

    def fit_transform(self, X, y=None):
        """Fit the model with X and apply the dimensionality reduction on X.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            Training data, where n_samples is the number of samples
            and n_features is the number of features.

        Returns
        -------
        X_new : array-like, shape (n_samples, n_components)

        """
        U, S, V = self._fit(X)
        U = U[:, :self.n_components_]

        if self.whiten:
            # X_new = X * V / S * sqrt(n_samples) = U * sqrt(n_samples)
            U *= sqrt(X.shape[0])
        else:
            # X_new = X * V = U * S * V^T * V = U * S
            U *= S[:self.n_components_]

        return U

    def _fit(self, X):
        """Fit the model on X

        Parameters
        ----------
        X: array-like, shape (n_samples, n_features)
            Training vector, where n_samples in the number of samples and
            n_features is the number of features.

        Returns
        -------
        U, s, V : ndarrays
            The SVD of the input data, copied and centered when
            requested.
        """
        X = check_array(X)
        n_samples, n_features = X.shape
        X = as_float_array(X, copy=self.copy)
        # Center data
        self.mean_ = np.mean(X, axis=0)
        X -= self.mean_
        U, S, V = linalg.svd(X, full_matrices=False)
        explained_variance_ = (S ** 2) / n_samples
        explained_variance_ratio_ = (explained_variance_ /
                                     explained_variance_.sum())

        components_ = V

        n_components = self.n_components
        if n_components is None:
            n_components = n_features
        elif n_components == 'mle':
            if n_samples < n_features:
                raise ValueError("n_components='mle' is only supported "
                                 "if n_samples >= n_features")

            n_components = _infer_dimension_(explained_variance_,
                                             n_samples, n_features)
        elif not 0 <= n_components <= n_features:
            raise ValueError("n_components=%r invalid for n_features=%d"
                             % (n_components, n_features))

        if 0 < n_components < 1.0:
            # number of components for which the cumulated explained variance
            # percentage is superior to the desired threshold
            ratio_cumsum = explained_variance_ratio_.cumsum()
            n_components = np.sum(ratio_cumsum < n_components) + 1

        # Compute noise covariance using Probabilistic PCA model
        # The sigma2 maximum likelihood (cf. eq. 12.46)
        if n_components < n_features:
            self.noise_variance_ = explained_variance_[n_components:].mean()
        else:
            self.noise_variance_ = 0.

        # store n_samples to revert whitening when getting covariance
        self.n_samples_ = n_samples

        self.components_ = components_[:n_components]
        self.explained_variance_ = explained_variance_[:n_components]
        explained_variance_ratio_ = explained_variance_ratio_[:n_components]
        self.explained_variance_ratio_ = explained_variance_ratio_
        self.n_components_ = n_components

        return (U, S, V)

    def get_covariance(self):
        """Compute data covariance with the generative model.

        ``cov = components_.T * S**2 * components_ + sigma2 * eye(n_features)``
        where  S**2 contains the explained variances.

        Returns
        -------
        cov : array, shape=(n_features, n_features)
            Estimated covariance of data.
        """
        components_ = self.components_
        exp_var = self.explained_variance_
        if self.whiten:
            components_ = components_ * np.sqrt(exp_var[:, np.newaxis])
        exp_var_diff = np.maximum(exp_var - self.noise_variance_, 0.)
        cov = np.dot(components_.T * exp_var_diff, components_)
        cov.flat[::len(cov) + 1] += self.noise_variance_  # modify diag inplace
        return cov

    def get_precision(self):
        """Compute data precision matrix with the generative model.

        Equals the inverse of the covariance but computed with
        the matrix inversion lemma for efficiency.

        Returns
        -------
        precision : array, shape=(n_features, n_features)
            Estimated precision of data.
        """
        n_features = self.components_.shape[1]

        # handle corner cases first
        if self.n_components_ == 0:
            return np.eye(n_features) / self.noise_variance_
        if self.n_components_ == n_features:
            return linalg.inv(self.get_covariance())

        # Get precision using matrix inversion lemma
        components_ = self.components_
        exp_var = self.explained_variance_
        exp_var_diff = np.maximum(exp_var - self.noise_variance_, 0.)
        precision = np.dot(components_, components_.T) / self.noise_variance_
        precision.flat[::len(precision) + 1] += 1. / exp_var_diff
        precision = np.dot(components_.T,
                           np.dot(linalg.inv(precision), components_))
        precision /= -(self.noise_variance_ ** 2)
        precision.flat[::len(precision) + 1] += 1. / self.noise_variance_
        return precision

    def transform(self, X):
        """Apply the dimensionality reduction on X.

        X is projected on the first principal components previous extracted
        from a training set.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            New data, where n_samples is the number of samples
            and n_features is the number of features.

        Returns
        -------
        X_new : array-like, shape (n_samples, n_components)

        """
        check_is_fitted(self, 'mean_')

        X = check_array(X)
        if self.mean_ is not None:
            X = X - self.mean_
        X_transformed = fast_dot(X, self.components_.T)
        if self.whiten:
            X_transformed /= np.sqrt(self.explained_variance_)
        return X_transformed

    def inverse_transform(self, X):
        """Transform data back to its original space, i.e.,
        return an input X_original whose transform would be X

        Parameters
        ----------
        X : array-like, shape (n_samples, n_components)
            New data, where n_samples is the number of samples
            and n_components is the number of components.

        Returns
        -------
        X_original array-like, shape (n_samples, n_features)
        """
        check_is_fitted(self, 'mean_')

        if self.whiten:
            return fast_dot(
                X,
                np.sqrt(self.explained_variance_[:, np.newaxis]) *
                self.components_) + self.mean_
        else:
            return fast_dot(X, self.components_) + self.mean_

    def score_samples(self, X):
        """Return the log-likelihood of each sample

        See. "Pattern Recognition and Machine Learning"
        by C. Bishop, 12.2.1 p. 574
        or http://www.miketipping.com/papers/met-mppca.pdf

        Parameters
        ----------
        X: array, shape(n_samples, n_features)
            The data.

        Returns
        -------
        ll: array, shape (n_samples,)
            Log-likelihood of each sample under the current model
        """
        check_is_fitted(self, 'mean_')

        X = check_array(X)
        Xr = X - self.mean_
        n_features = X.shape[1]
        log_like = np.zeros(X.shape[0])
        precision = self.get_precision()
        log_like = -.5 * (Xr * (np.dot(Xr, precision))).sum(axis=1)
        log_like -= .5 * (n_features * log(2. * np.pi)
                          - fast_logdet(precision))
        return log_like

    def score(self, X, y=None):
        """Return the average log-likelihood of all samples

        See. "Pattern Recognition and Machine Learning"
        by C. Bishop, 12.2.1 p. 574
        or http://www.miketipping.com/papers/met-mppca.pdf

        Parameters
        ----------
        X: array, shape(n_samples, n_features)
            The data.

        Returns
        -------
        ll: float
            Average log-likelihood of the samples under the current model
        """
        return np.mean(self.score_samples(X))

class RandomizedPCA(BaseEstimator, TransformerMixin):
    """Principal component analysis (PCA) using randomized SVD

    Linear dimensionality reduction using approximated Singular Value
    Decomposition of the data and keeping only the most significant
    singular vectors to project the data to a lower dimensional space.

    Read more in the :ref:`User Guide <RandomizedPCA>`.

    Parameters
    ----------
    n_components : int, optional
        Maximum number of components to keep. When not given or None, this
        is set to n_features (the second dimension of the training data).

    copy : bool
        If False, data passed to fit are overwritten and running
        fit(X).transform(X) will not yield the expected results,
        use fit_transform(X) instead.

    iterated_power : int, optional
        Number of iterations for the power method. 3 by default.

    whiten : bool, optional
        When True (False by default) the `components_` vectors are divided
        by the singular values to ensure uncorrelated outputs with unit
        component-wise variances.

        Whitening will remove some information from the transformed signal
        (the relative variance scales of the components) but can sometime
        improve the predictive accuracy of the downstream estimators by
        making their data respect some hard-wired assumptions.

    random_state : int or RandomState instance or None (default)
        Pseudo Random Number generator seed control. If None, use the
        numpy.random singleton.

    Attributes
    ----------
    components_ : array, [n_components, n_features]
        Components with maximum variance.

    explained_variance_ratio_ : array, [n_components]
        Percentage of variance explained by each of the selected components. \
        k is not set then all components are stored and the sum of explained \
        variances is equal to 1.0

    mean_ : array, [n_features]
        Per-feature empirical mean, estimated from the training set.

    Examples
    --------
    >>> import numpy as np
    >>> from sklearn.decomposition import RandomizedPCA
    >>> X = np.array([[-1, -1], [-2, -1], [-3, -2], [1, 1], [2, 1], [3, 2]])
    >>> pca = RandomizedPCA(n_components=2)
    >>> pca.fit(X)                 # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
    RandomizedPCA(copy=True, iterated_power=3, n_components=2,
           random_state=None, whiten=False)
    >>> print(pca.explained_variance_ratio_) # doctest: +ELLIPSIS
    [ 0.99244...  0.00755...]

    See also
    --------
    PCA
    TruncatedSVD

    References
    ----------

    .. [Halko2009] `Finding structure with randomness: Stochastic algorithms
      for constructing approximate matrix decompositions Halko, et al., 2009
      (arXiv:909)`

    .. [MRT] `A randomized algorithm for the decomposition of matrices
      Per-Gunnar Martinsson, Vladimir Rokhlin and Mark Tygert`

    """

    def __init__(self, n_components=None, copy=True, iterated_power=3,
                 whiten=False, random_state=None):
        self.n_components = n_components
        self.copy = copy
        self.iterated_power = iterated_power
        self.whiten = whiten
        self.random_state = random_state

    def fit(self, X, y=None):
        """Fit the model with X by extracting the first principal components.

        Parameters
        ----------
        X: array-like, shape (n_samples, n_features)
            Training data, where n_samples in the number of samples
            and n_features is the number of features.

        Returns
        -------
        self : object
            Returns the instance itself.
        """
        self._fit(check_array(X))
        return self

    def _fit(self, X):
        """Fit the model to the data X.

        Parameters
        ----------
        X: array-like, shape (n_samples, n_features)
            Training vector, where n_samples in the number of samples and
            n_features is the number of features.

        Returns
        -------
        X : ndarray, shape (n_samples, n_features)
            The input data, copied, centered and whitened when requested.
        """
        random_state = check_random_state(self.random_state)
        X = np.atleast_2d(as_float_array(X, copy=self.copy))

        n_samples = X.shape[0]

        # Center data
        self.mean_ = np.mean(X, axis=0)
        X -= self.mean_
        if self.n_components is None:
            n_components = X.shape[1]
        else:
            n_components = self.n_components

        U, S, V = randomized_svd(X, n_components,
                                 n_iter=self.iterated_power,
                                 random_state=random_state)

        self.explained_variance_ = exp_var = (S ** 2) / n_samples
        full_var = np.var(X, axis=0).sum()
        self.explained_variance_ratio_ = exp_var / full_var

        if self.whiten:
            self.components_ = V / S[:, np.newaxis] * sqrt(n_samples)
        else:
            self.components_ = V

        return X

    def transform(self, X, y=None):
        """Apply dimensionality reduction on X.

        X is projected on the first principal components previous extracted
        from a training set.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            New data, where n_samples in the number of samples
            and n_features is the number of features.

        Returns
        -------
        X_new : array-like, shape (n_samples, n_components)

        """
        check_is_fitted(self, 'mean_')

        X = check_array(X)
        if self.mean_ is not None:
            X = X - self.mean_

        X = fast_dot(X, self.components_.T)
        return X

    def fit_transform(self, X, y=None):
        """Fit the model with X and apply the dimensionality reduction on X.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            New data, where n_samples in the number of samples
            and n_features is the number of features.

        Returns
        -------
        X_new : array-like, shape (n_samples, n_components)

        """
        X = check_array(X)
        X = self._fit(X)
        return fast_dot(X, self.components_.T)

    def inverse_transform(self, X, y=None):
        """Transform data back to its original space.

        Returns an array X_original whose transform would be X.

        Parameters
        ----------
        X : array-like, shape (n_samples, n_components)
            New data, where n_samples in the number of samples
            and n_components is the number of components.

        Returns
        -------
        X_original array-like, shape (n_samples, n_features)

        Notes
        -----
        If whitening is enabled, inverse_transform does not compute the
        exact inverse operation of transform.
        """
        check_is_fitted(self, 'mean_')

        X_original = fast_dot(X, self.components_)
        if self.mean_ is not None:
            X_original = X_original + self.mean_
        return X_original


import pandas as pd
import numpy as np
import re
import json
import os
import warnings
import shutil
from pathlib import Path
import codecs

"""
Helper functions to load the article data. The main method to use
is load_data().
"""

# Caution! Modifying this in code will have no effect since the
# default arguments are populated with this reference at creation
# time, so post-hoc modifications will do nothing.
__data_folder = os.path.join(os.path.split(__file__)[0], '..', 'data')

def clean_string(s):
    """
    Clean all the HTML/Unicode nastiness out of a string.
    Replaces newlines with spaces.
    """

    return s.replace('\r', '').replace('\n', ' ').replace('\xa0', ' ').strip()

def load_articles(data_folder=__data_folder, nrows=None):
    """
    Loads the articles CSV. Can optionally only load the first
    `nrows` number of rows.
    """
    column_names = ['id',
                    'feedname',
                    'url',
                    'orig_html',
                    'title',
                    'bodytext',
                    'relevant',
                    'created',
                    'last_modified',
                    'news_source_id',
                    'author']

    return pd.read_csv(os.path.join(data_folder,
                                    'newsarticles_article.csv'),
                       header=None,
                       names=column_names,
                       nrows=nrows,
                       dtype={'orig_html': str, 'author': str})

def load_taggings(data_folder=__data_folder):
    """Loads the type-of-crime human tagging of the articles."""
    uc_column_names = ['id', 'date', 'relevant',
                       'article_id', 'user_id', 'locations']

    uc = pd.read_csv(os.path.join(data_folder,
                                  'newsarticles_usercoding.csv'),
                     header=None,
                     names=uc_column_names)

    uc.set_index('id', drop=True, inplace=True)

    uc_tags_column_names = ['id', 'usercoding_id', 'category_id']

    uc_tags = pd.read_csv(
        os.path.join(data_folder, 'newsarticles_usercoding_categories.csv'),
        header=None,
        names=uc_tags_column_names
    )
    uc_tags.set_index('usercoding_id', drop=True, inplace=True)

    uc_tags['article_id'] = uc.loc[uc_tags.index, 'article_id']
    return uc_tags

def load_model_categories(data_folder=__data_folder):
    tcr_names = ['id', 'relevance', 'category_id', 'coding_id']
    tc_names = ['id', 'date', 'model_info', 'relevance', 'article_id']

    tcr = pd.read_csv(
        os.path.join(data_folder, 'newsarticles_trainedcategoryrelevance.csv'),
        names=tcr_names
    )
    tc = pd.read_csv(
        os.path.join(data_folder, 'newsarticles_trainedcoding.csv'),
        names=tc_names
    ).set_index('id', drop=True)
    tcr['article_id'] = tc.loc[tcr['coding_id']]['article_id'].values
    return tcr

def load_model_locations(data_folder=__data_folder):
    tl_names = ['id', 'text', 'latitude', 'longitude', 'coding_id']
    tc_names = ['id', 'date', 'model_info', 'relevance', 'article_id']

    tl = pd.read_csv(
        os.path.join(data_folder, 'newsarticles_trainedlocation.csv'),
        names=tl_names
    )
    tc = pd.read_csv(
        os.path.join(data_folder, 'newsarticles_trainedcoding.csv'),
        names=tc_names
    ).set_index('id', drop=True)
    tl['article_id'] = tc.loc[tl['coding_id']]['article_id'].values
    return tl

def load_locations(data_folder=__data_folder):
    """Load the human-extracted locations from the articles."""
    uc_column_names = ['id', 'date', 'relevant',
                       'article_id', 'user_id', 'locations']

    uc = pd.read_csv(os.path.join(data_folder,
                                  'newsarticles_usercoding.csv'),
                     header=None,
                     names=uc_column_names)

    uc['locations'] = uc['locations'].apply(lambda x: json.loads(x))

    return uc

def load_categories(data_folder=__data_folder):
    """Loads the mapping of id to names/abbrevations of categories"""
    column_names = ['id', 'category_name', 'abbreviation', 'created',
                    'active', 'kind']

    return pd.read_csv(os.path.join(data_folder, 'newsarticles_category.csv'),
                       header=None,
                       names=column_names)

def load_data(data_folder=__data_folder, nrows=None):
    """
    Creates a dataframe of the article information and k-hot encodes the tags
    into columns called cat_NUMBER. The k-hot encoding is done assuming that
    the categories are 1-indexed and there are as many categories as the
    maximum value of the numerical cateogry_id column.

    Inputs:
        data_folder:
            A folder containing the data files in CSV format.
        nrows:
            Number of articles to load. Defaults to all, which uses about 4
            GB of memory.
    """

    df = load_articles(data_folder=data_folder, nrows=nrows)
    df['relevant'] = df['relevant'] == 't'
    df.rename(columns={'id': 'article_id'}, inplace=True)
    df.set_index('article_id', drop=True, inplace=True)
    # hopefully this will save some memory/space, can add back if needed
    del(df['orig_html'])

    tags_df = load_taggings(data_folder)
    # will help cacheing
    tags_df.sort_values(by='article_id', inplace=True)
    tags_df = tags_df.loc[tags_df['article_id'].isin(
        df.index.intersection(tags_df['article_id']))]

    locs_df = load_locations(data_folder)
    locs_df.sort_values(by='article_id', inplace=True)
    locs_df = locs_df.loc[locs_df['article_id'].isin(
        df.index.intersection(locs_df['article_id']))]

    model_tags_df = load_model_categories(data_folder)
    # will help cacheing
    model_tags_df.sort_values(by='article_id', inplace=True)
    model_tags_df = model_tags_df.loc[model_tags_df['article_id'].isin(
        df.index.intersection(model_tags_df['article_id']))]

    # init with empty lists
    df['locations'] = np.empty([df.shape[0], 0]).tolist()
    loc_article_ids = locs_df['article_id'].values
    df.loc[loc_article_ids, 'locations'] = locs_df['locations'].values

    def find_loc_in_string(locs, string):
        """
        The locations are generated from JavaScript, which means there's
        going to be some problems getting things to line up exactly and
        neatly. This function will hopefully performa all necessary
        transformations to find the given location text within the
        larger string.

        Inputs:
            locs: list of locations as loaded by load_locations
            string: bodytext of article in which to find locs
        Returns:
            updated_locs: list of locations as loaded by
                load_locations, but with a couple
                extra fields ('cleaned text' and 'cleaned span').
        """

        for i, loc in enumerate(locs):
            loc_txt = loc['text']

            loc_txt = clean_string(loc_txt)
            string = clean_string(string)

            loc['cleaned text'] = loc_txt

            spans = [x.span() for x in re.finditer(re.escape(loc_txt), string)]
            if spans:
                # The string may have occurred multiple times, and since the
                # spans don't line up perfectly we can't know which one is the
                # "correct" one. Best we can do is find the python span closest
                # to the expected javascript span.
                closest = np.argmin(np.abs(
                    np.array([x[0] for x in spans]) - loc['start']
                ))
                loc['cleaned span'] = spans[closest]

            locs[i] = loc

        return locs

    df['locations'] = df.apply(
        lambda r: find_loc_in_string(r['locations'], r['bodytext']),
        axis=1
    )

    num_no_match = df['locations'].apply(
        lambda locs: any([('cleaned span' not in loc) for loc in locs])
    ).sum()
    if num_no_match:
        warnings.warn(('{} location strings were not found in'
                       ' the bodytext.').format(num_no_match),
                      RuntimeWarning)

    model_locations_df = load_model_locations(data_folder)
    model_locations_df = model_locations_df.set_index('article_id')
    model_locations_gb = model_locations_df.groupby('article_id')
    model_locations_text = model_locations_gb['text'].apply(list)
    df['model_location_text'] = model_locations_text

    categories_df = load_categories(data_folder)
    categories_df.set_index('id', drop=True, inplace=True)

    # tags_df['category_id'] = tags_df['category_id'].astype(str)
    tags_df['category_abbreviation'] = (categories_df
                                        ['abbreviation']
                                        [tags_df['category_id']]
                                        .values)
    model_tags_df['category_abbreviation'] = (categories_df
                                              ['abbreviation']
                                              [model_tags_df['category_id']]
                                              .values)

    if np.setdiff1d(tags_df['article_id'].values, df.index.values).size:
        warnings.warn('Tags were found for article IDs that do not exist.',
                      RuntimeWarning)

    def update_df_with_categories(article_ids, cat_abbreviations, vals,
                                  is_model):
        # for some reason, some articles that are tagged don't show up
        # in the articles CSV. filter those out.
        existing_ids_filter = np.isin(article_ids, df.index.values)

        article_ids = article_ids[existing_ids_filter]
        cat_abbreviations = cat_abbreviations[existing_ids_filter]
        vals = vals[existing_ids_filter]

        for i in range(categories_df.shape[0]):
            cat_name = categories_df.loc[i+1, 'abbreviation']
            if is_model:
                cat_name += '_model'
            df[cat_name] = 0
            if not is_model:
                df[cat_name] = df[cat_name].astype('int8')
            matches = cat_abbreviations == cat_name
            if not matches.sum():
                continue
            df.loc[article_ids[matches], cat_name] = vals[matches]

    update_df_with_categories(
        model_tags_df['article_id'].values,
        model_tags_df['category_abbreviation'].values + '_model',
        model_tags_df['relevance'].values,
        is_model=True
    )
    update_df_with_categories(
        tags_df['article_id'].values,
        tags_df['category_abbreviation'].values,
        np.ones((tags_df['article_id'].values.shape), dtype='int8'),
        is_model=False
    )

    df.loc[df['bodytext'].isnull(), 'bodytext'] = ''

    return df

def subsample_and_resave(out_folder, n=5, input_folder=__data_folder,
                         random_seed=5):
    """
    Subsamples the CSV data files so that we have at least
    `n` articles from each type-of-crime tag as determined
    by the human coding. Saves the subsampled CSV data
    into `out_folder`. If there are fewer than `n` articles
    tagged with a type-of-crime, then we will use all of
    the articles with that tag.

    Inputs
    ------
    out_folder : str
        Path to folder where data should be saved. Should already exist.
    n : int
        How many examples from each category should we have?
    input_folder : str
        Path to where the full CSV files are saved.
    random_seed : None or int
        np.random.RandomState() will be seeded with this value
        in order to perform the random subsampling.
    """
    out_folder = str(Path(out_folder).expanduser().absolute())
    input_folder = str(Path(input_folder).expanduser().absolute())
    if out_folder == input_folder:
        raise RuntimeError('out_folder cannot match input_folder.')

    random_state = np.random.RandomState(random_seed)

    df = load_data(input_folder)
    chosen_indexes = []
    for crime_type in df.loc[:, 'OEMC':].columns:
        is_type = df[crime_type].astype(bool)
        n_samps = min(n, is_type.sum())
        chosen_indexes += (df.loc[is_type, :]
                           .sample(n_samps, random_state=random_state)
                           .index
                           .tolist())
    del df

    chosen_indexes = sorted(list(set(chosen_indexes)))

    # newsarticles_article.csv
    articles_df = load_articles(input_folder)
    sample = (articles_df
              .reset_index()
              .set_index('id')
              .loc[chosen_indexes, 'index'])
    articles_df = articles_df.loc[sample, :]
    # garble garble
    articles_df['bodytext'] = articles_df['bodytext'].astype(str).apply(
        lambda x: codecs.encode(x, 'rot-13')
    )
    articles_df.to_csv(os.path.join(out_folder, 'newsarticles_article.csv'),
                       header=None, index=False)
    del articles_df

    # newsarticles_category.csv
    shutil.copyfile(os.path.join(input_folder, 'newsarticles_category.csv'),
                    os.path.join(out_folder, 'newsarticles_category.csv'))

    # newsarticles_usercoding.csv
    uc_column_names = ['id', 'date', 'relevant',
                       'article_id', 'user_id', 'locations']

    uc_df = pd.read_csv(os.path.join(input_folder,
                                     'newsarticles_usercoding.csv'),
                        header=None,
                        names=uc_column_names)

    sample = np.where(uc_df['article_id'].isin(chosen_indexes))[0]
    uc_df.loc[sample, :].to_csv(
        os.path.join(out_folder, 'newsarticles_usercoding.csv'),
        header=None, index=False
    )

    uc_tags_column_names = ['id', 'usercoding_id', 'category_id']

    # newsarticles_usercoding_categories.csv
    uc_tags_df = pd.read_csv(
        os.path.join(input_folder,
                     'newsarticles_usercoding_categories.csv'),
        header=None,
        names=uc_tags_column_names,
        dtype={'id': int, 'usercoding_id': int, 'category_id': int}
    )
    sample = np.where(uc_df
                      .set_index('id')
                      .loc[uc_tags_df['usercoding_id'], 'article_id']
                      .isin(chosen_indexes)
                      )[0]
    uc_tags_df = uc_tags_df.loc[sample, :]
    uc_tags_df.to_csv(
        os.path.join(out_folder, 'newsarticles_usercoding_categories.csv'),
        header=None, index=False
    )

    # newsarticles_trainedcoding
    tc_names = ['id', 'date', 'model_info', 'relevance', 'article_id']
    tc = pd.read_csv(
        'tagnews/data/newsarticles_trainedcoding.csv',
        names=tc_names
    )
    tc = tc.loc[tc['article_id'].isin(chosen_indexes)]
    tc.to_csv(
        os.path.join(out_folder, 'newsarticles_trainedcoding.csv'),
        header=False, index=False
    )

    # newsarticles_trainedcategoryrelevance
    tcr_names = ['id', 'relevance', 'category_id', 'coding_id']
    tcr = pd.read_csv(
        'tagnews/data/newsarticles_trainedcategoryrelevance.csv',
        names=tcr_names
    )
    tcr = tcr.loc[tcr['coding_id'].isin(tc['id'])]
    tcr.to_csv(
        os.path.join(out_folder, 'newsarticles_trainedcategoryrelevance.csv'),
        header=False, index=False
    )

    # newsarticles_trainedlocation
    tl_names = ['id', 'text', 'latitude', 'longitude', 'coding_id']
    tl = pd.read_csv(
        'tagnews/data/newsarticles_trainedlocation.csv',
        names=tl_names
    )
    tl = tl.loc[tl['coding_id'].isin(tc['id'])]
    tl.to_csv(
        os.path.join(out_folder, 'newsarticles_trainedlocation.csv'),
        header=False, index=False
    )

def load_crime_data(data_folder=__data_folder):
    crimes = pd.read_csv(os.path.join(data_folder, 'Crimes.csv'))
    crimes = crimes[crimes['Year'] > 2010]

    crime_string = pd.Series('', crimes.index)

    # ['ID', 'Case Number', 'Date', 'Block', 'IUCR', 'Primary Type',
    #  'Description', 'Location Description', 'Arrest', 'Domestic', 'Beat',
    #  'District', 'Ward', 'Community Area', 'FBI Code', 'X Coordinate',
    #  'Y Coordinate', 'Year', 'Updated On', 'Latitude', 'Longitude',
    #  'Location']

    # TODO: synonyms on this for month name, weekday name,
    # time of day (e.g. afternoon), etc.
    crime_string += crimes['Date'] + ' '

    # TODO: synonyms?
    crime_string += crimes['Primary Type'] + ' '

    # TODO: synonyms?
    crime_string += crimes['Description'] + ' '

    # TODO: synonyms?
    crime_string += crimes['Location Description'] + ' '

    # TODO: synonyms?
    iucr = pd.read_csv(os.path.join(data_folder, 'IUCR.csv'))
    iucr.set_index('IUCR', drop=True, inplace=True)
    idx = iucr.index
    idx_values = idx.values
    idx_values[idx.str.len() == 3] = '0' + idx_values[idx.str.len() == 3]
    crime_string += (iucr.loc[crimes['IUCR'], 'PRIMARY DESCRIPTION']
                     .fillna('')
                     .values
                     + ' ')
    crime_string += (iucr.loc[crimes['IUCR'], 'SECONDARY DESCRIPTION']
                     .fillna('')
                     .values
                     + ' ')
    community_areas = pd.read_csv(os.path.join(data_folder, 'CommAreas.csv'))
    community_areas.set_index('AREA_NUM_1', inplace=True, drop=True)
    crime_string += (community_areas.loc[crimes['Community Area'], 'COMMUNITY']
                     .fillna('')
                     .values
                     + ' ')

    return crimes, crime_string

def load_ner_data(data_folder=__data_folder):
    """
    Loads ner.csv from the specified data folder.

    The column 'stag' is a binary value indicating whether or not
    the row corresponds to the entity "geo". Typically, you will
    want to use column 'word' to predict the column 'stag'.
    """
    df = pd.read_csv(os.path.join(data_folder, 'ner.csv'),
                     encoding="ISO-8859-1",
                     error_bad_lines=False,
                     index_col=0)

    df.dropna(subset=['word', 'tag'], inplace=True)
    df.reset_index(inplace=True, drop=True)
    df['stag'] = (df['tag'] == 'B-geo') | (df['tag'] == 'I-geo')
    df['all_tags'] = df['tag']
    df['tag'] = df['stag']
    df = df[['word', 'all_tags', 'tag']]

    return df


"""
=================================================
Hyper-parameters of Approximate Nearest Neighbors
=================================================

This example demonstrates the behaviour of the
accuracy of the nearest neighbor queries of Locality Sensitive Hashing
Forest as the number of candidates and the number of estimators (trees)
vary.

In the first plot, accuracy is measured with the number of candidates. Here,
the term "number of candidates" refers to maximum bound for the number of
distinct points retrieved from each tree to calculate the distances. Nearest
neighbors are selected from this pool of candidates. Number of estimators is
maintained at three fixed levels (1, 5, 10).

In the second plot, the number of candidates is fixed at 50. Number of trees
is varied and the accuracy is plotted against those values. To measure the
accuracy, the true nearest neighbors are required, therefore
:class:`sklearn.neighbors.NearestNeighbors` is used to compute the exact
neighbors.
"""
from __future__ import division
print(__doc__)

# Author: Maheshakya Wijewardena <maheshakya.10@cse.mrt.ac.lk>
#
# License: BSD 3 clause

###############################################################################
import numpy as np
from sklearn.datasets.samples_generator import make_blobs
from sklearn.neighbors import LSHForest
from sklearn.neighbors import NearestNeighbors
import matplotlib.pyplot as plt

# Initialize size of the database, iterations and required neighbors.
n_samples = 10000
n_features = 100
n_queries = 30
rng = np.random.RandomState(42)

# Generate sample data
X, _ = make_blobs(n_samples=n_samples + n_queries,
                  n_features=n_features, centers=10,
                  random_state=0)
X_index = X[:n_samples]
X_query = X[n_samples:]
# Get exact neighbors
nbrs = NearestNeighbors(n_neighbors=1, algorithm='brute',
                        metric='cosine').fit(X_index)
neighbors_exact = nbrs.kneighbors(X_query, return_distance=False)

# Set `n_candidate` values
n_candidates_values = np.linspace(10, 500, 5).astype(np.int)
n_estimators_for_candidate_value = [1, 5, 10]
n_iter = 10
stds_accuracies = np.zeros((len(n_estimators_for_candidate_value),
                            n_candidates_values.shape[0]),
                           dtype=float)
accuracies_c = np.zeros((len(n_estimators_for_candidate_value),
                         n_candidates_values.shape[0]), dtype=float)

# LSH Forest is a stochastic index: perform several iteration to estimate
# expected accuracy and standard deviation displayed as error bars in
# the plots
for j, value in enumerate(n_estimators_for_candidate_value):
    for i, n_candidates in enumerate(n_candidates_values):
        accuracy_c = []
        for seed in range(n_iter):
            lshf = LSHForest(n_estimators=value,
                             n_candidates=n_candidates, n_neighbors=1,
                             random_state=seed)
            # Build the LSH Forest index
            lshf.fit(X_index)
            # Get neighbors
            neighbors_approx = lshf.kneighbors(X_query,
                                               return_distance=False)
            accuracy_c.append(np.sum(np.equal(neighbors_approx,
                                              neighbors_exact)) /
                              n_queries)

        stds_accuracies[j, i] = np.std(accuracy_c)
        accuracies_c[j, i] = np.mean(accuracy_c)

# Set `n_estimators` values
n_estimators_values = [1, 5, 10, 20, 30, 40, 50]
accuracies_trees = np.zeros(len(n_estimators_values), dtype=float)

# Calculate average accuracy for each value of `n_estimators`
for i, n_estimators in enumerate(n_estimators_values):
    lshf = LSHForest(n_estimators=n_estimators, n_neighbors=1)
    # Build the LSH Forest index
    lshf.fit(X_index)
    # Get neighbors
    neighbors_approx = lshf.kneighbors(X_query, return_distance=False)
    accuracies_trees[i] = np.sum(np.equal(neighbors_approx,
                                          neighbors_exact))/n_queries

###############################################################################
# Plot the accuracy variation with `n_candidates`
plt.figure()
colors = ['c', 'm', 'y']
for i, n_estimators in enumerate(n_estimators_for_candidate_value):
    label = 'n_estimators = %d ' % n_estimators
    plt.plot(n_candidates_values, accuracies_c[i, :],
             'o-', c=colors[i], label=label)
    plt.errorbar(n_candidates_values, accuracies_c[i, :],
                 stds_accuracies[i, :], c=colors[i])

plt.legend(loc='upper left', fontsize='small')
plt.ylim([0, 1.2])
plt.xlim(min(n_candidates_values), max(n_candidates_values))
plt.ylabel("Accuracy")
plt.xlabel("n_candidates")
plt.grid(which='both')
plt.title("Accuracy variation with n_candidates")

# Plot the accuracy variation with `n_estimators`
plt.figure()
plt.scatter(n_estimators_values, accuracies_trees, c='k')
plt.plot(n_estimators_values, accuracies_trees, c='g')
plt.ylim([0, 1.2])
plt.xlim(min(n_estimators_values), max(n_estimators_values))
plt.ylabel("Accuracy")
plt.xlabel("n_estimators")
plt.grid(which='both')
plt.title("Accuracy variation with n_estimators")

plt.show()


# -*- coding: utf-8 -*-
from __future__ import absolute_import #导入3.x的特征函数
from __future__ import print_function

import yaml
import sys
reload(sys)
sys.setdefaultencoding('utf8')

import pandas as pd #导入Pandas
import numpy as np #导入Numpy
import jieba #导入结巴分词
import h5py, pickle, os, datetime
from keras.models import model_from_json, save_model

from keras.preprocessing import sequence
from keras.optimizers import SGD, RMSprop, Adagrad
from keras.utils import np_utils
from keras.models import Sequential, model_from_yaml
from keras.layers.core import Dense, Dropout, Activation
from keras.layers.embeddings import Embedding
from keras.layers.recurrent import LSTM, GRU
sys.setrecursionlimit(1000000)

# save model http://www.linuxdiyf.com/linux/22940.html
# http://www.linuxdiyf.com/linux/22937.html
#
# http://spaces.ac.cn/archives/3414/
# https://github.com/BUPTLdy/Sentiment-Analysis
# http://blog.sina.com.cn/s/blog_735f29100102wjwu.html
# http://blog.csdn.net/weixin_36541072/article/details/53786020
# https://keras-cn.readthedocs.io/en/latest/getting_started/concepts/
# https://keras-cn.readthedocs.io/en/latest/getting_started/keras_linux/

# 设置参数
maxlen = 50
lstm_batch_size = 16
lstm_epochs = 15

datadir = ''
modeldir = '../model/lstm_didi'
testdir = ''

# 加载训练文件
def loadfile():
  print("读取语料数据")
  neg=pd.read_excel(datadir + '/neg.xls',header=None,index=None)
  mid=pd.read_excel(datadir + '/pos.xls',header=None,index=None)
  print("读取训练语料完毕")
  print("给训练语料贴上标签")
  mid['mark']=1
  neg['mark']=0
  print("合并语料")
  pn=pd.concat([mid,neg],ignore_index=True)
  neglen=len(neg)
  midlen=len(mid) #计算语料数目
  print('neg count:' + str(neglen))
  print('pos count:' + str(midlen))
  return pn

def tokenizer(text):
  cw = lambda x: list(jieba.cut(x)) #定义分词函数
  text['words'] = text[0].apply(cw)
  return text

def generatedict(text):
  # 计算词典并保存
  d2v_train = pd.concat([text['words']], ignore_index = True)
  w = [] #将所有词语整合在一起
  for i in d2v_train:
    w.extend(i)
  dict = pd.DataFrame(pd.Series(w).value_counts()) #统计词的出现次数
  del w,d2v_train
  dict['id'] = list(range(1,len(dict)+1))
  # 这个 dict 需要保存下来
  outputFile = modeldir + '/dict.data'
  fw = open(outputFile, 'w')
  pickle.dump(dict,fw)
  fw.close()
  return dict

def word2index(text, dict):
  get_sent = lambda x: list(dict['id'][x])
  text['sent'] = text['words'].apply(get_sent)
  print("Pad sequences (samples x time)")
  text['sent'] = list(sequence.pad_sequences(text['sent'], maxlen=maxlen))
  return text

def getdata(text):
  x = np.array(list(text['sent']))[::2] #训练集
  y = np.array(list(text['mark']))[::2]
  xt = np.array(list(text['sent']))[1::2] #测试集
  yt = np.array(list(text['mark']))[1::2]
  xa = np.array(list(text['sent'])) #全集
  ya = np.array(list(text['mark']))
  return x,y,xt,yt,xa,ya

def train_lstm(dict,x,y,xt,yt):
  model = Sequential()
  model.add(Embedding(len(dict)+1, 256, input_length=maxlen))
  model.add(LSTM(output_dim=128, activation='sigmoid', inner_activation='hard_sigmoid'))
  model.add(Dropout(0.5))
  model.add(Dense(1))
  # model.add(Dense(input_dim = 32, output_dim = 1))
  model.add(Activation('sigmoid'))
  print ('模型构建完成')
  #model.compile(loss='binary_crossentropy', optimizer='adam', class_mode="binary")
  model.compile(optimizer='rmsprop', loss='binary_crossentropy', metrics=['accuracy'])
  print ("模型编译完成")
  model.fit(x, y, batch_size=lstm_batch_size, epochs=lstm_epochs, verbose=0)
  print ("模型训练完成")
  print ("保存模型")
  yaml_string = model.to_yaml()
  with open(modeldir + '/lstm.yml', 'w') as outfile:
    outfile.write( yaml.dump(yaml_string, default_flow_style=True) )
  model.save_weights(modeldir + '/lstm.h5')
  print ("测试集评估")
  score = model.evaluate(xt, yt, verbose=0)
  print ("准确率:",score[1])
  return model

def saveresult(model, xt, text):
  classes = model.predict_classes(xt, verbose=1)
  proba = model.predict_proba(xt, verbose=1)

  print ("\n输出结果")
  filename = 'result.txt'
  f = open('result.txt', 'w')
  i = 1
  j = 0
  for c in classes:
    f.write(str(c))
    f.write(",")
    f.write(str(proba[j]))
    f.write(",")
    line = "".join(text['words'][i])
    f.write(line.encode('utf-8'))
    f.write("\n")
    i = i + 2
    j = j + 1
  f.close()
  print ("\n排序结果")
  num = 1
  result = []
  with open(filename, 'r') as f:
    while True:
      line = f.readline()
      if not line:
        break
      print("processing line #" + str(num))
      num = num + 1
      arr = line.split(',')
      item = (int(arr[0][1:-1]), float(arr[1][2:-1]), "".join(arr[2:]))
      result.append(item)
    result.sort(key=lambda tup:tup[1])
    print(len(result))
    f = open('sorted.txt', 'w')
    for item in result:
      f.write(str(item[0]))
      f.write(",")
      f.write(str(item[1]))
      f.write(",")
      f.write(item[2])
  print("done")

def loaddict():
  fr = open(modeldir + '/dict.data')
  dict = pickle.load(fr)
  return dict

#训练模型，并保存
def train():
  print('Loading Data...')
  pn = loadfile()
  print('Tokenising...')
  pn = tokenizer(pn)
  print('Generating Dict...')
  dict = generatedict(pn)
  print('Word to Index...')
  pn = word2index(pn, dict)
  print('Preparing data...')
  x,y,xt,yt,xa,ya = getdata(pn)
  print('Model Stage...')
  # 这里训练全量模型
  model = train_lstm(dict, xa, ya, xt, yt)
  #print('Save Test Result...')
  #saveresult(model, xt, pn)
  print("Done")

def batchtest(filepath):
  dict = loaddict()

  with open(modeldir + '/lstm.yml', 'r') as f:
    yaml_string = yaml.load(f)
  model = model_from_yaml(yaml_string)
  model.load_weights(modeldir + '/lstm.h5')
  model.compile(optimizer='rmsprop', loss='binary_crossentropy', metrics=['accuracy'])

  # 读取测试文件
  # 开始一个 for 循环，外面要统计
  test_count = 0
  correct_count = 0
  if os.path.exists(filepath):
    f = open(filepath, 'r')
    try:
      lines = f.readlines()
      for line in lines:
        if len(line) <= 0:
          continue
        else:
          arr = line.split(',')
          label = arr[0]
          test_count += 1
          text = ",".join(arr[1:])
          textarr = list(jieba.cut(text))
          textvec = []
          add = 1
          for item in textarr:
            # 如果不在词典里，则直接丢弃（因为出现的次数也非常少，不考虑）
            if item in dict['id']:
              textvec.append(dict['id'][item])
          textvec = pd.Series(textvec)
          textvec = sequence.pad_sequences([textvec], maxlen=maxlen)
          # 概率
          proba = model.predict_proba(textvec, verbose=0)
          # 判断是否计算正确
          for s in proba:
            if s[0] > 0.5 and label == '1' or s[0] <= 0.5 and label == '0':
              correct_count += 1
              print('[' + str(test_count) + ']: ' + label + ' ' + str(s[0]) + ' ' + text[:-1])
            else:
              print('[' + str(test_count) + ']:[x] ' + label + ' ' + str(s[0]) + ' ' + text[:-1])
    finally:
      f.close() # 确保关闭
  return correct_count, test_count

# 批量预测，减少内存使用，传入一个字符串数组
def predict_arr(arr):
  dict = loaddict()

  probas = []
  with open(modeldir + '/lstm.yml', 'r') as f:
    yaml_string = yaml.load(f)
  model = model_from_yaml(yaml_string)
  model.load_weights(modeldir + '/lstm.h5')
  model.compile(optimizer='rmsprop', loss='binary_crossentropy', metrics=['accuracy'])

  for s in arr:
    textarr = list(jieba.cut(s))
    textvec = []
    add = 1
    for item in textarr:
      # 如果不在词典里，则直接丢弃（因为出现的次数也非常少，不考虑）
      if item in dict['id']:
        textvec.append(dict['id'][item])
    textvec = pd.Series(textvec)
    textvec = sequence.pad_sequences([textvec], maxlen=maxlen)

    proba = model.predict_proba(textvec, verbose=0)
    probas.append(proba[0][0])

  return probas

def predict(text):
  print('Loading Dict Data..')
  dict = loaddict()

  # 把每个词转化为在词典里的数字，更新词典的计数（参考上面的格式）
  textarr = list(jieba.cut(text))

  textvec = []
  add = 1
  for item in textarr:
    # 如果不在词典里，则直接丢弃（因为出现的次数也非常少，不考虑）
    if item in dict['id']:
      textvec.append(dict['id'][item])

  textvec = pd.Series(textvec)
  textvec = sequence.pad_sequences([textvec], maxlen=maxlen)

  # ----
  print('loading model......')
  with open(modeldir + '/lstm.yml', 'r') as f:
    yaml_string = yaml.load(f)
  model = model_from_yaml(yaml_string)

  print('loading weights......')
  model.load_weights(modeldir + '/lstm.h5')
  model.compile(optimizer='rmsprop', loss='binary_crossentropy', metrics=['accuracy'])

  # 至此模型已经载入完成，可以进行预测
  #classes = model.predict_classes(textvec, verbose=1)
  proba = model.predict_proba(textvec, verbose=0)
  # 这里因为知识图谱暂时改变输入格式
  #for s in proba:
  #  if s[0] > 0.5:
  #    print('positive ' + str(s[0]) + ' ' + text)
  #  else:
  #    print('negative ' + str(s[0]) + ' ' + text)
  return proba[0][0]

if __name__=='__main__':
    argvs_length = len(sys.argv)
    if argvs_length >= 4:
        argvs = sys.argv
        action = argvs[1]
        if action == 'train': # 训练
            datadir = argvs[2]
            modeldir = argvs[3]
            begin = datetime.datetime.now()
            train()
            end = datetime.datetime.now()
            # 统计训练时间、模型大小，写入到 result.txt 中
            with open(modeldir + '/result.txt', "w") as f:
                f.write('训练时长: ' + str(end-begin))
        elif action == 'predict':
            modeldir = argvs[2]
            sentence = " ".join(argvs[3:])
            predict(sentence)
        elif action == 'test':
            datadir = argvs[2]
            modeldir = argvs[3]
            testdir = argvs[4]
            begin = datetime.datetime.now()
            result = batchtest(datadir+'/test.txt')
            end = datetime.datetime.now()
            # 统计训练时间、模型大小，写入到 result.txt 中
            with open(testdir + '/result.txt', "w") as f:
                f.write('测试时长: ' + str(end-begin) + '\n')
                f.write('正确率: ' + str(float(result[0])/float(result[1])) + ' (' + str(result[0]) + '/' + str(result[1]) + ')\n')


# coding: utf-8

#read in libraries
import cPickle as pickle
from webcrawler import coredump
from dataloader import get_trawled_data, introduce_weighting
from ratings import PowerRater
from history import historical, model_the_model
from predict import predict
from oddsmaker import read_odds
from betting import wager

csv_file = 'NBA_data_2015' # whence the data come
weight_factor = 60 # number of days over which to decrease the weight by 40%
# weight factor needs justification
HCA = 1.8
sigma = 13.5

# ask user for guidence on reading in data from web and write to csv file
user_input = raw_input("Do you want to trawl the web for the latest data? ")
if user_input in ['y', 'Y', 'yes', 'Yes', 'YES']:
    website = 'http://www.basketball-reference.com/leagues/NBA_2016_games.html'
    coredump(website, csv_file)

#load data from csv as a pandas DataFrame
data = get_trawled_data(csv_file, ['Date', 'Away', 'Away Score', 'Home', 'Home Score'])

# compile a list of historical predictions and actual outcomes
history_file = 'Predictive_outcomes_2015'
past_predictions = historical(data, weight_factor, history_file, HCA, sigma)

# get the fit parameters needed to correct errors in the historical model
beta_correct = model_the_model(past_predictions)
print 'Checking on the model parameters: %s' % beta_correct

# make predictions
todays_schedule = pickle.load(open('Today', 'rb'))
data = introduce_weighting(data, weight_factor,home_court_advantage=HCA) # add weights column
PwrRt = PowerRater(data).power_ratings # generate latest ratings
print PwrRt.sort_values(by='rating', ascending=False)
prob = []
for i in xrange(todays_schedule.shape[0]):
    prob.append(predict(todays_schedule.iloc[i], PwrRt, HCA, sigma))
todays_schedule['Prob'] = prob

# pull in odds
odds =  read_odds('Odds.csv', todays_schedule)

# determine optimal betting strategy
print wager(odds)

# pull in 538 predictions
# model the 538 model


import numpy as np
import pandas as pd
from pandas.core.dtypes.common import (
    is_scalar,
    is_numeric_dtype,
    is_decimal,
    is_datetime_or_timedelta_dtype,
    is_number,
    _ensure_object)
from pandas.core.dtypes.generic import ABCSeries, ABCIndexClass
from pandas.core.dtypes.cast import maybe_downcast_to_dtype
from pandas._libs import lib

def to_numeric(arg, errors='raise', downcast=None):
    """
    Convert argument to a numeric type.

    The default return dtype is `float64` or `int64`
    depending on the data supplied. Use the `downcast` parameter
    to obtain other dtypes.

    Parameters
    ----------
    arg : list, tuple, 1-d array, or Series
    errors : {'ignore', 'raise', 'coerce'}, default 'raise'
        - If 'raise', then invalid parsing will raise an exception
        - If 'coerce', then invalid parsing will be set as NaN
        - If 'ignore', then invalid parsing will return the input
    downcast : {'integer', 'signed', 'unsigned', 'float'} , default None
        If not None, and if the data has been successfully cast to a
        numerical dtype (or if the data was numeric to begin with),
        downcast that resulting data to the smallest numerical dtype
        possible according to the following rules:

        - 'integer' or 'signed': smallest signed int dtype (min.: np.int8)
        - 'unsigned': smallest unsigned int dtype (min.: np.uint8)
        - 'float': smallest float dtype (min.: np.float32)

        As this behaviour is separate from the core conversion to
        numeric values, any errors raised during the downcasting
        will be surfaced regardless of the value of the 'errors' input.

        In addition, downcasting will only occur if the size
        of the resulting data's dtype is strictly larger than
        the dtype it is to be cast to, so if none of the dtypes
        checked satisfy that specification, no downcasting will be
        performed on the data.

        .. versionadded:: 0.19.0

    Returns
    -------
    ret : numeric if parsing succeeded.
        Return type depends on input.  Series if Series, otherwise ndarray

    Examples
    --------
    Take separate series and convert to numeric, coercing when told to

    >>> s = pd.Series(['1.0', '2', -3])
    >>> pd.to_numeric(s)
    0    1.0
    1    2.0
    2   -3.0
    dtype: float64
    >>> pd.to_numeric(s, downcast='float')
    0    1.0
    1    2.0
    2   -3.0
    dtype: float32
    >>> pd.to_numeric(s, downcast='signed')
    0    1
    1    2
    2   -3
    dtype: int8
    >>> s = pd.Series(['apple', '1.0', '2', -3])
    >>> pd.to_numeric(s, errors='ignore')
    0    apple
    1      1.0
    2        2
    3       -3
    dtype: object
    >>> pd.to_numeric(s, errors='coerce')
    0    NaN
    1    1.0
    2    2.0
    3   -3.0
    dtype: float64

    See also
    --------
    pandas.DataFrame.astype : Cast argument to a specified dtype.
    pandas.to_datetime : Convert argument to datetime.
    pandas.to_timedelta : Convert argument to timedelta.
    numpy.ndarray.astype : Cast a numpy array to a specified type.
    """
    if downcast not in (None, 'integer', 'signed', 'unsigned', 'float'):
        raise ValueError('invalid downcasting method provided')

    is_series = False
    is_index = False
    is_scalars = False

    if isinstance(arg, ABCSeries):
        is_series = True
        values = arg.values
    elif isinstance(arg, ABCIndexClass):
        is_index = True
        values = arg.asi8
        if values is None:
            values = arg.values
    elif isinstance(arg, (list, tuple)):
        values = np.array(arg, dtype='O')
    elif is_scalar(arg):
        if is_decimal(arg):
            return float(arg)
        if is_number(arg):
            return arg
        is_scalars = True
        values = np.array([arg], dtype='O')
    elif getattr(arg, 'ndim', 1) > 1:
        raise TypeError('arg must be a list, tuple, 1-d array, or Series')
    else:
        values = arg

    try:
        if is_numeric_dtype(values):
            pass
        elif is_datetime_or_timedelta_dtype(values):
            values = values.astype(np.int64)
        else:
            values = _ensure_object(values)
            coerce_numeric = False if errors in ('ignore', 'raise') else True
            values = lib.maybe_convert_numeric(values, set(),
                                               coerce_numeric=coerce_numeric)

    except Exception:
        if errors == 'raise':
            raise

    # attempt downcast only if the data has been successfully converted
    # to a numerical dtype and if a downcast method has been specified
    if downcast is not None and is_numeric_dtype(values):
        typecodes = None

        if downcast in ('integer', 'signed'):
            typecodes = np.typecodes['Integer']
        elif downcast == 'unsigned' and np.min(values) >= 0:
            typecodes = np.typecodes['UnsignedInteger']
        elif downcast == 'float':
            typecodes = np.typecodes['Float']

            # pandas support goes only to np.float32,
            # as float dtypes smaller than that are
            # extremely rare and not well supported
            float_32_char = np.dtype(np.float32).char
            float_32_ind = typecodes.index(float_32_char)
            typecodes = typecodes[float_32_ind:]

        if typecodes is not None:
            # from smallest to largest
            for dtype in typecodes:
                if np.dtype(dtype).itemsize <= values.dtype.itemsize:
                    values = maybe_downcast_to_dtype(values, dtype)

                    # successful conversion
                    if values.dtype == dtype:
                        break

    if is_series:
        return pd.Series(values, index=arg.index, name=arg.name)
    elif is_index:
        # because we want to coerce to numeric if possible,
        # do not use _shallow_copy_with_infer
        return pd.Index(values, name=arg.name)
    elif is_scalars:
        return values[0]
    else:
        return values


# This file was auto-generated by the PsychChart.py script in wrappers/Python/CoolProp/Plots

if __name__ == '__main__':
    import numpy, matplotlib
    from CoolProp.HumidAirProp import HAPropsSI
    from CoolProp.Plots.Plots import InlineLabel

    p = 101325
    Tdb = numpy.linspace(-10, 60, 100) + 273.15

    # Make the figure and the axes
    fig = matplotlib.pyplot.figure(figsize=(10, 8))
    ax = fig.add_axes((0.1, 0.1, 0.85, 0.85))

    # Saturation line
    w = [HAPropsSI('W', 'T', T, 'P', p, 'R', 1.0) for T in Tdb]
    ax.plot(Tdb - 273.15, w, lw=2)

    # Humidity lines
    RHValues = [0.05, 0.1, 0.15, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]
    for RH in RHValues:
        w = [HAPropsSI('W', 'T', T, 'P', p, 'R', RH) for T in Tdb]
        ax.plot(Tdb - 273.15, w, 'r', lw=1)

    # Humidity lines
    for H in [-20000, -10000, 0, 10000, 20000, 30000, 40000, 50000, 60000, 70000, 80000, 90000]:
        # Line goes from saturation to zero humidity ratio for this enthalpy
        T1 = HAPropsSI('T', 'H', H, 'P', p, 'R', 1.0) - 273.15
        T0 = HAPropsSI('T', 'H', H, 'P', p, 'R', 0.0) - 273.15
        w1 = HAPropsSI('W', 'H', H, 'P', p, 'R', 1.0)
        w0 = HAPropsSI('W', 'H', H, 'P', p, 'R', 0.0)
        ax.plot(numpy.r_[T1, T0], numpy.r_[w1, w0], 'r', lw=1)

    ax.set_xlim(Tdb[0] - 273.15, Tdb[-1] - 273.15)
    ax.set_ylim(0, 0.03)
    ax.set_xlabel(r"$T_{db}$ [$^{\circ}$C]")
    ax.set_ylabel(r"$W$ ($m_{w}/m_{da}$) [-]")

    xv = Tdb  # [K]
    for RH in [0.05, 0.1, 0.15, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]:
        yv = [HAPropsSI('W', 'T', T, 'P', p, 'R', RH) for T in Tdb]
        y = HAPropsSI('W', 'P', p, 'H', 65000.000000, 'R', RH)
        T_K, w, rot = InlineLabel(xv, yv, y=y, axis=ax)
        string = r'$\phi$=' + '{s:0.0f}'.format(s=RH * 100) + '%'
        bbox_opts = dict(boxstyle='square,pad=0.0', fc='white', ec='None', alpha=0.5)
        ax.text(T_K - 273.15, w, string, rotation=rot, ha='center', va='center', bbox=bbox_opts)

    matplotlib.pyplot.show()


"""Unit testing of tests in docs/source/examples"""
import unittest
import os
import numpy as np

from gpkit import settings
from gpkit.tests.helpers import generate_example_tests
from gpkit.small_scripts import mag
from gpkit.small_classes import Quantity

def assert_logtol(first, second, logtol=1e-6):
    "Asserts that the logs of two arrays have a given abstol"
    np.testing.assert_allclose(np.log(mag(first)), np.log(mag(second)),
                               atol=logtol, rtol=0)

# pylint: disable=too-many-public-methods
class TestExamples(unittest.TestCase):
    """
    To test a new example, add a function called `test_$EXAMPLENAME`, where
    $EXAMPLENAME is the name of your example in docs/source/examples without
    the file extension.

    This function should accept two arguments (e.g. 'self' and 'example').
    The imported example script will be passed to the second: anything that
    was a global variable (e.g, "sol") in the original script is available
    as an attribute (e.g., "example.sol")

    If you don't want to perform any checks on the example besides making
    sure it runs, just put "pass" as the function's body, e.g.:

          def test_dummy_example(self, example):
              pass

    But it's good practice to ensure the example's solution as well, e.g.:

          def test_dummy_example(self, example):
              self.assertAlmostEqual(example.sol["cost"], 3.121)
    """

    # TODO: allow enabling plotting examples, make plots in correct folder...
    # def test_plot_sweep1d(self, _):
    #     import matplotlib.pyplot as plt
    #     plt.close("all")

    def test_autosweep(self, example):
        from gpkit import ureg
        bst1, tol1 = example.bst1, example.tol1
        bst2, tol2 = example.bst2, example.tol2

        l_ = np.linspace(1, 10, 100)
        for bst in [bst1, example.bst1_loaded]:
            sol1 = bst.sample_at(l_)
            assert_logtol(sol1("l"), l_)
            assert_logtol(sol1("A"), l_**2 + 1, tol1)
            assert_logtol(sol1["cost"], (l_**2 + 1)**2, tol1)
            if hasattr(sol1["cost"], "units"):  # loaded costs are unitless
                self.assertEqual(Quantity(1.0, sol1["cost"].units),
                                 Quantity(1.0, ureg.m)**4)
            self.assertEqual(Quantity(1.0, sol1("A").units),
                             Quantity(1.0, ureg.m)**2)

        ndig = -int(np.log10(tol2))
        self.assertAlmostEqual(bst2.cost_at("cost", 3), 1.0, ndig)
        # before corner
        A_bc = np.linspace(1, 3, 50)
        sol_bc = bst2.sample_at(A_bc)
        assert_logtol(sol_bc("A"), (A_bc/3)**0.5, tol2)
        assert_logtol(sol_bc["cost"], A_bc/3, tol2)
        # after corner
        A_ac = np.linspace(3, 10, 50)
        sol_ac = bst2.sample_at(A_ac)
        assert_logtol(sol_ac("A"), (A_ac/3)**2, tol2)
        assert_logtol(sol_ac["cost"], (A_ac/3)**4, tol2)

    def test_model_var_access(self, example):
        model = example.PS
        _ = model["E"]
        with self.assertRaises(ValueError):
            _ = model["m"]  # multiple variables called m

    def test_performance_modeling(self, example):
        pass

    def test_sp_to_gp_sweep(self, example):
        pass

    def test_boundschecking(self, example):
        pass

    def test_vectorize(self, example):
        pass

    def test_primal_infeasible_ex1(self, example):
        with self.assertRaises(RuntimeWarning) as cm:
            example.m.solve(verbosity=0)
        err = cm.exception
        if "mosek" in err.message:
            self.assertIn("PRIM_INFEAS_CER", err.message)
        elif "cvxopt" in err.message:
            self.assertIn("unknown", err.message)

    def test_primal_infeasible_ex2(self, example):
        with self.assertRaises(RuntimeWarning):
            example.m.solve(verbosity=0)

    def test_docstringparsing(self, example):
        pass

    def test_debug(self, example):
        pass

    def test_simple_sp(self, example):
        pass

    def test_simple_box(self, example):
        pass

    def test_x_greaterthan_1(self, example):
        pass

    def test_beam(self, example):
        self.assertFalse(np.isnan(example.sol("w")).any())

    def test_water_tank(self, example):
        pass

    def test_sin_approx_example(self, example):
        pass

    def test_external_sp(self, example):
        pass

    def test_external_sp2(self, example):
        pass

    def test_simpleflight(self, example):
        self.assertTrue(example.sol.almost_equal(example.sol_loaded))
        for sol in [example.sol, example.sol_loaded]:
            freevarcheck = {
                "A": 8.46,
                "C_D": 0.0206,
                "C_f": 0.0036,
                "C_L": 0.499,
                "Re": 3.68e+06,
                "S": 16.4,
                "W": 7.34e+03,
                "V": 38.2,
                "W_w": 2.40e+03
            }
            # sensitivity values from p. 34 of W. Hoburg's thesis
            senscheck = {
                r"(\frac{S}{S_{wet}})": 0.4300,
                "e": -0.4785,
                "V_{min}": -0.3691,
                "k": 0.4300,
                r"\mu": 0.0860,
                "(CDA0)": 0.0915,
                "C_{L,max}": -0.1845,
                r"\tau": -0.2903,
                "N_{ult}": 0.2903,
                "W_0": 1.0107,
                r"\rho": -0.2275
            }
            for key in freevarcheck:
                sol_rat = mag(sol["variables"][key])/freevarcheck[key]
                self.assertTrue(abs(1-sol_rat) < 1e-2)
            for key in senscheck:
                sol_rat = sol["sensitivities"]["constants"][key]/senscheck[key]
                self.assertTrue(abs(1-sol_rat) < 1e-2)

    def test_relaxation(self, example):
        pass

    def test_unbounded(self, example):
        pass

FILE_DIR = os.path.dirname(os.path.realpath(__file__))
EXAMPLE_DIR = os.path.abspath(FILE_DIR + '../../../docs/source/examples')
SOLVERS = settings["installed_solvers"]
if os.path.isdir(EXAMPLE_DIR):
    TESTS = generate_example_tests(EXAMPLE_DIR, [TestExamples], SOLVERS)
else:
    TESTS = []

if __name__ == "__main__":
    # pylint:disable=wrong-import-position
    from gpkit.tests.helpers import run_tests
    run_tests(TESTS)


import math
import re

import matplotlib
import matplotlib.pyplot as plt
from matplotlib import collections, lines, patches

from analysis import heatmaps

import config

# Source: https://gist.github.com/jasonmc/1160951
def _set_foregroundcolor(ax, color):
    '''For the specified axes, sets the color of the frame, major ticks,
    tick labels, axis labels, title and legend
    '''
    for tl in ax.get_xticklines() + ax.get_yticklines():
        tl.set_color(color)
    for spine in ax.spines:
        ax.spines[spine].set_edgecolor(color)
    for tick in ax.xaxis.get_major_ticks():
        tick.label1.set_color(color)
    for tick in ax.yaxis.get_major_ticks():
        tick.label1.set_color(color)
    ax.axes.xaxis.label.set_color(color)
    ax.axes.yaxis.label.set_color(color)
    ax.axes.xaxis.get_offset_text().set_color(color)
    ax.axes.yaxis.get_offset_text().set_color(color)
    ax.axes.title.set_color(color)
    lh = ax.get_legend()
    if lh is not None:
        lh.get_title().set_color(color)
        lh.legendPatch.set_edgecolor('none')
        labels = lh.get_texts()
        for lab in labels:
            lab.set_color(color)
    for tl in ax.get_xticklabels():
        tl.set_color(color)
    for tl in ax.get_yticklabels():
        tl.set_color(color)

# Source: https://gist.github.com/jasonmc/1160951
def _set_backgroundcolor(ax, color):
    '''Sets the background color of the current axes (and legend).
    Use 'None' (with quotes) for transparent. To get transparent
    background on saved figures, use:
    pp.savefig("fig1.svg", transparent=True)
    '''
    ax.patch.set_facecolor(color)
    lh = ax.get_legend()
    if lh is not None:
        lh.legendPatch.set_facecolor(color)

def format_axis(ax):
    _set_foregroundcolor(ax, '0.5')
    _set_backgroundcolor(ax, '0.08')
    # drop plot borders
    for spine in ax.spines:
        ax.spines[spine].set_visible(False)

def _format_figure(fig):
    fig.patch.set_facecolor('0.12')
    plt.tight_layout()

def show():
    ''' Shows the current figure (on screen, if using a GUI backend).
        Create a plot first using a TrackPlotter object. '''
    fig = plt.gcf()
    _format_figure(fig)
    plt.show()
    plt.close('all')

def savefig(outfile, format=None):
    ''' Saves the current figure to the given filename or file-like object.

        Format is inferred from the file extension if a name is given,
        otherwise specify it manually with the format parameter.

        Large (tall) figures are broken into multiple images (vertical tiles)
        if outfile is a string (filename).

        Create a plot first using a TrackPlotter object or other code that
        creates a pyplot figure.
    '''
    fig = plt.gcf()
    _format_figure(fig)

    # A bit of an ugly hack to split giant images into multiple parts
    # Only used if outfile is given as a string (filename)
    max_height = 100 if isinstance(outfile, str) else float('inf')
    # plot height in inches
    height = fig.get_window_extent().transformed(fig.dpi_scale_trans.inverted()).height
    if height > max_height:
        numparts = int(height / max_height) + 1
        for i in range(numparts):
            filename = re.sub(r"(\.[^\.]+)$", r"%02d\1" % (numparts-i), outfile)
            bbox = matplotlib.transforms.Bbox.from_extents([0,i*max_height,12,min(height,(i+1)*max_height)])
            plt.savefig(filename, facecolor=fig.get_facecolor(), edgecolor='none', bbox_inches=bbox, format=format)
    else:
        plt.savefig(outfile, facecolor=fig.get_facecolor(), edgecolor='none', format=format)

    plt.close('all')

class TrackPlotter(object):
    def __init__(self, track_processor, dbgframes=None):
        self._track = track_processor
        self._dbgframes = dbgframes

    @staticmethod
    def _speed2color(speed):
        # setup ranges, where 0 maps to first number, 1.0 maps to second
        color_ranges = {
            'r': (0.8, 1.0),
            'g': (0.8, 0.0),
            'b': (0.8, 0.0)
        }

        def scale(inval, color):
            range = color_ranges[color]
            scaled = range[0] + (range[1] - range[0]) * inval
            return min(1, max(0, scaled))  # constrain to 0-1

        r = scale(speed, 'r')
        g = scale(speed, 'g')
        b = scale(speed, 'b')
        return (r,g,b, 0.5)

    def plot_trace(self):
        # one minute per subplot
        numplots = self._track.len_minutes
        fig = plt.figure(figsize=(12,2*(numplots+1)))

        # Draw the legend at the top
        self.draw_legend(plt.subplot(numplots+1, 1, 1))

        for i in range(numplots):
            ax = plt.subplot(numplots+1, 1, i+2)
            self._plot_trace_portion(ax, start_min=i, end_min=i+1)

        return fig

    def draw_legend(self, legend_ax):
        # Make a legend with proxy artists
        xpos_artist = lines.Line2D([],[], color='orange')
        ypos_artist = lines.Line2D([],[], color='limegreen')
        numpts_artist = lines.Line2D([],[], color='purple', linewidth=1)
        frozen_artist = patches.Rectangle((0,0), 1, 1, fc='lightblue', ec='None')
        missing_artist = patches.Rectangle((0,0), 1, 1, fc='yellow', ec='None')
        lost_artist = patches.Rectangle((0,0), 1, 1, fc='red', ec='None')
        # Place it in center of top "subplot" area
        legend_ax.legend(
            [xpos_artist, ypos_artist, numpts_artist,
                frozen_artist, missing_artist, lost_artist],
            ['x-pos', 'y-pos', '# Detection pts',
                'Frozen', 'Missing', 'Lost'],
            loc='center',
            fontsize=12,
            ncol=4,
        )
        legend_ax.axis('off')
        format_axis(legend_ax)

    def plot_invalidheatmap(self):
        title = "Map of shame (loc of invalid data)"

        plt.figure(figsize=(4, 4))
        ax = plt.gca()

        ax.set_title(title)
        format_axis(ax)

        nbins = 50

        badpoints = (self._track.df.valid != True)  # noqa: E712
        heatmaps.plot_heatmap(ax, self._track.df.x[badpoints], self._track.df.y[badpoints], nbins=nbins)

    def plot_heatmap(self, plot_type='overall'):
        assert plot_type in ('per-minute', 'per-phase', 'overall')
        if plot_type == 'per-minute':
            numplots = self._track.len_minutes
        elif plot_type == 'per-phase':
            numplots = self._track.num_phases()
            phase_starts = self._track.phase_starts()
            phase_ends = phase_starts[1:] + [2**30]
        elif plot_type == 'overall':
            numplots = 1

        numrows = int(math.ceil(numplots / 10.0))
        if plot_type == 'overall':
            plt.figure(figsize=(4, 4))
        else:
            plt.figure(figsize=(2*min(numplots, 10), 2*numrows))

        for i in range(numplots):
            if plot_type == 'per-minute':
                start_min = i
                end_min = i+1
                title = "{}:00-{}:00".format(start_min, end_min)
            elif plot_type == 'per-phase':
                start_min = phase_starts[i]
                end_min = phase_ends[i]
                title = "Phase {} ({}:00-{}:00)".format(i+1, start_min, end_min)
            elif plot_type == 'overall':
                start_min = 0
                end_min = 2**30
                title = "Overall heatmap"

            ax = plt.subplot(numrows, min(numplots, 10), i+1)

            if numplots > 1:
                ax.axes.get_xaxis().set_visible(False)
                ax.axes.get_yaxis().set_visible(False)

            format_axis(ax)

            ax.set_title(title)

            nbins = 50
            start_sec = start_min*60
            end_sec = end_min*60
            heatmaps.plot_heatmap(ax, self._track.df.x[start_sec:end_sec], self._track.df.y[start_sec:end_sec], nbins=nbins)

    def _plot_trace_portion(self, ax, start_min, end_min):
        ''' Parameters:
                start_min, end_min:
                    Integer minutes.
                    Plot should be from start:00 to end:00.
        '''
        # shorthand
        df = self._track.df

        start = start_min * 60
        end = end_min * 60

        time = df.index.to_series()[start:end].values
        #theta = self._track.theta[start:end]
        #speed = self._track.speed[start:end]
        #valid = self._track.valid[start:end]
        lost = df.lost[start:end].values
        missing = df.missing[start:end].values
        frozen = df.frozen[start:end].values
        x = df.x[start:end].values
        y = df.y[start:end].values
        numpts = df.numpts[start:end].values

        # Format nicely
        format_axis(ax)
        ax.axes.get_yaxis().set_visible(False)

        # Get set axes (specifically, we don't want the y-axis to be autoscaled for us)
        ax.axis([start, end, -1.0, 1.0])

        # Mark lost/missing sections
        lost_collection = collections.BrokenBarHCollection.span_where(
            time,
            -1.0, -0.9,
            lost,
            edgecolors='none',
            facecolors='red',
        )
        ax.add_collection(lost_collection)
        missing_collection = collections.BrokenBarHCollection.span_where(
            time,
            -1.0, -0.9,
            missing,
            edgecolors='none',
            facecolors='yellow',
        )
        ax.add_collection(missing_collection)

        # Mark frozen sections
        frozen_collection = collections.BrokenBarHCollection.span_where(
            time,
            -0.85, -0.8,
            frozen,
            edgecolors='none',
            facecolors='lightblue',
        )
        ax.add_collection(frozen_collection)

        # Plot horizontal position
        ax.plot(time, x*2-1, color='orange', label='x position')

        # Plot height
        ax.plot(time, y*2-1, color='limegreen', label='y position')

        # Plot numpts (scaled so 0 = -1.0 (plot bottom), 20 = 1.0 (top))
        ax.plot(time, -1.0+(numpts/10.0), color='purple', linewidth=1, label='# detected points')

        # Add stick plot of movement (where valid)
#        ax.quiver(
#            time, [0] * len(time),
#            speed*np.cos(theta), speed*np.sin(theta),
#            color=[self._speed2color(s) for s in speed],
#            scale=1,  # scale all to a speed of 1, which should be close to max (tank is 1.0x1.0)
#            scale_units='y',
#            width=0.01,
#            units='inches',
#            headlength=0, headwidth=0, headaxislength=0     # no arrowheads
#        )

        # Add markers/links to debugframes if given
        # Get [tracking]:start_frame for proper offset of debug frame numbers into track data here
        start_frame = int(self._track.config['tracking']['start_frame'])
        for dbgframe in self._dbgframes:
            nameparts = dbgframe.name.split('_')
            frameindex = max(0, int(nameparts[1]) - start_frame)  # restrict to index 0 at minimum
            frametime = self._track.df.index[frameindex]
            if start <= frametime < end:
                marker = matplotlib.patches.Circle(
                    (frametime, -1.1), radius=0.08,
                    color='#337AB7',
                    clip_on=False,
                    url=str("/data" / dbgframe.relative_to(config.DATADIR))
                )
                ax.add_artist(marker)


#!/usr/bin/python3

import os
from urllib.request import urlopen
from bs4 import BeautifulSoup
import pandas
import time

# 读取 00.get_metadata.R 获取的相关目录信息
D0 = pandas.read_csv("all_aisixiang_2017-05-24.csv")

# 意外中断时，可以修改 j 的值
j = 0
D = D0[j:]

for i in D['ID']:
    Url = "http://www.aisixiang.com/data/" + str(i) + ".html"
    print(Url)
    try:
        html = urlopen(Url)
    except:
        f1 = open("broken-new.txt", 'a')
        Broken = str(i) + '-' + Url + ',' + '\n'
        f1.write(Broken)
        f1.close
        print(Broken)
        j += 1

        Availability = 3

        f2 = open("Av.txt", 'a')
        f2.write(str(Availability) + '_' + str(i) + ',' + '\n')
        f2.close

    else:
        Soup = BeautifulSoup(html, "html.parser")
        Article = Soup.find(id = "content2")

        Article_page = ''

        if type(Article) == type(None):
            Availability = 0
        else:
            Availability = 1
            Page = Soup.find(class_ = "list_page")

            if type(Page) == type(None):
                Article_page = Article_page + Article.get_text()
            else:
                Page_number = Page.find_all("a")
                N = int(Page_number[-2].get_text())
                for k in range(1, N+1):
                    Url2 = Url[:(len(Url)-5)] + '-' + str(k) + '.html'
                    print(Url2)
                    try:
                        html2 = urlopen(Url2)
                    except:
                        k += 1
                        ft2 = open("broken2.txt", 'a')
                        Broken2 = str(i) + '-' + Url2 + ',' + '\n'
                        ft2.write(Broken2)
                        ft2.close
                        print(Broken2)
                    else:
                        Soup2 = BeautifulSoup(html2, "html.parser")
                        Article = Soup2.find(id = "content2")
                        Article_page = Article_page + Article.get_text()
                        time.sleep(1)
        Name = str(Availability) + '-' + str(i) + '-' + D0.iloc[j,0] + '.txt'
        Name = Name.replace('/','')
        f = open(Name, 'w')
        f.write(Article_page)
        f.close()
        print(Name + '\n')
        j += 1
        time.sleep(1)

        f2 = open("Av.txt", 'a')
        f2.write(str(Availability) + '_' + str(i) + ',' + '\n')
        f2.close