#!/usr/bin/env python2
"""
Author        :Julio Sanz
Website       :www.elarraydejota.com
Email         :juliojosesb@gmail.com
Description   :Script to create a graph about swap usage
Dependencies  :Python 2.x, matplotlib
Usage         :python swap.py
License       :GPLv3
"""

import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import csv
from datetime import datetime
import matplotlib.dates

# ======================
# VARIABLES
# ======================

# Aesthetic parameters
plt.rcParams.update({'font.size': 8})
plt.rcParams['lines.linewidth'] = 1.5
time_format = matplotlib.dates.DateFormatter('%H:%M:%S')
plt.gca().xaxis.set_major_formatter(time_format)
plt.gcf().autofmt_xdate()

# Time (column 0)
x = []
# Data arrays
swap_free = []
swap_used = []

# ======================
# FUNCTIONS
# ======================

def generate_graph():
    with open('../../data/swap.dat', 'r') as csvfile:
        data_source = csv.reader(csvfile, delimiter=' ', skipinitialspace=True)
        for row in data_source:
            # [0] column is a time column
            # Convert to datetime data type
            a = datetime.strptime((row[0]),'%H:%M:%S')
            x.append((a))
            # The remaining columns contain data
            swap_free.append(str(int(row[1])/1024))
            swap_used.append(str(int(row[2])/1024))

    # Plot lines
    plt.plot(x,swap_used, label='Used', color='r', antialiased=True)
    plt.plot(x,swap_free, label='Free', color='g', antialiased=True)

    # Graph properties
    plt.xlabel('Time',fontstyle='italic')
    plt.ylabel('SWAP (MB)',fontstyle='italic')
    plt.title('SWAP usage graph')
    plt.grid(linewidth=0.4, antialiased=True)
    plt.legend(loc='upper center', bbox_to_anchor=(0.5, -0.15), ncol=2, fancybox=True, shadow=True)
    plt.autoscale(True)

    # Graph saved to PNG file
    plt.savefig('../../graphs/swap.png', bbox_inches='tight')
    #plt.show()

# ======================
# MAIN
# ======================

if __name__ == '__main__':
    generate_graph()


# Authors: Alexandre Gramfort <alexandre.gramfort@inria.fr>
#          Vincent Michel <vincent.michel@inria.fr>
#          Gilles Louppe <g.louppe@gmail.com>
#
# License: BSD 3 clause

"""Recursive feature elimination for feature ranking"""

import warnings
import numpy as np
from ..utils import check_X_y, safe_sqr
from ..utils.metaestimators import if_delegate_has_method
from ..base import BaseEstimator
from ..base import MetaEstimatorMixin
from ..base import clone
from ..base import is_classifier
from ..cross_validation import check_cv
from ..cross_validation import _safe_split, _score
from ..metrics.scorer import check_scoring
from .base import SelectorMixin

class RFE(BaseEstimator, MetaEstimatorMixin, SelectorMixin):
    """Feature ranking with recursive feature elimination.

    Given an external estimator that assigns weights to features (e.g., the
    coefficients of a linear model), the goal of recursive feature elimination
    (RFE) is to select features by recursively considering smaller and smaller
    sets of features. First, the estimator is trained on the initial set of
    features and weights are assigned to each one of them. Then, features whose
    absolute weights are the smallest are pruned from the current set features.
    That procedure is recursively repeated on the pruned set until the desired
    number of features to select is eventually reached.

    Read more in the :ref:`User Guide <rfe>`.

    Parameters
    ----------
    estimator : object
        A supervised learning estimator with a `fit` method that updates a
        `coef_` attribute that holds the fitted parameters. Important features
        must correspond to high absolute values in the `coef_` array.

        For instance, this is the case for most supervised learning
        algorithms such as Support Vector Classifiers and Generalized
        Linear Models from the `svm` and `linear_model` modules.

    n_features_to_select : int or None (default=None)
        The number of features to select. If `None`, half of the features
        are selected.

    step : int or float, optional (default=1)
        If greater than or equal to 1, then `step` corresponds to the (integer)
        number of features to remove at each iteration.
        If within (0.0, 1.0), then `step` corresponds to the percentage
        (rounded down) of features to remove at each iteration.

    estimator_params : dict
        Parameters for the external estimator.
        This attribute is deprecated as of version 0.16 and will be removed in
        0.18. Use estimator initialisation or set_params method instead.

    verbose : int, default=0
        Controls verbosity of output.

    Attributes
    ----------
    n_features_ : int
        The number of selected features.

    support_ : array of shape [n_features]
        The mask of selected features.

    ranking_ : array of shape [n_features]
        The feature ranking, such that ``ranking_[i]`` corresponds to the
        ranking position of the i-th feature. Selected (i.e., estimated
        best) features are assigned rank 1.

    estimator_ : object
        The external estimator fit on the reduced dataset.

    Examples
    --------
    The following example shows how to retrieve the 5 right informative
    features in the Friedman #1 dataset.

    >>> from sklearn.datasets import make_friedman1
    >>> from sklearn.feature_selection import RFE
    >>> from sklearn.svm import SVR
    >>> X, y = make_friedman1(n_samples=50, n_features=10, random_state=0)
    >>> estimator = SVR(kernel="linear")
    >>> selector = RFE(estimator, 5, step=1)
    >>> selector = selector.fit(X, y)
    >>> selector.support_ # doctest: +NORMALIZE_WHITESPACE
    array([ True,  True,  True,  True,  True,
            False, False, False, False, False], dtype=bool)
    >>> selector.ranking_
    array([1, 1, 1, 1, 1, 6, 4, 3, 2, 5])

    References
    ----------

    .. [1] Guyon, I., Weston, J., Barnhill, S., & Vapnik, V., "Gene selection
           for cancer classification using support vector machines",
           Mach. Learn., 46(1-3), 389--422, 2002.
    """
    def __init__(self, estimator, n_features_to_select=None, step=1,
                 estimator_params=None, verbose=0):
        self.estimator = estimator
        self.n_features_to_select = n_features_to_select
        self.step = step
        self.estimator_params = estimator_params
        self.verbose = verbose

    @property
    def _estimator_type(self):
        return self.estimator._estimator_type

    def fit(self, X, y):
        """Fit the RFE model and then the underlying estimator on the selected
           features.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape = [n_samples, n_features]
            The training input samples.

        y : array-like, shape = [n_samples]
            The target values.
        """
        return self._fit(X, y)

    def _fit(self, X, y, step_score=None):
        X, y = check_X_y(X, y, "csc")
        # Initialization
        n_features = X.shape[1]
        if self.n_features_to_select is None:
            n_features_to_select = n_features // 2
        else:
            n_features_to_select = self.n_features_to_select

        if 0.0 < self.step < 1.0:
            step = int(max(1, self.step * n_features))
        else:
            step = int(self.step)
        if step <= 0:
            raise ValueError("Step must be >0")

        if self.estimator_params is not None:
            warnings.warn("The parameter 'estimator_params' is deprecated as "
                          "of version 0.16 and will be removed in 0.18. The "
                          "parameter is no longer necessary because the value "
                          "is set via the estimator initialisation or "
                          "set_params method.", DeprecationWarning)

        support_ = np.ones(n_features, dtype=np.bool)
        ranking_ = np.ones(n_features, dtype=np.int)

        if step_score:
            self.scores_ = []

        # Elimination
        while np.sum(support_) > n_features_to_select:
            # Remaining features
            features = np.arange(n_features)[support_]

            # Rank the remaining features
            estimator = clone(self.estimator)
            if self.estimator_params:
                estimator.set_params(**self.estimator_params)
            if self.verbose > 0:
                print("Fitting estimator with %d features." % np.sum(support_))

            estimator.fit(X[:, features], y)

            # Get coefs
            if hasattr(estimator, 'coef_'):
                coefs = estimator.coef_
            elif hasattr(estimator, 'feature_importances_'):
                coefs = estimator.feature_importances_
            else:
                raise RuntimeError('The classifier does not expose '
                                   '"coef_" or "feature_importances_" '
                                   'attributes')

            # Get ranks
            if coefs.ndim > 1:
                ranks = np.argsort(safe_sqr(coefs).sum(axis=0))
            else:
                ranks = np.argsort(safe_sqr(coefs))

            # for sparse case ranks is matrix
            ranks = np.ravel(ranks)

            # Eliminate the worse features
            threshold = min(step, np.sum(support_) - n_features_to_select)

            # Compute step score on the previous selection iteration
            # because 'estimator' must use features
            # that have not been eliminated yet
            if step_score:
                self.scores_.append(step_score(estimator, features))
            support_[features[ranks][:threshold]] = False
            ranking_[np.logical_not(support_)] += 1

        # Set final attributes
        features = np.arange(n_features)[support_]
        self.estimator_ = clone(self.estimator)
        if self.estimator_params:
            self.estimator_.set_params(**self.estimator_params)
        self.estimator_.fit(X[:, features], y)

        # Compute step score when only n_features_to_select features left
        if step_score:
            self.scores_.append(step_score(self.estimator_, features))
        self.n_features_ = support_.sum()
        self.support_ = support_
        self.ranking_ = ranking_

        return self

    @if_delegate_has_method(delegate='estimator')
    def predict(self, X):
        """Reduce X to the selected features and then predict using the
           underlying estimator.

        Parameters
        ----------
        X : array of shape [n_samples, n_features]
            The input samples.

        Returns
        -------
        y : array of shape [n_samples]
            The predicted target values.
        """
        return self.estimator_.predict(self.transform(X))

    @if_delegate_has_method(delegate='estimator')
    def score(self, X, y):
        """Reduce X to the selected features and then return the score of the
           underlying estimator.

        Parameters
        ----------
        X : array of shape [n_samples, n_features]
            The input samples.

        y : array of shape [n_samples]
            The target values.
        """
        return self.estimator_.score(self.transform(X), y)

    def _get_support_mask(self):
        return self.support_

    @if_delegate_has_method(delegate='estimator')
    def decision_function(self, X):
        return self.estimator_.decision_function(self.transform(X))

    @if_delegate_has_method(delegate='estimator')
    def predict_proba(self, X):
        return self.estimator_.predict_proba(self.transform(X))

    @if_delegate_has_method(delegate='estimator')
    def predict_log_proba(self, X):
        return self.estimator_.predict_log_proba(self.transform(X))

class RFECV(RFE, MetaEstimatorMixin):
    """Feature ranking with recursive feature elimination and cross-validated
    selection of the best number of features.

    Read more in the :ref:`User Guide <rfe>`.

    Parameters
    ----------
    estimator : object
        A supervised learning estimator with a `fit` method that updates a
        `coef_` attribute that holds the fitted parameters. Important features
        must correspond to high absolute values in the `coef_` array.

        For instance, this is the case for most supervised learning
        algorithms such as Support Vector Classifiers and Generalized
        Linear Models from the `svm` and `linear_model` modules.

    step : int or float, optional (default=1)
        If greater than or equal to 1, then `step` corresponds to the (integer)
        number of features to remove at each iteration.
        If within (0.0, 1.0), then `step` corresponds to the percentage
        (rounded down) of features to remove at each iteration.

    cv : int, cross-validation generator or an iterable, optional
        Determines the cross-validation splitting strategy.
        Possible inputs for cv are:

        - None, to use the default 3-fold cross-validation,
        - integer, to specify the number of folds.
        - An object to be used as a cross-validation generator.
        - An iterable yielding train/test splits.

        For integer/None inputs, if ``y`` is binary or multiclass,
        :class:`StratifiedKFold` used. If the estimator is a classifier
        or if ``y`` is neither binary nor multiclass, :class:`KFold` is used.

        Refer :ref:`User Guide <cross_validation>` for the various
        cross-validation strategies that can be used here.

    scoring : string, callable or None, optional, default: None
        A string (see model evaluation documentation) or
        a scorer callable object / function with signature
        ``scorer(estimator, X, y)``.

    estimator_params : dict
        Parameters for the external estimator.
        This attribute is deprecated as of version 0.16 and will be removed in
        0.18. Use estimator initialisation or set_params method instead.

    verbose : int, default=0
        Controls verbosity of output.

    Attributes
    ----------
    n_features_ : int
        The number of selected features with cross-validation.

    support_ : array of shape [n_features]
        The mask of selected features.

    ranking_ : array of shape [n_features]
        The feature ranking, such that `ranking_[i]`
        corresponds to the ranking
        position of the i-th feature.
        Selected (i.e., estimated best)
        features are assigned rank 1.

    grid_scores_ : array of shape [n_subsets_of_features]
        The cross-validation scores such that
        ``grid_scores_[i]`` corresponds to
        the CV score of the i-th subset of features.

    estimator_ : object
        The external estimator fit on the reduced dataset.

    Notes
    -----
    The size of ``grid_scores_`` is equal to ceil((n_features - 1) / step) + 1,
    where step is the number of features removed at each iteration.

    Examples
    --------
    The following example shows how to retrieve the a-priori not known 5
    informative features in the Friedman #1 dataset.

    >>> from sklearn.datasets import make_friedman1
    >>> from sklearn.feature_selection import RFECV
    >>> from sklearn.svm import SVR
    >>> X, y = make_friedman1(n_samples=50, n_features=10, random_state=0)
    >>> estimator = SVR(kernel="linear")
    >>> selector = RFECV(estimator, step=1, cv=5)
    >>> selector = selector.fit(X, y)
    >>> selector.support_ # doctest: +NORMALIZE_WHITESPACE
    array([ True,  True,  True,  True,  True,
            False, False, False, False, False], dtype=bool)
    >>> selector.ranking_
    array([1, 1, 1, 1, 1, 6, 4, 3, 2, 5])

    References
    ----------

    .. [1] Guyon, I., Weston, J., Barnhill, S., & Vapnik, V., "Gene selection
           for cancer classification using support vector machines",
           Mach. Learn., 46(1-3), 389--422, 2002.
    """
    def __init__(self, estimator, step=1, cv=None, scoring=None,
                 estimator_params=None, verbose=0):
        self.estimator = estimator
        self.step = step
        self.cv = cv
        self.scoring = scoring
        self.estimator_params = estimator_params
        self.verbose = verbose

    def fit(self, X, y):
        """Fit the RFE model and automatically tune the number of selected
           features.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape = [n_samples, n_features]
            Training vector, where `n_samples` is the number of samples and
            `n_features` is the total number of features.

        y : array-like, shape = [n_samples]
            Target values (integers for classification, real numbers for
            regression).
        """
        X, y = check_X_y(X, y, "csr")
        if self.estimator_params is not None:
            warnings.warn("The parameter 'estimator_params' is deprecated as "
                          "of version 0.16 and will be removed in 0.18. "
                          "The parameter is no longer necessary because the "
                          "value is set via the estimator initialisation or "
                          "set_params method.", DeprecationWarning)
        # Initialization
        cv = check_cv(self.cv, X, y, is_classifier(self.estimator))
        scorer = check_scoring(self.estimator, scoring=self.scoring)
        n_features = X.shape[1]
        n_features_to_select = 1

        # Determine the number of subsets of features
        scores = []

        # Cross-validation
        for n, (train, test) in enumerate(cv):
            X_train, y_train = _safe_split(self.estimator, X, y, train)
            X_test, y_test = _safe_split(self.estimator, X, y, test, train)

            rfe = RFE(estimator=self.estimator,
                      n_features_to_select=n_features_to_select,
                      step=self.step, estimator_params=self.estimator_params,
                      verbose=self.verbose - 1)

            rfe._fit(X_train, y_train, lambda estimator, features:
                     _score(estimator, X_test[:, features], y_test, scorer))
            scores.append(np.array(rfe.scores_[::-1]).reshape(1, -1))
        scores = np.sum(np.concatenate(scores, 0), 0)
        # The index in 'scores' when 'n_features' features are selected
        n_feature_index = np.ceil((n_features - n_features_to_select) /
                                  float(self.step))
        n_features_to_select = max(n_features_to_select,
                                   n_features - ((n_feature_index -
                                                 np.argmax(scores)) *
                                                 self.step))
        # Re-execute an elimination with best_k over the whole set
        rfe = RFE(estimator=self.estimator,
                  n_features_to_select=n_features_to_select,
                  step=self.step, estimator_params=self.estimator_params)

        rfe.fit(X, y)

        # Set final attributes
        self.support_ = rfe.support_
        self.n_features_ = rfe.n_features_
        self.ranking_ = rfe.ranking_
        self.estimator_ = clone(self.estimator)
        if self.estimator_params:
            self.estimator_.set_params(**self.estimator_params)
        self.estimator_.fit(self.transform(X), y)

        # Fixing a normalization error, n is equal to len(cv) - 1
        # here, the scores are normalized by len(cv)
        self.grid_scores_ = scores / len(cv)
        return self


# coding: utf-8
import numpy as np
import pandas as pd
import time
from datetime import datetime, timedelta, date
from time import sleep, gmtime, strftime
from pandas import DataFrame, Series, read_csv

from salesforce_bulk_api import SalesforceBulkJob
from SalesforceBulkQuery import *
from simple_salesforce import *

###################################################################################################

# Salesforce Credentials

# Creates SimpleSalesforce Login Instance
sf = Salesforce(username='', password='', security_token='', sandbox=, client_id='')

###################################################################################################

def getBlankDF():
    return pd.DataFrame(np.nan, index=[], columns=[])

def NameCaseAsTitles(x):
    if (str(x).isupper() or str(x).islower()) and '@' not in str(x):
        return str(x).title()
    else:
        return x

def getDate(days):
    return(datetime.today() - timedelta(days=days)).strftime('%Y-%m-%dT00:00:00z') # YYYY-MM-DDThh:mm:ssz

def SFNulls(df, FillWith='#N/A'):
    """
        Description: Fills 0's and NAN's with "#N/A" which is the value that the Salesforce Bulk API recognizes as Null.
        Parameters:
            df = Pandas.DataFrame
        Recognizes 'float64', 'int64', and 'int32' data types.
    """
    df.apply(lambda s: pd.to_numeric(s, errors='ignore'))
    NumCol = df.columns.values.tolist()
    for col in NumCol:
        df[col] = df[col].replace(0, np.NAN).fillna('%s' % FillWith)

def SFQuery(SOQL: str, InList=None, LowerHeaders=True, CheckParentChild=True, KeepAttributes=False):
    """
        Description: Queries Salesforce returning all results in a pandas dataframe.  This also sets all possible data types to numbers and sets column headers to lower case. If using InList, this functionality is built with pandas dataframe columns in mind to help simplify filtering from other SOQL results.
        Parameters:
            SOQL = Salesforce SOQL Statement
            InList* = List of items for an "IN" filter. Apex SOQL - "SELECT Id, Name FROM Account Where Id IN :ids"
                SOQL parameter must be written out to the point where the : would be set in a SOQL query in Apex.
                EX: SFQuery("SELECT Id, Name From Contact WHERE FirstName = 'Alex' and Id IN", IdsList)
                InList format - ['id1', 'id2', 'id2', 'id3', 'id3', 'id4', 'id5'] becomes ('id1', 'id2', 'id3', 'id4', 'id5')
                I usually use this with a dataframe column.
                ex: "SFQuery("Select Id, Name From Contact Where Id In", InList=list(your_dataframe['column_name']))
            LowerHeader = Returns Dataframe with column headers lowercase, defaulted true for previous projects
            CheckParentChild = This checks for the relationships by looking for the ordered dictionaries returned by Salesforce.  It loops through to ensure reached the end of the line if stepping through multiple parent relationships.  Turn off if queries need to run slighly faster.

            InList* - This is not an efficent use of api calls.  There are limitations to the length of the queries so this is capped out at a default of 300 elements.  Nested Select statements in the where clause is a more efficent use for api calls but there are always tradeoffs.  At some point it would make more sense to utilize tuples, but unfortunately salesforce did not like the format with the last comma.
    """
    def basicSOQL(SOQLstr : str):
        # formats the Salesforce ordered dictionary into a pandas dataframe
        try:
            od = sf.query_all("%s" % SOQLstr)
            items = {val: dict(od['records'][val]) for val in range(len(od['records'])) }
            res = DataFrame.from_dict(items, orient='index')
            if LowerHeaders == True:
                res.columns = map(str.lower, res.columns)
            return res.apply(lambda s: pd.to_numeric(s, errors='ignore'))
        except ValueError:
            pass
    def CreateFilterStr(ListToStr):
        # creates a string from a list
        # ['id1', 'id2', 'id3', 'id4', 'id5'] -> ('id1', 'id2', 'id3', 'id4', 'id5')
        resStr = "("
        r = 0
        for rl in ListToStr:
            if rl is not None:
                if r == 0:
                    resStr += "'"+str(rl)+"'"
                    r = 1
                elif r == 1:
                    resStr += ",'"+str(rl)+"'"
        resStr += ")"
        return resStr
    def BatchQueryList(toBatchList):
        # filters the list of duplicates then batches the lists in groups
        # [('id1', 'id2', 'id3', id4', 'id5'),('id6', 'id7', 'id8', 'id9', 'id10')]
        batchSize = 300
        newList = list(set(toBatchList))
        listSize = len(newList)
        startPoint = 0
        endPoint = batchSize
        res = []
        while startPoint < listSize:
            tempStr = CreateFilterStr(newList[startPoint:endPoint])
            res.append([tempStr])
            startPoint = endPoint
            endPoint += batchSize
        return res
    def InListQuery(SOQL, InList):
        # runs a query for each list from the batched lists and stacks the results
        filterLists = BatchQueryList(InList)

        resDF = None
        i = 0
        for i in range(0,len(filterLists)):
            tempDF = basicSOQL(SOQLstr = "%s %s" % (SOQL, filterLists[i][0]))
            try: resDF = resDF.append(tempDF, ignore_index=True)
            except AttributeError: resDF = tempDF
            i += 1
        return resDF
    def getChildRecords(obj, row):
        if row == None:
            return None

        size = row.get('totalSize')
        records = row.get('records')
        tempDic = {}

        for i in range(0,size):
            tempDic[i] = {}
            for field in records[i].keys():
                try:
                    records[i].get(field).keys()
                    continue
                except AttributeError:
                    pass
                tempDic[i][obj + '.' + field] = records[i].get(field)
        return tempDic
    def getParentRecords(field, row):
        if row == None:
            return None
        else:
            return row.get(field)

    rs = None
    if InList == None:
        rs = basicSOQL(SOQL)
    else:
        InList = list(InList)
        rs = InListQuery(SOQL, InList)

    # Drops the attributes column passed through by Salesforce
    if CheckParentChild == False and KeepAttributes == False:
        rs = rs.drop(['attributes'], axis=1)

    while CheckParentChild:
        CheckParentChild = False

        indexCols = []
        for col in rs:
            obj = None
            relationship = None
            for i in range(len(rs[col])):
                # scans down each column until finding an ordered dict to parse
                if rs[col][i] == None:
                    continue
                try:
                    if rs[col][i].get('type') != None and col == 'attributes':
                        if KeepAttributes == False:
                            rs = rs.drop([col], axis=1)
                        break
                except AttributeError:
                    indexCols.append(col) # will use this later for creating a multi indexed dataframe
                    break

                # Determines whether parent or child query and the object type
                try:
                    obj = rs[col][i].get('attributes').get('type')
                    relationship = 'Parent'
                except:
                    pass
                try:
                    obj = rs[col][i].get('records')[0].get('attributes').get('type')
                    relationship = 'Child'
                except:
                    pass
                break

            if relationship == 'Child' and obj != None:
                rs[col] = rs.apply(lambda row: getChildRecords(obj, row[col]), axis=1)

            elif relationship == 'Parent' and obj != None:
                fields = []
                for i in range(len(rs[col])):
                    if rs[col][i] != None:
                        fields.extend(list(rs[col][i].keys()))
                        fields = list(set(fields))

                if KeepAttributes == False:
                    try:
                        fields.remove('attributes')
                    except ValueError:
                        pass
                for field in fields:
                    rs[obj + '.' + field] = rs.apply(lambda row: getParentRecords(field, row[col]), axis=1)
                rs = rs.drop([col], axis=1)

                CheckParentChild = True

        # next I'd like to setup an option for child relationship queries to return a multi indexed dataframe
        # print(indexCols)
    return rs

def SFFormat(df, SObject, EnforceNulls=False):
    """
        Description: Looks up data types and dynamically formats columns to a correct format for the Bulk Api. Returns error messages for invalid data types or column headers.  If EnforceNulls is true fills all blanks with #N/A, if false will set blanks to ''.
        Parameters:
            df = Pandas.DataFrame
            SObject = Type of object for the upload. Ex: 'Account'
            EnforceNulls = If true will fill blanks with #N/A to set as null in Salesforce

        *Currently only formats dates and datetimes
    """
    NoFieldError = ''
    InvalidDataError = ''

    df.columns = map(str.lower, df.columns)
    fieldDict = getattr(sf, '%s' % SObject).describe()["fields"]
    numFields = len(fieldDict)

    NumCol = df.columns.values.tolist()
    for col in NumCol:
        i = 0
        for x in fieldDict:
            if x['name'].lower() == col:
                dtype = x['type']
                length = x['length']
                try:
                    if dtype == 'date':
                        df[col] = pd.to_datetime(df[col]).dt.strftime('%Y-%m-%d').replace(to_replace='NaT', value='#N/A')
                    elif dtype == 'datetime':
                        df[col] = pd.to_datetime(df[col]).dt.strftime('%Y-%m-%dT%H:%M:%S').replace(to_replace='NaT', value='#N/A')
                except ValueError:
                    InvalidDataError += ("Invalid "+dtype+" : "+col+"\n")
                break
            i += 1
            if i >= numFields:
                NoFieldError += (SObject+" does not contain : "+col+"\n")

    SFNulls(df)
    if EnforceNulls == False:
        for col in NumCol:
            df[col] = df[col].replace('#N/A','')
    errors = NoFieldError+InvalidDataError
    if len(errors) > 0:
        return(errors)
    else:
        return('No Errors')

def SFUpload(df, UploadType, Sobject, batchSize=49995, hangtime=0):
    """
        Description: Upload a pandas dataframe through the Salesforce Bulk API in batches of 50k. Can run either an insert or update to the listed Sobject.  Sobject and UploadType must be listed as a string. ex: 'Update', 'Account'
        Parameters:
            df         = Pandas.DataFrame
            UploadType = Update or Insert
            Sobject    = Salesforce object in the upload. Ex - Accounts, Contact
            batchSize  = Number of rows that the upload will run before submitting the next group of rows in the dataset. Defaults to 49,995 (5 batches of 9999)
            hangtime   = Number of seconds to wait before uploading a new batch. Defaults to 0.
    """

    if len(df) == 0:
        return

    startRow = 0
    endRow = batchSize

    while startRow < len(df):
        upload = df[startRow:endRow]

        Headers = upload.columns.tolist()
        Data = upload.to_records(index=False)
        job = SalesforceBulkJob(UploadType, Sobject, salesforce=sf)
        job.upload(Headers,Data)

        startRow = endRow
        endRow = startRow + batchSize
        time.sleep(hangtime)

def SFBulkQuery(SObject, SOQL):
    """
        Description: Runs a query through the bulk api.  Creates, Tracks, and Closes the Request and returns the results as a Pandas Dataframe.  Currently there are lots of slighly obnoxious messages to help with tracking the current status.
        Parameters:
            SObject = Salesforce Object, ex: Account, Contact
            SOQL    = Salesforce SOQL Statement for bulk query
    """
    sfbulk = SalesforceBulk(sessionId=sf.session_id, host=sf.sf_instance)
    job = sfbulk.create_query_job(SObject, contentType='CSV')
    batch = sfbulk.query(job, SOQL)
    while not sfbulk.is_batch_done(job, batch):
        time.sleep(10)
    sfbulk.close_job(job)
    res = sfbulk.get_batch_result_iter(job, batch)
    return res


# Author: Robert McGibbon <rmcgibbo@gmail.com>
# Contributors:
# Copyright (c) 2014, Stanford University
# All rights reserved.
"""Scan the implied timescales of MarkovStateModels with respect to lag time.

This command will build a series of MarkovStateModels at different lag times,
and save a file to disk containing the relaxation timescales of each of the
models.

A plot of these data can then be used to choose the lag time [1].

References
----------
.. [1] Beauchamp, Kyle A., et al. "MSMBuilder2: modeling conformational
   dynamics on the picosecond to millisecond scale." J. Chem. Theory.
   Comput. 7.10 (2011): 3412-3419.
"""

# -----------------------------------------------------------------------------
# Imports
# -----------------------------------------------------------------------------

from __future__ import print_function, division, absolute_import
from os.path import splitext
import sys
import json

import pandas as pd

from ..dataset import dataset
from ..cmdline import Command, argument, argument_group, rangetype, FlagAction
from ..msm import MarkovStateModel, implied_timescales

class ImpliedTimescales(Command):
    _group = 'MSM'
    _concrete = True
    description = __doc__
    lag_times = argument('-l', '--lag_times', default='1:10', help='''Range
        of lag times. Specify as 'start:stop' or 'start:stop:step. The
        endpoints are inclusive.''', type=rangetype)
    inp = argument(
        '-i', '--inp', help='''Path to input dataset, a collection of 1D
        integer sequences (such as the output from clustering)''',
        required=True)
    out = argument('--out', help='''Output file''',
        default='timescales.csv')
    fmt = argument('--fmt', help='Output file format', default='csv',
        choices=('csv', 'json', 'excel'))
    _extensions = {'csv': '.csv', 'json': '.json', 'excel': '.xlsx'}
    n_jobs = argument('--n_jobs', help='Number of parallel processes',
        default=1, type=int)

    p = argument_group('MSM parameters')
    n_timescales = p.add_argument('--n_timescales', default=10, help='''
        The number of dynamical timescales to calculate when diagonalizing
        the transition matrix.''',  type=int)
    reversible_type = p.add_argument('--reversible_type', help='''
        Method by which the reversibility of the transition matrix
        is enforced. 'mle' uses a maximum likelihood method that is
        solved by numerical optimization, and 'transpose'
        uses a more restrictive (but less computationally complex)
        direct symmetrization of the expected number of counts.''',
        choices=('mle', 'transpose'), default='mle')
    ergodic_cutoff = p.add_argument('--ergodic_cutoff', default=1, help='''
        Only the maximal strongly ergodic subgraph of the data is used to build
        an MSM. Ergodicity is determined by ensuring that each state is
        accessible from each other state via one or more paths involving edges
        with a number of observed directed counts greater than or equal to
        ``ergodic_cutoff``. Not that by setting ``ergodic_cutoff`` to 0, this
        trimming is effectively turned off.''',  type=int)
    prior_counts = p.add_argument('--prior_counts', help='''Add a number
        of "pseudo counts" to each entry in the counts matrix. When
        prior_counts == 0 (default), the assigned transition probability
        between two states with no observed transitions will be zero, whereas
        when prior_counts > 0, even this unobserved transitions will be
        given nonzero probability.''', type=float, default=0)
    verbose = p.add_argument('--verbose', default=True,
        help='Enable verbose printout', action=FlagAction)

    def __init__(self, args):
        self.args = args

    def start(self):
        kwargs = {
            'n_timescales': self.args.n_timescales,
            'reversible_type': self.args.reversible_type,
            'ergodic_cutoff': self.args.ergodic_cutoff,
            'prior_counts': self.args.prior_counts,
            'verbose': self.args.verbose,
        }

        with dataset(self.args.inp, mode='r') as ds:
            model = MarkovStateModel(**kwargs)
            lines = implied_timescales(
                ds, lag_times=self.args.lag_times,
                n_timescales=self.args.n_timescales,
                msm=model,
                n_jobs=self.args.n_jobs,
                verbose=self.args.verbose)

        cols = ['Timescale %d' % (d+1) for d in range(len(lines[0]))]
        df = pd.DataFrame(data=lines, columns=cols)
        df['Lag Time'] = self.args.lag_times
        df = df.reindex_axis(sorted(df.columns), axis=1)
        self.write_output(df)

    def write_output(self, df):
        outfile = splitext(self.args.out)[0] + self._extensions[self.args.fmt]

        print('Writing %s' % outfile)
        if self.args.fmt == 'csv':
            df.to_csv(outfile)
        elif self.args.fmt == 'json':
            with open(outfile, 'w') as f:
                json.dump(df.to_dict(orient='records'), f)
        elif self.args.fmt == 'excel':
            df.to_excel(outfile)
        else:
            raise RuntimeError('unknown fmt: %s' % fmt)
        print('All done!')


# Copyright 2018 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

"""Helper functions for creating the training graph and plotting.

"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os
from matplotlib.backends.backend_pdf import PdfPages
import matplotlib.pyplot as plt
import numpy as np
import tensorflow as tf

import ensembles  # pylint: disable=g-bad-import-order

np.seterr(invalid="ignore")

def get_place_cell_ensembles(
    env_size, neurons_seed, targets_type, lstm_init_type, n_pc, pc_scale):
  """Create the ensembles for the Place cells."""
  place_cell_ensembles = [
      ensembles.PlaceCellEnsemble(
          n,
          stdev=s,
          pos_min=-env_size / 2.0,
          pos_max=env_size / 2.0,
          seed=neurons_seed,
          soft_targets=targets_type,
          soft_init=lstm_init_type)
      for n, s in zip(n_pc, pc_scale)
  ]
  return place_cell_ensembles

def get_head_direction_ensembles(
    neurons_seed, targets_type, lstm_init_type, n_hdc, hdc_concentration):
  """Create the ensembles for the Head direction cells."""
  head_direction_ensembles = [
      ensembles.HeadDirectionCellEnsemble(
          n,
          concentration=con,
          seed=neurons_seed,
          soft_targets=targets_type,
          soft_init=lstm_init_type)
      for n, con in zip(n_hdc, hdc_concentration)
  ]
  return head_direction_ensembles

def encode_initial_conditions(init_pos, init_hd, place_cell_ensembles,
                              head_direction_ensembles):
  initial_conds = []
  for ens in place_cell_ensembles:
    initial_conds.append(
        tf.squeeze(ens.get_init(init_pos[:, tf.newaxis, :]), axis=1))
  for ens in head_direction_ensembles:
    initial_conds.append(
        tf.squeeze(ens.get_init(init_hd[:, tf.newaxis, :]), axis=1))
  return initial_conds

def encode_targets(target_pos, target_hd, place_cell_ensembles,
                   head_direction_ensembles):
  ensembles_targets = []
  for ens in place_cell_ensembles:
    ensembles_targets.append(ens.get_targets(target_pos))
  for ens in head_direction_ensembles:
    ensembles_targets.append(ens.get_targets(target_hd))
  return ensembles_targets

def clip_all_gradients(g, var, limit):
  # print(var.name)
  return (tf.clip_by_value(g, -limit, limit), var)

def clip_bottleneck_gradient(g, var, limit):
  if ("bottleneck" in var.name or "pc_logits" in var.name):
    return (tf.clip_by_value(g, -limit, limit), var)
  else:
    return (g, var)

def no_clipping(g, var):
  return (g, var)

def concat_dict(acc, new_data):
  """Dictionary concatenation function."""

  def to_array(kk):
    if isinstance(kk, np.ndarray):
      return kk
    else:
      return np.asarray([kk])

  for k, v in new_data.iteritems():
    if isinstance(v, dict):
      if k in acc:
        acc[k] = concat_dict(acc[k], v)
      else:
        acc[k] = concat_dict(dict(), v)
    else:
      v = to_array(v)
      if k in acc:
        acc[k] = np.concatenate([acc[k], v])
      else:
        acc[k] = np.copy(v)
  return acc

def get_scores_and_plot(scorer,
                        data_abs_xy,
                        activations,
                        directory,
                        filename,
                        plot_graphs=True,  # pylint: disable=unused-argument
                        nbins=20,  # pylint: disable=unused-argument
                        cm="jet",
                        sort_by_score_60=True):
  """Plotting function."""

  # Concatenate all trajectories
  xy = data_abs_xy.reshape(-1, data_abs_xy.shape[-1])
  act = activations.reshape(-1, activations.shape[-1])
  n_units = act.shape[1]
  # Get the rate-map for each unit
  s = [
      scorer.calculate_ratemap(xy[:, 0], xy[:, 1], act[:, i])
      for i in xrange(n_units)
  ]
  # Get the scores
  score_60, score_90, max_60_mask, max_90_mask, sac = zip(
      *[scorer.get_scores(rate_map) for rate_map in s])
  # Separations
  # separations = map(np.mean, max_60_mask)
  # Sort by score if desired
  if sort_by_score_60:
    ordering = np.argsort(-np.array(score_60))
  else:
    ordering = range(n_units)
  # Plot
  cols = 16
  rows = int(np.ceil(n_units / cols))
  fig = plt.figure(figsize=(24, rows * 4))
  for i in xrange(n_units):
    rf = plt.subplot(rows * 2, cols, i + 1)
    acr = plt.subplot(rows * 2, cols, n_units + i + 1)
    if i < n_units:
      index = ordering[i]
      title = "%d (%.2f)" % (index, score_60[index])
      # Plot the activation maps
      scorer.plot_ratemap(s[index], ax=rf, title=title, cmap=cm)
      # Plot the autocorrelation of the activation maps
      scorer.plot_sac(
          sac[index],
          mask_params=max_60_mask[index],
          ax=acr,
          title=title,
          cmap=cm)
  # Save
  if not os.path.exists(directory):
    os.makedirs(directory)
  with PdfPages(os.path.join(directory, filename), "w") as f:
    plt.savefig(f, format="pdf")
  plt.close(fig)
  return (np.asarray(score_60), np.asarray(score_90),
          np.asarray(map(np.mean, max_60_mask)),
          np.asarray(map(np.mean, max_90_mask)))


# vim: set encoding=utf-8

#  Copyright (c) 2016 Intel Corporation
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
#

"""Sample LDA/Groupby example"""
import unittest
from sparktkregtests.lib import sparktk_test
import numpy

class LDAExample(sparktk_test.SparkTKTestCase):

    def test_lda_example(self):
        """LDA demo from examples directory"""

        # this is a full worked example of lda and groupby
        # with known correct values
        data = [['nytimes', 'harry', 3], ['nytimes', 'economy', 35], ['nytimes', 'jobs', 40], ['nytimes', 'magic', 1],
                ['nytimes', 'realestate', 15], ['nytimes', 'movies', 6], ['economist', 'economy', 50],
                ['economist', 'jobs', 35], ['economist', 'realestate', 20], ['economist', 'movies', 1],
                ['economist', 'harry', 1], ['economist', 'magic', 1], ['harrypotter', 'harry', 40],
                ['harrypotter', 'magic', 30], ['harrypotter', 'chamber', 20], ['harrypotter', 'secrets', 30]]
        frame = self.context.frame.create(
            data,
            schema=[('doc_id', str),
                    ('word_id', str),
                    ('word_count', long)])

        model = self.context.models.clustering.lda.train(
                frame, "doc_id", "word_id", "word_count", max_iterations=3, num_topics=2)

        doc_results = model.topics_given_doc_frame
        word_results = model.word_given_topics_frame

        doc_results.rename_columns({'topic_probabilities': 'lda_results_doc'})
        word_results.rename_columns(
            {'topic_probabilities': 'lda_results_word'})

        frame = frame.join_left(
            doc_results, left_on="doc_id", right_on="doc_id")
        frame = frame.join_left(
            word_results, left_on="word_id", right_on="word_id")

        # similar to calling predict on a model
        frame.dot_product(
            ['lda_results_doc'], ['lda_results_word'], 'lda_score')

        word_hist = frame.histogram('word_count', 4)
        lda_hist = frame.histogram('lda_score', 2)

        group_frame = frame.group_by(
            'word_id_L',
            {'word_count': self.context.agg.histogram(
                cutoffs=word_hist.cutoffs,
                include_lowest=True,
                strict_binning=False),
             'lda_score': self.context.agg.histogram(lda_hist.cutoffs)})
        pandas = group_frame.to_pandas()

        for (index, row) in pandas.iterrows():
            if str(row["word_id_L"]) == "magic":
                numpy.testing.assert_equal(list(row["word_count_HISTOGRAM"]), [float(2.0/3.0), 0, float(1.0/3.0), 0])

if __name__ == "__main__":
    unittest.main()