# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

"""Tests for DNNLinearCombinedEstimators."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import functools
import tempfile

import numpy as np
import tensorflow as tf

from tensorflow.contrib.learn.python.learn.estimators import _sklearn
from tensorflow.contrib.learn.python.learn.estimators import estimator_test_utils

def _get_quantile_based_buckets(feature_values, num_buckets):
  quantiles = np.percentile(
      np.array(feature_values), ([100 * (i + 1.) / (num_buckets + 1.)
                                  for i in range(num_buckets)]))
  return list(quantiles)

def _prepare_iris_data_for_logistic_regression():
  # Converts iris data to a logistic regression problem.
  iris = tf.contrib.learn.datasets.load_iris()
  ids = np.where((iris.target == 0) | (iris.target == 1))
  iris = tf.contrib.learn.datasets.base.Dataset(data=iris.data[ids],
                                                target=iris.target[ids])
  return iris

def _iris_input_multiclass_fn():
  iris = tf.contrib.learn.datasets.load_iris()
  return {
      'feature': tf.constant(iris.data, dtype=tf.float32)
  }, tf.constant(iris.target, shape=[150, 1], dtype=tf.int32)

def _iris_input_logistic_fn():
  iris = _prepare_iris_data_for_logistic_regression()
  return {
      'feature': tf.constant(iris.data, dtype=tf.float32)
  }, tf.constant(iris.target, shape=[100, 1], dtype=tf.int32)

class DNNLinearCombinedClassifierTest(tf.test.TestCase):

  def testEstimatorContract(self):
    estimator_test_utils.assert_estimator_contract(
        self, tf.contrib.learn.DNNLinearCombinedClassifier)

  def testLogisticRegression_MatrixData(self):
    """Tests binary classification using matrix data as input."""
    iris = _prepare_iris_data_for_logistic_regression()
    cont_features = [
        tf.contrib.layers.real_valued_column('feature', dimension=4)]
    bucketized_feature = [tf.contrib.layers.bucketized_column(
        cont_features[0], _get_quantile_based_buckets(iris.data, 10))]

    classifier = tf.contrib.learn.DNNLinearCombinedClassifier(
        linear_feature_columns=bucketized_feature,
        dnn_feature_columns=cont_features,
        dnn_hidden_units=[3, 3])

    classifier.fit(input_fn=_iris_input_logistic_fn, steps=100)
    scores = classifier.evaluate(input_fn=_iris_input_logistic_fn, steps=100)
    self.assertGreater(scores['accuracy'], 0.9)

  def testLogisticRegression_TensorData(self):
    """Tests binary classification using Tensor data as input."""
    def _input_fn():
      iris = _prepare_iris_data_for_logistic_regression()
      features = {}
      for i in range(4):
        # The following shows how to provide the Tensor data for
        # RealValuedColumns.
        features.update({
            str(i): tf.reshape(tf.constant(iris.data[:, i], dtype=tf.float32),
                               [-1, 1])})
      # The following shows how to provide the SparseTensor data for
      # a SparseColumn.
      features['dummy_sparse_column'] = tf.SparseTensor(
          values=['en', 'fr', 'zh'],
          indices=[[0, 0], [0, 1], [60, 0]],
          shape=[len(iris.target), 2])
      target = tf.reshape(tf.constant(iris.target, dtype=tf.int32), [-1, 1])
      return features, target

    iris = _prepare_iris_data_for_logistic_regression()
    cont_features = [tf.contrib.layers.real_valued_column(str(i))
                     for i in range(4)]
    linear_features = [
        tf.contrib.layers.bucketized_column(
            cont_features[i], _get_quantile_based_buckets(iris.data[:, str(i)],
                                                          10)) for i in range(4)
    ]
    linear_features.append(tf.contrib.layers.sparse_column_with_hash_bucket(
        'dummy_sparse_column', hash_bucket_size=100))

    classifier = tf.contrib.learn.DNNLinearCombinedClassifier(
        linear_feature_columns=linear_features,
        dnn_feature_columns=cont_features,
        dnn_hidden_units=[3, 3])

    classifier.fit(input_fn=_input_fn, steps=100)
    scores = classifier.evaluate(input_fn=_input_fn, steps=100)
    self.assertGreater(scores['accuracy'], 0.9)

  def testTrainWithPartitionedVariables(self):
    """Tests training with partitioned variables."""
    def _input_fn():
      features = {
          'language': tf.SparseTensor(values=['en', 'fr', 'zh'],
                                      indices=[[0, 0], [0, 1], [2, 0]],
                                      shape=[3, 2])
      }
      target = tf.constant([[1], [0], [0]])
      return features, target

    sparse_features = [
        # The given hash_bucket_size results in variables larger than the
        # default min_slice_size attribute, so the variables are partitioned.
        tf.contrib.layers.sparse_column_with_hash_bucket('language',
                                                         hash_bucket_size=2e7)
    ]
    embedding_features = [
        tf.contrib.layers.embedding_column(sparse_features[0], dimension=1)
    ]

    classifier = tf.contrib.learn.DNNLinearCombinedClassifier(
        linear_feature_columns=sparse_features,
        dnn_feature_columns=embedding_features,
        dnn_hidden_units=[3, 3],
        # Because we did not start a distributed cluster, we need to pass an
        # empty ClusterSpec, otherwise the device_setter will look for
        # distributed jobs, such as "/job:ps" which are not present.
        config=tf.contrib.learn.RunConfig(
            num_ps_replicas=2, cluster_spec=tf.train.ClusterSpec({})))

    classifier.fit(input_fn=_input_fn, steps=100)
    scores = classifier.evaluate(input_fn=_input_fn, steps=1)
    self.assertGreater(scores['accuracy'], 0.9)

  def testMultiClass(self):
    """Tests multi-class classification using matrix data as input.

    Please see testLogisticRegression_TensorData() for how to use Tensor
    data as input instead.
    """
    iris = tf.contrib.learn.datasets.load_iris()
    cont_features = [
        tf.contrib.layers.real_valued_column('feature', dimension=4)]
    bucketized_features = [
        tf.contrib.layers.bucketized_column(
            cont_features[0], _get_quantile_based_buckets(iris.data, 10))]

    classifier = tf.contrib.learn.DNNLinearCombinedClassifier(
        n_classes=3,
        linear_feature_columns=bucketized_features,
        dnn_feature_columns=cont_features,
        dnn_hidden_units=[3, 3])

    classifier.fit(input_fn=_iris_input_multiclass_fn, steps=100)
    scores = classifier.evaluate(input_fn=_iris_input_multiclass_fn, steps=100)
    self.assertGreater(scores['accuracy'], 0.9)

  def testWeightAndBiasNames(self):
    """Tests that weight and bias names haven't changed."""
    iris = tf.contrib.learn.datasets.load_iris()
    cont_features = [
        tf.contrib.layers.real_valued_column('feature', dimension=4)]
    bucketized_features = [
        tf.contrib.layers.bucketized_column(
            cont_features[0], _get_quantile_based_buckets(iris.data, 10))]

    classifier = tf.contrib.learn.DNNLinearCombinedClassifier(
        n_classes=3,
        linear_feature_columns=bucketized_features,
        dnn_feature_columns=cont_features,
        dnn_hidden_units=[3, 3])
    classifier.fit(input_fn=_iris_input_multiclass_fn, steps=100)

    self.assertEquals(4, len(classifier.dnn_bias_))
    self.assertEquals(3, len(classifier.dnn_weights_))
    self.assertEquals(3, len(classifier.linear_bias_))
    self.assertEquals(44, len(classifier.linear_weights_))

  def testLoss(self):
    """Tests loss calculation."""

    def _input_fn_train():
      # Create 4 rows, one of them (y = x), three of them (y=Not(x))
      # The logistic prediction should be (y = 0.25).
      target = tf.constant([[1], [0], [0], [0]])
      features = {
          'x': tf.ones(shape=[4, 1], dtype=tf.float32),
      }
      return features, target

    classifier = tf.contrib.learn.DNNLinearCombinedClassifier(
        n_classes=2,
        linear_feature_columns=[tf.contrib.layers.real_valued_column('x')],
        dnn_feature_columns=[tf.contrib.layers.real_valued_column('x')],
        dnn_hidden_units=[3, 3],
        config=tf.contrib.learn.RunConfig(tf_random_seed=1))

    classifier.fit(input_fn=_input_fn_train, steps=100)
    scores = classifier.evaluate(input_fn=_input_fn_train, steps=1)
    # Cross entropy = -0.25*log(0.25)-0.75*log(0.75) = 0.562
    self.assertAlmostEqual(scores['loss'], 0.562, delta=0.1)

  def testLossWithWeights(self):
    """Tests loss calculation with weights."""

    def _input_fn_train():
      # 4 rows with equal weight, one of them (y = x), three of them (y=Not(x))
      # The logistic prediction should be (y = 0.25).
      target = tf.constant([[1.], [0.], [0.], [0.]])
      features = {
          'x': tf.ones(shape=[4, 1], dtype=tf.float32),
          'w': tf.constant([[1.], [1.], [1.], [1.]])
      }
      return features, target

    def _input_fn_eval():
      # 4 rows, with different weights.
      target = tf.constant([[1.], [0.], [0.], [0.]])
      features = {
          'x': tf.ones(shape=[4, 1], dtype=tf.float32),
          'w': tf.constant([[7.], [1.], [1.], [1.]])
      }
      return features, target

    classifier = tf.contrib.learn.DNNLinearCombinedClassifier(
        weight_column_name='w',
        n_classes=2,
        linear_feature_columns=[tf.contrib.layers.real_valued_column('x')],
        dnn_feature_columns=[tf.contrib.layers.real_valued_column('x')],
        dnn_hidden_units=[3, 3],
        config=tf.contrib.learn.RunConfig(tf_random_seed=1))
    classifier.fit(input_fn=_input_fn_train, steps=100)
    scores = classifier.evaluate(input_fn=_input_fn_eval, steps=1)
    # Weighted cross entropy = (-7*log(0.25)-3*log(0.75))/10 = 1.06
    self.assertAlmostEqual(scores['loss'], 1.06, delta=0.1)

  def testTrainWithWeights(self):
    """Tests training with given weight column."""

    def _input_fn_train():
      # Create 4 rows, one of them (y = x), three of them (y=Not(x))
      # First row has more weight than others. Model should fit (y=x) better
      # than (y=Not(x)) due to the relative higher weight of the first row.
      target = tf.constant([[1], [0], [0], [0]])
      features = {
          'x': tf.ones(shape=[4, 1], dtype=tf.float32),
          'w': tf.constant([[100.], [3.], [2.], [2.]])
      }
      return features, target

    def _input_fn_eval():
      # Create 4 rows (y = x)
      target = tf.constant([[1], [1], [1], [1]])
      features = {
          'x': tf.ones(shape=[4, 1], dtype=tf.float32),
          'w': tf.constant([[1.], [1.], [1.], [1.]])
      }
      return features, target

    classifier = tf.contrib.learn.DNNLinearCombinedClassifier(
        weight_column_name='w',
        linear_feature_columns=[tf.contrib.layers.real_valued_column('x')],
        dnn_feature_columns=[tf.contrib.layers.real_valued_column('x')],
        dnn_hidden_units=[3, 3],
        config=tf.contrib.learn.RunConfig(tf_random_seed=1))
    classifier.fit(input_fn=_input_fn_train, steps=100)
    scores = classifier.evaluate(input_fn=_input_fn_eval, steps=1)
    # The model should learn (y = x) because of the weights, so the accuracy
    # should be close to 1.
    self.assertGreater(scores['accuracy'], 0.9)

  def testCustomOptimizerByObject(self):
    """Tests binary classification using matrix data as input."""
    iris = _prepare_iris_data_for_logistic_regression()
    cont_features = [
        tf.contrib.layers.real_valued_column('feature', dimension=4)]
    bucketized_features = [
        tf.contrib.layers.bucketized_column(
            cont_features[0], _get_quantile_based_buckets(iris.data, 10))]

    classifier = tf.contrib.learn.DNNLinearCombinedClassifier(
        linear_feature_columns=bucketized_features,
        linear_optimizer=tf.train.FtrlOptimizer(learning_rate=0.1),
        dnn_feature_columns=cont_features,
        dnn_hidden_units=[3, 3],
        dnn_optimizer=tf.train.AdagradOptimizer(learning_rate=0.1))

    classifier.fit(input_fn=_iris_input_logistic_fn, steps=100)
    scores = classifier.evaluate(input_fn=_iris_input_logistic_fn, steps=100)
    self.assertGreater(scores['accuracy'], 0.9)

  def testCustomOptimizerByString(self):
    """Tests binary classification using matrix data as input."""
    iris = _prepare_iris_data_for_logistic_regression()
    cont_features = [
        tf.contrib.layers.real_valued_column('feature', dimension=4)]
    bucketized_features = [
        tf.contrib.layers.bucketized_column(
            cont_features[0], _get_quantile_based_buckets(iris.data, 10))]

    classifier = tf.contrib.learn.DNNLinearCombinedClassifier(
        linear_feature_columns=bucketized_features,
        linear_optimizer='Ftrl',
        dnn_feature_columns=cont_features,
        dnn_hidden_units=[3, 3],
        dnn_optimizer='Adagrad')

    classifier.fit(input_fn=_iris_input_logistic_fn, steps=100)
    scores = classifier.evaluate(input_fn=_iris_input_logistic_fn, steps=100)
    self.assertGreater(scores['accuracy'], 0.9)

  def testCustomOptimizerByFunction(self):
    """Tests binary classification using matrix data as input."""
    iris = _prepare_iris_data_for_logistic_regression()
    cont_features = [
        tf.contrib.layers.real_valued_column('feature', dimension=4)
    ]
    bucketized_features = [
        tf.contrib.layers.bucketized_column(
            cont_features[0], _get_quantile_based_buckets(iris.data, 10))
    ]

    def _optimizer_exp_decay():
      global_step = tf.contrib.framework.get_global_step()
      learning_rate = tf.train.exponential_decay(learning_rate=0.1,
                                                 global_step=global_step,
                                                 decay_steps=100,
                                                 decay_rate=0.001)
      return tf.train.AdagradOptimizer(learning_rate=learning_rate)

    classifier = tf.contrib.learn.DNNLinearCombinedClassifier(
        linear_feature_columns=bucketized_features,
        linear_optimizer=_optimizer_exp_decay,
        dnn_feature_columns=cont_features,
        dnn_hidden_units=[3, 3],
        dnn_optimizer=_optimizer_exp_decay)

    classifier.fit(input_fn=_iris_input_logistic_fn, steps=100)
    scores = classifier.evaluate(input_fn=_iris_input_logistic_fn, steps=100)
    self.assertGreater(scores['accuracy'], 0.8)

  def testPredict(self):
    """Tests weight column in evaluation."""
    def _input_fn_train():
      # Create 4 rows, one of them (y = x), three of them (y=Not(x))
      target = tf.constant([[1], [0], [0], [0]])
      features = {'x': tf.ones(shape=[4, 1], dtype=tf.float32)}
      return features, target

    def _input_fn_predict():
      y = tf.train.limit_epochs(
          tf.ones(shape=[4, 1], dtype=tf.float32), num_epochs=1)
      features = {'x': y}
      return features

    classifier = tf.contrib.learn.DNNLinearCombinedClassifier(
        linear_feature_columns=[tf.contrib.layers.real_valued_column('x')],
        dnn_feature_columns=[tf.contrib.layers.real_valued_column('x')],
        dnn_hidden_units=[3, 3])

    classifier.fit(input_fn=_input_fn_train, steps=100)

    probs = list(classifier.predict_proba(input_fn=_input_fn_predict))
    self.assertAllClose([[0.75, 0.25]] * 4, probs, 0.05)
    classes = list(classifier.predict(input_fn=_input_fn_predict))
    self.assertListEqual([0] * 4, classes)

  def testCustomMetrics(self):
    """Tests custom evaluation metrics."""

    def _input_fn(num_epochs=None):
      # Create 4 rows, one of them (y = x), three of them (y=Not(x))
      target = tf.constant([[1], [0], [0], [0]])
      features = {
          'x': tf.train.limit_epochs(
              tf.ones(shape=[4, 1], dtype=tf.float32), num_epochs=num_epochs)}
      return features, target

    def _my_metric_op(predictions, targets):
      # For the case of binary classification, the 2nd column of "predictions"
      # denotes the model predictions.
      targets = tf.to_float(targets)
      predictions = tf.slice(predictions, [0, 1], [-1, 1])
      return tf.reduce_sum(tf.mul(predictions, targets))

    classifier = tf.contrib.learn.DNNLinearCombinedClassifier(
        linear_feature_columns=[tf.contrib.layers.real_valued_column('x')],
        dnn_feature_columns=[tf.contrib.layers.real_valued_column('x')],
        dnn_hidden_units=[3, 3])

    classifier.fit(input_fn=_input_fn, steps=100)
    scores = classifier.evaluate(
        input_fn=_input_fn,
        steps=100,
        metrics={
            'my_accuracy': tf.contrib.metrics.streaming_accuracy,
            ('my_precision', 'classes'): tf.contrib.metrics.streaming_precision,
            ('my_metric', 'probabilities'): _my_metric_op
        })
    self.assertTrue(
        set(['loss', 'my_accuracy', 'my_precision', 'my_metric'
            ]).issubset(set(scores.keys())))
    predict_input_fn = functools.partial(_input_fn, num_epochs=1)
    predictions = np.array(
        list(classifier.predict(input_fn=predict_input_fn)))
    self.assertEqual(_sklearn.accuracy_score([1, 0, 0, 0], predictions),
                     scores['my_accuracy'])

    # Test the case where the 2nd element of the key is neither "classes" nor
    # "probabilities".
    with self.assertRaises(KeyError):
      classifier.evaluate(
          input_fn=_input_fn,
          steps=100,
          metrics={('bad_name', 'bad_type'): tf.contrib.metrics.streaming_auc})

    # Test the case where the tuple of the key doesn't have 2 elements.
    with self.assertRaises(ValueError):
      classifier.evaluate(
          input_fn=_input_fn,
          steps=100,
          metrics={
              ('bad_length_name', 'classes', 'bad_length'):
                  tf.contrib.metrics.streaming_accuracy
          })

  def testVariableQuery(self):
    """Tests bias is centered or not."""
    def _input_fn_train():
      # Create 4 rows, three (y = x), one (y=Not(x))
      target = tf.constant([[1], [1], [1], [0]])
      features = {'x': tf.ones(shape=[4, 1], dtype=tf.float32),}
      return features, target

    classifier = tf.contrib.learn.DNNLinearCombinedClassifier(
        linear_feature_columns=[tf.contrib.layers.real_valued_column('x')],
        dnn_feature_columns=[tf.contrib.layers.real_valued_column('x')],
        dnn_hidden_units=[3, 3])

    classifier.fit(input_fn=_input_fn_train, steps=500)
    var_names = classifier.get_variable_names()
    self.assertGreater(len(var_names), 3)
    for name in var_names:
      classifier.get_variable_value(name)

  def testCenteredBias(self):
    """Tests bias is centered or not."""
    def _input_fn_train():
      # Create 4 rows, three (y = x), one (y=Not(x))
      target = tf.constant([[1], [1], [1], [0]])
      features = {'x': tf.ones(shape=[4, 1], dtype=tf.float32),}
      return features, target

    classifier = tf.contrib.learn.DNNLinearCombinedClassifier(
        linear_feature_columns=[tf.contrib.layers.real_valued_column('x')],
        dnn_feature_columns=[tf.contrib.layers.real_valued_column('x')],
        dnn_hidden_units=[3, 3])

    classifier.fit(input_fn=_input_fn_train, steps=500)
    # logodds(0.75) = 1.09861228867
    self.assertAlmostEqual(
        1.0986,
        float(classifier.get_variable_value('centered_bias_weight')[0]),
        places=2)

  def testDisableCenteredBias(self):
    """Tests bias is centered or not."""
    def _input_fn_train():
      # Create 4 rows, three (y = x), one (y=Not(x))
      target = tf.constant([[1], [1], [1], [0]])
      features = {'x': tf.ones(shape=[4, 1], dtype=tf.float32),}
      return features, target

    classifier = tf.contrib.learn.DNNLinearCombinedClassifier(
        linear_feature_columns=[tf.contrib.layers.real_valued_column('x')],
        dnn_feature_columns=[tf.contrib.layers.real_valued_column('x')],
        dnn_hidden_units=[3, 3],
        enable_centered_bias=False)

    classifier.fit(input_fn=_input_fn_train, steps=500)
    self.assertFalse('centered_bias_weight' in classifier.get_variable_names())

  def testLinearOnly(self):
    """Tests that linear-only instantiation works."""
    def input_fn():
      return {
          'age': tf.constant([1]),
          'language': tf.SparseTensor(values=['english'],
                                      indices=[[0, 0]],
                                      shape=[1, 1])
      }, tf.constant([[1]])

    language = tf.contrib.layers.sparse_column_with_hash_bucket('language', 100)
    age = tf.contrib.layers.real_valued_column('age')

    classifier = tf.contrib.learn.DNNLinearCombinedClassifier(
        linear_feature_columns=[age, language])
    classifier.fit(input_fn=input_fn, steps=100)
    loss1 = classifier.evaluate(input_fn=input_fn, steps=1)['loss']
    classifier.fit(input_fn=input_fn, steps=200)
    loss2 = classifier.evaluate(input_fn=input_fn, steps=1)['loss']
    self.assertLess(loss2, loss1)
    self.assertLess(loss2, 0.01)
    self.assertTrue('centered_bias_weight' in classifier.get_variable_names())

    self.assertNotIn('dnn/logits/biases', classifier.get_variable_names())
    self.assertNotIn('dnn/logits/weights', classifier.get_variable_names())
    self.assertEquals(1, len(classifier.linear_bias_))
    self.assertEquals(2, len(classifier.linear_weights_))
    self.assertEquals(1, len(classifier.linear_weights_['linear/age/weight']))
    self.assertEquals(
        100, len(classifier.linear_weights_['linear/language/weights']))

  def testLinearOnlyOneFeature(self):
    """Tests that linear-only instantiation works for one feature only."""
    def input_fn():
      return {
          'language': tf.SparseTensor(values=['english'],
                                      indices=[[0, 0]],
                                      shape=[1, 1])
      }, tf.constant([[1]])

    language = tf.contrib.layers.sparse_column_with_hash_bucket('language', 99)

    classifier = tf.contrib.learn.DNNLinearCombinedClassifier(
        linear_feature_columns=[language])
    classifier.fit(input_fn=input_fn, steps=100)
    loss1 = classifier.evaluate(input_fn=input_fn, steps=1)['loss']
    classifier.fit(input_fn=input_fn, steps=200)
    loss2 = classifier.evaluate(input_fn=input_fn, steps=1)['loss']
    self.assertLess(loss2, loss1)
    self.assertLess(loss2, 0.01)
    self.assertTrue('centered_bias_weight' in classifier.get_variable_names())

    self.assertNotIn('dnn/logits/biases', classifier.get_variable_names())
    self.assertNotIn('dnn/logits/weights', classifier.get_variable_names())
    self.assertEquals(1, len(classifier.linear_bias_))
    self.assertEquals(99, len(classifier.linear_weights_))

  def testDNNOnly(self):
    """Tests that DNN-only instantiation works."""
    cont_features = [
        tf.contrib.layers.real_valued_column('feature', dimension=4)]

    classifier = tf.contrib.learn.DNNLinearCombinedClassifier(
        n_classes=3, dnn_feature_columns=cont_features, dnn_hidden_units=[3, 3])

    classifier.fit(input_fn=_iris_input_multiclass_fn, steps=1000)
    classifier.evaluate(input_fn=_iris_input_multiclass_fn, steps=100)
    self.assertTrue('centered_bias_weight' in classifier.get_variable_names())

    self.assertEquals(4, len(classifier.dnn_bias_))
    self.assertEquals(3, len(classifier.dnn_weights_))
    self.assertNotIn('linear/bias_weight', classifier.get_variable_names())
    self.assertNotIn('linear/feature_BUCKETIZED_weights',
                     classifier.get_variable_names())

  def testDNNWeightsBiasesNames(self):
    """Tests the names of DNN weights and biases in the checkpoints."""
    def _input_fn_train():
      # Create 4 rows, three (y = x), one (y=Not(x))
      target = tf.constant([[1], [1], [1], [0]])
      features = {'x': tf.ones(shape=[4, 1], dtype=tf.float32),}
      return features, target
    classifier = tf.contrib.learn.DNNLinearCombinedClassifier(
        linear_feature_columns=[tf.contrib.layers.real_valued_column('x')],
        dnn_feature_columns=[tf.contrib.layers.real_valued_column('x')],
        dnn_hidden_units=[3, 3])

    classifier.fit(input_fn=_input_fn_train, steps=5)
    # hiddenlayer_0/weights,hiddenlayer_1/weights and dnn_logits/weights.
    self.assertEquals(3, len(classifier.dnn_weights_))
    # hiddenlayer_0/biases, hiddenlayer_1/biases, dnn_logits/biases,
    # centered_bias_weight.
    self.assertEquals(4, len(classifier.dnn_bias_))

class DNNLinearCombinedRegressorTest(tf.test.TestCase):

  def testEstimatorContract(self):
    estimator_test_utils.assert_estimator_contract(
        self, tf.contrib.learn.DNNLinearCombinedRegressor)

  def testRegression_MatrixData(self):
    """Tests regression using matrix data as input."""
    cont_features = [
        tf.contrib.layers.real_valued_column('feature', dimension=4)]

    regressor = tf.contrib.learn.DNNLinearCombinedRegressor(
        linear_feature_columns=cont_features,
        dnn_feature_columns=cont_features,
        dnn_hidden_units=[3, 3],
        config=tf.contrib.learn.RunConfig(tf_random_seed=1))

    regressor.fit(input_fn=_iris_input_logistic_fn, steps=100)
    scores = regressor.evaluate(input_fn=_iris_input_logistic_fn, steps=1)
    self.assertLess(scores['loss'], 0.3)

  def testRegression_TensorData(self):
    """Tests regression using tensor data as input."""
    def _input_fn():
      # Create 4 rows of (y = x)
      target = tf.constant([[100.], [3.], [2.], [2.]])
      features = {'x': tf.constant([[100.], [3.], [2.], [2.]])}
      return features, target

    classifier = tf.contrib.learn.DNNLinearCombinedRegressor(
        linear_feature_columns=[tf.contrib.layers.real_valued_column('x')],
        dnn_feature_columns=[tf.contrib.layers.real_valued_column('x')],
        dnn_hidden_units=[3, 3],
        config=tf.contrib.learn.RunConfig(tf_random_seed=1))

    classifier.fit(input_fn=_input_fn, steps=100)
    classifier.evaluate(input_fn=_input_fn, steps=1)

  def testLoss(self):
    """Tests loss calculation."""

    def _input_fn_train():
      # Create 4 rows, one of them (y = x), three of them (y=Not(x))
      # The algorithm should learn (y = 0.25).
      target = tf.constant([[1.], [0.], [0.], [0.]])
      features = {
          'x': tf.ones(shape=[4, 1], dtype=tf.float32),
      }
      return features, target

    regressor = tf.contrib.learn.DNNLinearCombinedRegressor(
        linear_feature_columns=[tf.contrib.layers.real_valued_column('x')],
        dnn_feature_columns=[tf.contrib.layers.real_valued_column('x')],
        dnn_hidden_units=[3, 3],
        config=tf.contrib.learn.RunConfig(tf_random_seed=1))

    regressor.fit(input_fn=_input_fn_train, steps=100)
    scores = regressor.evaluate(input_fn=_input_fn_train, steps=1)
    # Average square loss = (0.75^2 + 3*0.25^2) / 4 = 0.1875
    self.assertAlmostEqual(scores['loss'], 0.1875, delta=0.1)

  def testLossWithWeights(self):
    """Tests loss calculation with weights."""

    def _input_fn_train():
      # 4 rows with equal weight, one of them (y = x), three of them (y=Not(x))
      # The algorithm should learn (y = 0.25).
      target = tf.constant([[1.], [0.], [0.], [0.]])
      features = {
          'x': tf.ones(shape=[4, 1], dtype=tf.float32),
          'w': tf.constant([[1.], [1.], [1.], [1.]])
      }
      return features, target

    def _input_fn_eval():
      # 4 rows, with different weights.
      target = tf.constant([[1.], [0.], [0.], [0.]])
      features = {
          'x': tf.ones(shape=[4, 1], dtype=tf.float32),
          'w': tf.constant([[7.], [1.], [1.], [1.]])
      }
      return features, target

    regressor = tf.contrib.learn.DNNLinearCombinedRegressor(
        weight_column_name='w',
        linear_feature_columns=[tf.contrib.layers.real_valued_column('x')],
        dnn_feature_columns=[tf.contrib.layers.real_valued_column('x')],
        dnn_hidden_units=[3, 3],
        config=tf.contrib.learn.RunConfig(tf_random_seed=1))

    regressor.fit(input_fn=_input_fn_train, steps=100)
    scores = regressor.evaluate(input_fn=_input_fn_eval, steps=1)
    # Weighted average square loss = (7*0.75^2 + 3*0.25^2) / 10 = 0.4125
    self.assertAlmostEqual(scores['loss'], 0.4125, delta=0.1)

  def testTrainWithWeights(self):
    """Tests training with given weight column."""

    def _input_fn_train():
      # Create 4 rows, one of them (y = x), three of them (y=Not(x))
      # First row has more weight than others. Model should fit (y=x) better
      # than (y=Not(x)) due to the relative higher weight of the first row.
      target = tf.constant([[1.], [0.], [0.], [0.]])
      features = {
          'x': tf.ones(shape=[4, 1], dtype=tf.float32),
          'w': tf.constant([[100.], [3.], [2.], [2.]])
      }
      return features, target

    def _input_fn_eval():
      # Create 4 rows (y = x)
      target = tf.constant([[1.], [1.], [1.], [1.]])
      features = {
          'x': tf.ones(shape=[4, 1], dtype=tf.float32),
          'w': tf.constant([[1.], [1.], [1.], [1.]])
      }
      return features, target

    regressor = tf.contrib.learn.DNNLinearCombinedRegressor(
        weight_column_name='w',
        linear_feature_columns=[tf.contrib.layers.real_valued_column('x')],
        dnn_feature_columns=[tf.contrib.layers.real_valued_column('x')],
        dnn_hidden_units=[3, 3],
        config=tf.contrib.learn.RunConfig(tf_random_seed=1))

    regressor.fit(input_fn=_input_fn_train, steps=100)
    scores = regressor.evaluate(input_fn=_input_fn_eval, steps=1)
    # The model should learn (y = x) because of the weights, so the loss should
    # be close to zero.
    self.assertLess(scores['loss'], 0.2)

  def testPredict_AsIterableFalse(self):
    """Tests predict method with as_iterable=False."""
    target = [1., 0., 0.2]
    def _input_fn(num_epochs=None):
      features = {
          'age': tf.train.limit_epochs(tf.constant([[0.8], [0.15], [0.]]),
                                       num_epochs=num_epochs),
          'language': tf.SparseTensor(values=['en', 'fr', 'zh'],
                                      indices=[[0, 0], [0, 1], [2, 0]],
                                      shape=[3, 2])
      }
      return features, tf.constant(target, dtype=tf.float32)

    language_column = tf.contrib.layers.sparse_column_with_hash_bucket(
        'language', hash_bucket_size=20)

    regressor = tf.contrib.learn.DNNLinearCombinedRegressor(
        linear_feature_columns=[
            language_column,
            tf.contrib.layers.real_valued_column('age')
        ],
        dnn_feature_columns=[
            tf.contrib.layers.embedding_column(language_column, dimension=1),
            tf.contrib.layers.real_valued_column('age')
        ],
        dnn_hidden_units=[3, 3],
        config=tf.contrib.learn.RunConfig(tf_random_seed=1))

    regressor.fit(input_fn=_input_fn, steps=100)

    scores = regressor.evaluate(input_fn=_input_fn, steps=1)
    self.assertLess(scores['loss'], 0.2)
    predictions = regressor.predict(input_fn=_input_fn, as_iterable=False)
    self.assertAllClose(predictions, target, atol=0.2)

  def testPredict_AsIterable(self):
    """Tests predict method with as_iterable=True."""
    target = [1., 0., 0.2]
    def _input_fn(num_epochs=None):
      features = {
          'age': tf.train.limit_epochs(tf.constant([[0.8], [0.15], [0.]]),
                                       num_epochs=num_epochs),
          'language': tf.SparseTensor(values=['en', 'fr', 'zh'],
                                      indices=[[0, 0], [0, 1], [2, 0]],
                                      shape=[3, 2])
      }
      return features, tf.constant(target, dtype=tf.float32)

    language_column = tf.contrib.layers.sparse_column_with_hash_bucket(
        'language', hash_bucket_size=20)

    regressor = tf.contrib.learn.DNNLinearCombinedRegressor(
        linear_feature_columns=[
            language_column,
            tf.contrib.layers.real_valued_column('age')
        ],
        dnn_feature_columns=[
            tf.contrib.layers.embedding_column(language_column, dimension=1),
            tf.contrib.layers.real_valued_column('age')
        ],
        dnn_hidden_units=[3, 3],
        config=tf.contrib.learn.RunConfig(tf_random_seed=1))

    regressor.fit(input_fn=_input_fn, steps=100)

    scores = regressor.evaluate(input_fn=_input_fn, steps=1)
    self.assertLess(scores['loss'], 0.2)
    predict_input_fn = functools.partial(_input_fn, num_epochs=1)
    predictions = list(
        regressor.predict(input_fn=predict_input_fn, as_iterable=True))
    self.assertAllClose(predictions, target, atol=0.2)

  def testCustomMetrics(self):
    """Tests custom evaluation metrics."""
    def _input_fn(num_epochs=None):
      # Create 4 rows, one of them (y = x), three of them (y=Not(x))
      target = tf.constant([[1.], [0.], [0.], [0.]])
      features = {'x': tf.train.limit_epochs(
          tf.ones(shape=[4, 1], dtype=tf.float32), num_epochs=num_epochs)}
      return features, target

    def _my_metric_op(predictions, targets):
      return tf.reduce_sum(tf.mul(predictions, targets))

    regressor = tf.contrib.learn.DNNLinearCombinedRegressor(
        linear_feature_columns=[tf.contrib.layers.real_valued_column('x')],
        dnn_feature_columns=[tf.contrib.layers.real_valued_column('x')],
        dnn_hidden_units=[3, 3],
        config=tf.contrib.learn.RunConfig(tf_random_seed=1))

    regressor.fit(input_fn=_input_fn, steps=100)
    scores = regressor.evaluate(
        input_fn=_input_fn,
        steps=1,
        metrics={
            'my_error': tf.contrib.metrics.streaming_mean_squared_error,
            'my_metric': _my_metric_op
        })
    self.assertIn('loss', set(scores.keys()))
    self.assertIn('my_error', set(scores.keys()))
    self.assertIn('my_metric', set(scores.keys()))
    predict_input_fn = functools.partial(_input_fn, num_epochs=1)
    predictions = np.array(list(regressor.predict(input_fn=predict_input_fn)))
    self.assertAlmostEqual(
        _sklearn.mean_squared_error(np.array([1, 0, 0, 0]), predictions),
        scores['my_error'])

    # Tests that when the key is a tuple, an error is raised.
    with self.assertRaises(KeyError):
      regressor.evaluate(
          input_fn=_input_fn,
          steps=1,
          metrics={('my_error', 'predictions'
                   ): tf.contrib.metrics.streaming_mean_squared_error})

  def testTrainSaveLoad(self):
    """Tests regression with restarting training / evaluate."""
    def _input_fn(num_epochs=None):
      # Create 4 rows of (y = x)
      target = tf.constant([[100.], [3.], [2.], [2.]])
      features = {'x': tf.train.limit_epochs(
          tf.constant([[100.], [3.], [2.], [2.]]), num_epochs=num_epochs)}
      return features, target

    model_dir = tempfile.mkdtemp()
    # pylint: disable=g-long-lambda
    new_estimator = lambda: tf.contrib.learn.DNNLinearCombinedRegressor(
        linear_feature_columns=[tf.contrib.layers.real_valued_column('x')],
        dnn_feature_columns=[tf.contrib.layers.real_valued_column('x')],
        dnn_hidden_units=[3, 3],
        model_dir=model_dir,
        config=tf.contrib.learn.RunConfig(tf_random_seed=1))

    predict_input_fn = functools.partial(_input_fn, num_epochs=1)
    classifier = new_estimator()
    classifier.fit(input_fn=_input_fn, steps=100)
    predictions = list(classifier.predict(input_fn=predict_input_fn))
    del classifier

    classifier = new_estimator()
    predictions2 = list(classifier.predict(input_fn=predict_input_fn))
    self.assertAllClose(predictions, predictions2)

  def testTrainWithPartitionedVariables(self):
    """Tests training with partitioned variables."""
    def _input_fn(num_epochs=None):
      features = {
          'age': tf.train.limit_epochs(tf.constant([[0.8], [0.15], [0.]]),
                                       num_epochs=num_epochs),
          'language': tf.SparseTensor(values=['en', 'fr', 'zh'],
                                      indices=[[0, 0], [0, 1], [2, 0]],
                                      shape=[3, 2])
      }
      return features, tf.constant([1., 0., 0.2], dtype=tf.float32)

    # The given hash_bucket_size results in variables larger than the
    # default min_slice_size attribute, so the variables are partitioned.
    language_column = tf.contrib.layers.sparse_column_with_hash_bucket(
        'language', hash_bucket_size=2e7)

    regressor = tf.contrib.learn.DNNLinearCombinedRegressor(
        linear_feature_columns=[
            language_column,
            tf.contrib.layers.real_valued_column('age')
        ],
        dnn_feature_columns=[
            tf.contrib.layers.embedding_column(language_column, dimension=1),
            tf.contrib.layers.real_valued_column('age')
        ],
        dnn_hidden_units=[3, 3],
        # Because we did not start a distributed cluster, we need to pass an
        # empty ClusterSpec, otherwise the device_setter will look for
        # distributed jobs, such as "/job:ps" which are not present.
        config=tf.contrib.learn.RunConfig(
            num_ps_replicas=2, cluster_spec=tf.train.ClusterSpec({}),
            tf_random_seed=1))

    regressor.fit(input_fn=_input_fn, steps=100)

    scores = regressor.evaluate(input_fn=_input_fn, steps=1)
    self.assertLess(scores['loss'], 0.2)

  def testDisableCenteredBias(self):
    """Tests that we can disable centered bias."""
    def _input_fn(num_epochs=None):
      features = {
          'age': tf.train.limit_epochs(tf.constant([[0.8], [0.15], [0.]]),
                                       num_epochs=num_epochs),
          'language': tf.SparseTensor(values=['en', 'fr', 'zh'],
                                      indices=[[0, 0], [0, 1], [2, 0]],
                                      shape=[3, 2])
      }
      return features, tf.constant([1., 0., 0.2], dtype=tf.float32)

    language_column = tf.contrib.layers.sparse_column_with_hash_bucket(
        'language', hash_bucket_size=20)

    regressor = tf.contrib.learn.DNNLinearCombinedRegressor(
        linear_feature_columns=[
            language_column,
            tf.contrib.layers.real_valued_column('age')
        ],
        dnn_feature_columns=[
            tf.contrib.layers.embedding_column(language_column, dimension=1),
            tf.contrib.layers.real_valued_column('age')
        ],
        dnn_hidden_units=[3, 3],
        enable_centered_bias=False,
        config=tf.contrib.learn.RunConfig(tf_random_seed=1))

    regressor.fit(input_fn=_input_fn, steps=100)

    scores = regressor.evaluate(input_fn=_input_fn, steps=1)
    self.assertLess(scores['loss'], 0.2)

  def testLinearOnly(self):
    """Tests linear-only instantiation and training."""
    def _input_fn(num_epochs=None):
      features = {
          'age': tf.train.limit_epochs(tf.constant([[0.8], [0.15], [0.]]),
                                       num_epochs=num_epochs),
          'language': tf.SparseTensor(values=['en', 'fr', 'zh'],
                                      indices=[[0, 0], [0, 1], [2, 0]],
                                      shape=[3, 2])
      }
      return features, tf.constant([1., 0., 0.2], dtype=tf.float32)

    language_column = tf.contrib.layers.sparse_column_with_hash_bucket(
        'language', hash_bucket_size=20)

    regressor = tf.contrib.learn.DNNLinearCombinedRegressor(
        linear_feature_columns=[
            language_column,
            tf.contrib.layers.real_valued_column('age')
        ],
        config=tf.contrib.learn.RunConfig(tf_random_seed=1))

    regressor.fit(input_fn=_input_fn, steps=100)

    scores = regressor.evaluate(input_fn=_input_fn, steps=1)
    self.assertLess(scores['loss'], 0.2)

  def testDNNOnly(self):
    """Tests DNN-only instantiation and training."""
    def _input_fn(num_epochs=None):
      features = {
          'age': tf.train.limit_epochs(tf.constant([[0.8], [0.15], [0.]]),
                                       num_epochs=num_epochs),
          'language': tf.SparseTensor(values=['en', 'fr', 'zh'],
                                      indices=[[0, 0], [0, 1], [2, 0]],
                                      shape=[3, 2])
      }
      return features, tf.constant([1., 0., 0.2], dtype=tf.float32)

    language_column = tf.contrib.layers.sparse_column_with_hash_bucket(
        'language', hash_bucket_size=20)

    regressor = tf.contrib.learn.DNNLinearCombinedRegressor(
        dnn_feature_columns=[
            tf.contrib.layers.embedding_column(language_column, dimension=1),
            tf.contrib.layers.real_valued_column('age')
        ],
        dnn_hidden_units=[3, 3],
        config=tf.contrib.learn.RunConfig(tf_random_seed=1))

    regressor.fit(input_fn=_input_fn, steps=100)

    scores = regressor.evaluate(input_fn=_input_fn, steps=1)
    self.assertLess(scores['loss'], 0.2)

class FeatureEngineeringFunctionTest(tf.test.TestCase):
  """Tests feature_engineering_fn."""

  def testNoneFeatureEngineeringFn(self):
    def input_fn():
      # Create 4 rows of (y = x)
      target = tf.constant([[100.], [3.], [2.], [2.]])
      features = {'x': tf.constant([[100.], [3.], [2.], [2.]])}
      return features, target

    def feature_engineering_fn(features, targets):
      _, _ = features, targets
      target = tf.constant([[1000.], [30.], [20.], [20.]])
      features = {'x': tf.constant([[1000.], [30.], [20.], [20.]])}
      return features, target

    estimator_with_fe_fn = tf.contrib.learn.DNNLinearCombinedRegressor(
        linear_feature_columns=[tf.contrib.layers.real_valued_column('x')],
        dnn_feature_columns=[tf.contrib.layers.real_valued_column('x')],
        dnn_hidden_units=[3, 3],
        config=tf.contrib.learn.RunConfig(tf_random_seed=1),
        feature_engineering_fn=feature_engineering_fn)
    estimator_with_fe_fn.fit(input_fn=input_fn, steps=100)

    estimator_without_fe_fn = tf.contrib.learn.DNNLinearCombinedRegressor(
        linear_feature_columns=[tf.contrib.layers.real_valued_column('x')],
        dnn_feature_columns=[tf.contrib.layers.real_valued_column('x')],
        dnn_hidden_units=[3, 3],
        config=tf.contrib.learn.RunConfig(tf_random_seed=1))
    estimator_without_fe_fn.fit(input_fn=input_fn, steps=100)

     # predictions = y
    prediction_with_fe_fn = next(
        estimator_with_fe_fn.predict(input_fn=input_fn, as_iterable=True))
    self.assertAlmostEqual(1000., prediction_with_fe_fn, delta=1.0)
    prediction_without_fe_fn = next(
        estimator_without_fe_fn.predict(input_fn=input_fn, as_iterable=True))
    self.assertAlmostEqual(100., prediction_without_fe_fn, delta=1.0)

if __name__ == '__main__':
  tf.test.main()


"""Tests for Incremental PCA."""
import numpy as np

from sklearn.utils.testing import assert_almost_equal
from sklearn.utils.testing import assert_array_almost_equal
from sklearn.utils.testing import assert_raises

from sklearn import datasets
from sklearn.decomposition import PCA, IncrementalPCA

iris = datasets.load_iris()

def test_incremental_pca():
    # Incremental PCA on dense arrays.
    X = iris.data
    batch_size = X.shape[0] // 3
    ipca = IncrementalPCA(n_components=2, batch_size=batch_size)
    pca = PCA(n_components=2)
    pca.fit_transform(X)

    X_transformed = ipca.fit_transform(X)

    np.testing.assert_equal(X_transformed.shape, (X.shape[0], 2))
    assert_almost_equal(ipca.explained_variance_ratio_.sum(),
                        pca.explained_variance_ratio_.sum(), 1)

    for n_components in [1, 2, X.shape[1]]:
        ipca = IncrementalPCA(n_components, batch_size=batch_size)
        ipca.fit(X)
        cov = ipca.get_covariance()
        precision = ipca.get_precision()
        assert_array_almost_equal(np.dot(cov, precision),
                                  np.eye(X.shape[1]))

def test_incremental_pca_check_projection():
    # Test that the projection of data is correct.
    rng = np.random.RandomState(1999)
    n, p = 100, 3
    X = rng.randn(n, p) * .1
    X[:10] += np.array([3, 4, 5])
    Xt = 0.1 * rng.randn(1, p) + np.array([3, 4, 5])

    # Get the reconstruction of the generated data X
    # Note that Xt has the same "components" as X, just separated
    # This is what we want to ensure is recreated correctly
    Yt = IncrementalPCA(n_components=2).fit(X).transform(Xt)

    # Normalize
    Yt /= np.sqrt((Yt ** 2).sum())

    # Make sure that the first element of Yt is ~1, this means
    # the reconstruction worked as expected
    assert_almost_equal(np.abs(Yt[0][0]), 1., 1)

def test_incremental_pca_inverse():
    # Test that the projection of data can be inverted.
    rng = np.random.RandomState(1999)
    n, p = 50, 3
    X = rng.randn(n, p)  # spherical data
    X[:, 1] *= .00001  # make middle component relatively small
    X += [5, 4, 3]  # make a large mean

    # same check that we can find the original data from the transformed
    # signal (since the data is almost of rank n_components)
    ipca = IncrementalPCA(n_components=2, batch_size=10).fit(X)
    Y = ipca.transform(X)
    Y_inverse = ipca.inverse_transform(Y)
    assert_almost_equal(X, Y_inverse, decimal=3)

def test_incremental_pca_validation():
    # Test that n_components is >=1 and <= n_features.
    X = [[0, 1], [1, 0]]
    for n_components in [-1, 0, .99, 3]:
        assert_raises(ValueError, IncrementalPCA(n_components,
                                                 batch_size=10).fit, X)

def test_incremental_pca_set_params():
    # Test that components_ sign is stable over batch sizes.
    rng = np.random.RandomState(1999)
    n_samples = 100
    n_features = 20
    X = rng.randn(n_samples, n_features)
    X2 = rng.randn(n_samples, n_features)
    X3 = rng.randn(n_samples, n_features)
    ipca = IncrementalPCA(n_components=20)
    ipca.fit(X)
    # Decreasing number of components
    ipca.set_params(n_components=10)
    assert_raises(ValueError, ipca.partial_fit, X2)
    # Increasing number of components
    ipca.set_params(n_components=15)
    assert_raises(ValueError, ipca.partial_fit, X3)
    # Returning to original setting
    ipca.set_params(n_components=20)
    ipca.partial_fit(X)

def test_incremental_pca_num_features_change():
    # Test that changing n_components will raise an error.
    rng = np.random.RandomState(1999)
    n_samples = 100
    X = rng.randn(n_samples, 20)
    X2 = rng.randn(n_samples, 50)
    ipca = IncrementalPCA(n_components=None)
    ipca.fit(X)
    assert_raises(ValueError, ipca.partial_fit, X2)

def test_incremental_pca_batch_signs():
    # Test that components_ sign is stable over batch sizes.
    rng = np.random.RandomState(1999)
    n_samples = 100
    n_features = 3
    X = rng.randn(n_samples, n_features)
    all_components = []
    batch_sizes = np.arange(10, 20)
    for batch_size in batch_sizes:
        ipca = IncrementalPCA(n_components=None, batch_size=batch_size).fit(X)
        all_components.append(ipca.components_)

    for i, j in zip(all_components[:-1], all_components[1:]):
        assert_almost_equal(np.sign(i), np.sign(j), decimal=6)

def test_incremental_pca_batch_values():
    # Test that components_ values are stable over batch sizes.
    rng = np.random.RandomState(1999)
    n_samples = 100
    n_features = 3
    X = rng.randn(n_samples, n_features)
    all_components = []
    batch_sizes = np.arange(20, 40, 3)
    for batch_size in batch_sizes:
        ipca = IncrementalPCA(n_components=None, batch_size=batch_size).fit(X)
        all_components.append(ipca.components_)

    for i, j in zip(all_components[:-1], all_components[1:]):
        assert_almost_equal(i, j, decimal=1)

def test_incremental_pca_partial_fit():
    # Test that fit and partial_fit get equivalent results.
    rng = np.random.RandomState(1999)
    n, p = 50, 3
    X = rng.randn(n, p)  # spherical data
    X[:, 1] *= .00001  # make middle component relatively small
    X += [5, 4, 3]  # make a large mean

    # same check that we can find the original data from the transformed
    # signal (since the data is almost of rank n_components)
    batch_size = 10
    ipca = IncrementalPCA(n_components=2, batch_size=batch_size).fit(X)
    pipca = IncrementalPCA(n_components=2, batch_size=batch_size)
    # Add one to make sure endpoint is included
    batch_itr = np.arange(0, n + 1, batch_size)
    for i, j in zip(batch_itr[:-1], batch_itr[1:]):
        pipca.partial_fit(X[i:j, :])
    assert_almost_equal(ipca.components_, pipca.components_, decimal=3)

def test_incremental_pca_against_pca_iris():
    # Test that IncrementalPCA and PCA are approximate (to a sign flip).
    X = iris.data

    Y_pca = PCA(n_components=2).fit_transform(X)
    Y_ipca = IncrementalPCA(n_components=2, batch_size=25).fit_transform(X)

    assert_almost_equal(np.abs(Y_pca), np.abs(Y_ipca), 1)

def test_incremental_pca_against_pca_random_data():
    # Test that IncrementalPCA and PCA are approximate (to a sign flip).
    rng = np.random.RandomState(1999)
    n_samples = 100
    n_features = 3
    X = rng.randn(n_samples, n_features) + 5 * rng.rand(1, n_features)

    Y_pca = PCA(n_components=3).fit_transform(X)
    Y_ipca = IncrementalPCA(n_components=3, batch_size=25).fit_transform(X)

    assert_almost_equal(np.abs(Y_pca), np.abs(Y_ipca), 1)

def test_explained_variances():
    # Test that PCA and IncrementalPCA calculations match
    X = datasets.make_low_rank_matrix(1000, 100, tail_strength=0.,
                                      effective_rank=10, random_state=1999)
    prec = 3
    n_samples, n_features = X.shape
    for nc in [None, 99]:
        pca = PCA(n_components=nc).fit(X)
        ipca = IncrementalPCA(n_components=nc, batch_size=100).fit(X)
        assert_almost_equal(pca.explained_variance_, ipca.explained_variance_,
                            decimal=prec)
        assert_almost_equal(pca.explained_variance_ratio_,
                            ipca.explained_variance_ratio_, decimal=prec)
        assert_almost_equal(pca.noise_variance_, ipca.noise_variance_,
                            decimal=prec)

def test_whitening():
    # Test that PCA and IncrementalPCA transforms match to sign flip.
    X = datasets.make_low_rank_matrix(1000, 10, tail_strength=0.,
                                      effective_rank=2, random_state=1999)
    prec = 3
    n_samples, n_features = X.shape
    for nc in [None, 9]:
        pca = PCA(whiten=True, n_components=nc).fit(X)
        ipca = IncrementalPCA(whiten=True, n_components=nc,
                              batch_size=250).fit(X)

        Xt_pca = pca.transform(X)
        Xt_ipca = ipca.transform(X)
        assert_almost_equal(np.abs(Xt_pca), np.abs(Xt_ipca), decimal=prec)
        Xinv_ipca = ipca.inverse_transform(Xt_ipca)
        Xinv_pca = pca.inverse_transform(Xt_pca)
        assert_almost_equal(X, Xinv_ipca, decimal=prec)
        assert_almost_equal(X, Xinv_pca, decimal=prec)
        assert_almost_equal(Xinv_pca, Xinv_ipca, decimal=prec)


#! /usr/bin/python
# Software License Agreement (BSD License)
#
# Copyright (c) 2013-2015, Rethink Robotics
# All rights reserved.
#
# Copyright (c) 2011, Ian McMahon
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
#  * Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
#  * Redistributions in binary form must reproduce the above
#    copyright notice, this list of conditions and the following
#    disclaimer in the documentation and/or other materials provided
#    with the distribution.
#  * Neither the name of Ian McMahon nor the names of its
#    contributors may be used to endorse or promote products derived
#    from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#

"""
The Bezier library was  implemented as a class project in CIS515,
Fundamentals of Linear Algebra, taught by Professor Jean Gallier
in the summer of 2011 at the University of Pennsylvania. For an
excellent explanation of Cubic Bezier Curves, and the math
represented in this library, see
http://www.cis.upenn.edu/~cis515/proj1-12.pdf

~~~~~~~~~~~~~~~~~~~~~~~~ Bezier ~~~~~~~~~~~~~~~~~~~~~~~~
A library for computing Bezier Cubic Splines for an arbitrary
set of control points in R2, R3, up to RN space.

Cubic Segment:
C(t) = (1 - t)^3*b0 + 3(1 - t)*b1 + 3(1 - t)*t^2*b2 + t^3*b3

Bezier Spline of Cubic Segments:
B(t) = C_(i)(t-i+1), i-1 <= t <= i
where C0 continuity exists: C_(i)(1) = C_(i+1)(0)
where C1 continuity exists: C'_(i)(1) = C'_(i+1)(0)
and where C2 continuity exists: C"_(i)(1) = C"_(i+1)(0)

ex. usage:
import numpy
import bezier
points_array = numpy.array([[1, 2, 3], [4, 4, 4],
                            [6, 4, 6], [2, 5, 6],
                            [5, 6, 7]])
d_pts = bezier.de_boor_control_pts(points_array)
b_coeffs = bezier.bezier_coefficients(points_array, d_pts)
b_curve = bezier.bezier_curve(b_coeffs, 50)
#  plotting example
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
fig = plt.figure()
ax = fig.gca(projection='3d')
#plot bezier curve
ax.plot(b_curve[:,0], b_curve[:,1], b_curve[:,2])
#plot specified points
ax.plot(points_array[:,0], points_array[:,1], points_array[:,2], 'g*')
ax.set_title("Cubic Bezier Spline")
ax.set_xlabel("X")
ax.set_ylabel("Y")
ax.set_zlabel("Z")
ax.legend(["Bezier Curve", "Control Points"], loc=2)
plt.show()
"""
import numpy as np

def de_boor_control_pts(points_array, d0=None,
                        dN=None, natural=True):
    """
    Compute the de Boor control points for a given
    set for control points

    params:
        points_array: array of user-supplied control points
            numpy.array of size N by k
            N is the number of input control points
            k is the number of dimensions for each point

        d0: the first control point - None if "natural"
            numpy.array of size 1 by k

        dN: the last control point - None if "natural"
            numpy.array of size 1 by k

        natural: flag to signify natural start/end conditions
            bool

    returns:
        d_pts: array of de Boor control points
            numpy.array of size N+3 by k
    """
    # N+3 auxiliary points required to compute d_pts
    # dpts_(-1) = x_(0)
    # dpts_(N+1) = x_(N)
    # so it is only necessary to find N+1 pts, dpts_(0) to to dpts_(N)
    (rows, k) = np.shape(points_array)
    N = rows - 1  # minus 1 because list includes x_(0)
    # Compute A matrix
    if natural:
        if N > 2:
            A = np.zeros((N-1, N-1))
            A[np.ix_([0], [0, 1])] = [4, 1]
            A[np.ix_([N-2], [N-3, N-2])] = [1, 4]
        else:
            A = 4.0
    else:
        if N > 2:
            A = np.zeros((N-1, N-1))
            A[np.ix_([0], [0, 1])] = [3.5, 1]
            A[np.ix_([N-2], [N-3, N-2])] = [1, 3.5]
        else:
            A = 3.5
    for i in range(1, N-2):
        A[np.ix_([i], [i-1, i, i+1])] = [1, 4, 1]
    # Construct de Boor Control Points from A matrix
    d_pts = np.zeros((N+3, k))
    for col in range(0, k):
        x = np.zeros((max(N-1, 1), 1))
        if N > 2:
            # Compute start / end conditions
            if natural:
                x[N-2, 0] = 6*points_array[-2, col] - points_array[-1, col]
                x[0, 0] = 6*points_array[1, col] - points_array[0, col]
            else:
                x[N-2, 0] = 6*points_array[-2, col] - 1.5*dN[0, col]
                x[0, 0] = 6*points_array[1, col] - 1.5*d0[0, col]
            x[range(1, N-3+1), 0] = 6*points_array[range(2, N-2+1), col]
            # Solve bezier interpolation
            d_pts[2:N+1, col] = np.linalg.solve(A, x).T
        else:
            # Compute start / end conditions
            if natural:
                x[0, 0] = 6*points_array[1, col] - points_array[0, col]
            else:
                x[0, 0] = 6*points_array[1, col] - 1.5*d0[col]
            # Solve bezier interpolation
            d_pts[2, col] = x / A
    # Store off start and end positions
    d_pts[0, :] = points_array[0, :]
    d_pts[-1, :] = points_array[-1, :]
    # Compute the second to last de Boor point based on end conditions
    if natural:
        one_third = (1.0/3.0)
        two_thirds = (2.0/3.0)
        d_pts[1, :] = (two_thirds)*points_array[0, :] + (one_third)*d_pts[2, :]
        d_pts[N+1, :] = ((one_third)*d_pts[-3, :] +
                         (two_thirds)*points_array[-1, :])
    else:
        d_pts[1, :] = d0
        d_pts[N+1, :] = dN
    return d_pts

def bezier_coefficients(points_array, d_pts):
    """
    Compute the Bezier coefficients for a given
    set for user-supplied control pts and
    de Boor control pts.

    These B coeffs are used to compute the cubic
    splines for each cubic spline segment as
    follows (where t is a percentage of time between
    b_coeff segments):
    C(t) = (1 - t)^3*b0 + 3(1 - t)*b1
            + 3(1 - t)*t^2*b2 + t^3*b3

    params:
        points_array: array of user-supplied control points
            numpy.array of size N by k
            N is the number of control points
            k is the number of dimensions for each point

        d_pts: array of de Boor control points
            numpy.array of size N+3 by k

    returns:
        b_coeffs: k-dimensional array of 4 Bezier coefficients
            for every control point
            numpy.array of size N by 4 by k
    """
    (rows, k) = np.shape(points_array)
    N = rows - 1  # N minus 1 because points array includes x_0
    b_coeffs = np.zeros(shape=(k, N, 4))
    for i in range(0, N):
        points_array_i = i+1
        d_pts_i = i + 2
        if i == 0:
            for axis_pos in range(0, k):
                b_coeffs[axis_pos, i, 0] = points_array[points_array_i - 1,
                                                        axis_pos]
                b_coeffs[axis_pos, i, 1] = d_pts[d_pts_i - 1, axis_pos]
                b_coeffs[axis_pos, i, 2] = (0.5 * d_pts[d_pts_i - 1, axis_pos]
                                            + 0.5 * d_pts[d_pts_i, axis_pos])
                b_coeffs[axis_pos, i, 3] = points_array[points_array_i,
                                                        axis_pos]
        elif i == N-1:
            for axis_pos in range(0, k):
                b_coeffs[axis_pos, i, 0] = points_array[points_array_i - 1,
                                                        axis_pos]
                b_coeffs[axis_pos, i, 1] = (0.5 * d_pts[d_pts_i - 1, axis_pos]
                                            + 0.5 * d_pts[d_pts_i, axis_pos])
                b_coeffs[axis_pos, i, 2] = d_pts[d_pts_i, axis_pos]
                b_coeffs[axis_pos, i, 3] = points_array[points_array_i,
                                                        axis_pos]
        else:
            for axis_pos in range(0, k):
                b_coeffs[axis_pos, i, 0] = points_array[points_array_i - 1,
                                                        axis_pos]
                b_coeffs[axis_pos, i, 1] = (2.0/3.0 * d_pts[d_pts_i - 1,
                                                            axis_pos]
                                            + 1.0/3.0 * d_pts[d_pts_i,
                                                              axis_pos])
                b_coeffs[axis_pos, i, 2] = (1.0/3.0 * d_pts[d_pts_i - 1,
                                                            axis_pos]
                                            + 2.0/3.0 * d_pts[d_pts_i,
                                                              axis_pos])
                b_coeffs[axis_pos, i, 3] = points_array[points_array_i,
                                                        axis_pos]

    return b_coeffs

def _cubic_spline_point(b_coeff, t):
    """
    Internal convenience function for calculating
    a k-dimensional point defined by the supplied
    Bezier coefficients. Finds the point that
    describes the current position along the bezier
    segment for k dimensions.

    params:
        b_coeff => b0...b3: Four k-dimensional Bezier
            coefficients each one is a numpy.array
            of size k by 1, so
            b_coeff is a numpy array of size k by 4
            k is the number of dimensions for each
            coefficient
        t: percentage of time elapsed for this segment
            0 <= int <= 1.0

    returns:
        current position in k dimensions
            numpy.array of size 1 by k
    """
    return (pow((1-t), 3)*b_coeff[:, 0] +
            3*pow((1-t), 2)*t*b_coeff[:, 1] +
            3*(1-t)*pow(t, 2)*b_coeff[:, 2] +
            pow(t, 3)*b_coeff[:, 3]
            )

def bezier_point(b_coeffs, b_index, t):
    """
    Finds the k values that describe the current
    position along the bezier curve for k dimensions.

    params:
        b_coeffs: k-dimensional array
            for every control point with 4 Bezier coefficients
            numpy.array of size k by N by 4
            N is the number of control points
            k is the number of dimensions for each point
        b_index: index position out between two of
            the N b_coeffs for this point in time
            int
        t: percentage of time that has passed between
            the two control points
            0 <= int <= 1.0

    returns:
        b_point: current position in k dimensions
            numpy.array of size 1 by k
    """
    if b_index <= 0:
        b_point = b_coeffs[:, 0, 0]
    elif b_index > b_coeffs.shape[1]:
        b_point = b_coeffs[:, -1, -1]
    else:
        t = 0.0 if t < 0.0 else t
        t = 1.0 if t > 1.0 else t
        b_coeff_set = b_coeffs[:, b_index-1, range(4)]
        b_point = _cubic_spline_point(b_coeff_set, t)
    return b_point

def bezier_curve(b_coeffs, num_intervals):
    """
    Iterpolation of the entire Bezier curve at once,
    using a specified number of intervals between
    control points (encapsulated by b_coeffs).

    params:
        b_coeffs: k-dimensional array of 4 Bezier coefficients
            for every control point
            numpy.array of size N by 4 by k
            N is the number of control points
            k is the number of dimensions for each point
        num_intervals: the number of intervals between
            control points
            int > 0

    returns:
        b_curve: positions along the bezier curve in k-dimensions
            numpy.array of size N*num_interval+1  by k
            (the +1 is to include the start position on the curve)
    """
    assert num_intervals > 0,\
        "Invalid number of intervals chosen (must be greater than 0)"
    interval = 1.0 / num_intervals
    (num_axes, num_bpts, _) = np.shape(b_coeffs)
    b_curve = np.zeros((num_bpts*num_intervals+1, num_axes))
    # Copy out initial point
    b_curve[0, :] = b_coeffs[:, 0, 0]
    for current_bpt in range(num_bpts):
            b_coeff_set = b_coeffs[:, current_bpt, range(4)]
            for iteration, t in enumerate(np.linspace(interval, 1,
                                                      num_intervals)):
                b_curve[(current_bpt *
                         num_intervals +
                         iteration+1), :] = _cubic_spline_point(b_coeff_set, t)
    return b_curve


import Gedi as gedi
import george
import numpy as np;
import matplotlib.pylab as pl; pl.close('all')
from time import time,sleep
import scipy.optimize as op
import sys
#####  INITIAL DATA ###########################################################

nrep = 1
pontos=[]
temposQP=[]
temposmulti=[]
georgeQP=[]
sleeptime=10
lista=[10,20,50,100,200,500]
#for i in np.arange(100,650,200):
#for i in np.arange(100,1400,350):

### Functions george
# Define the objective function (negative log-likelihood in this case).
def nll(p):
    # Update the kernel parameters and compute the likelihood.
    gp.kernel[:] = p
    ll = gp.lnlikelihood(y, quiet=True)

    # The scipy optimizer doesn't play well with infinities.
    return -ll if np.isfinite(ll) else 1e25

# And the gradient of the objective function.
def grad_nll(p):
    # Update the kernel parameters and compute the likelihood.
    gp.kernel[:] = p
    return -gp.grad_lnlikelihood(y, quiet=True)

### Functions gedi
def nll_gedi(p):
    global kernel
    # Update the kernel parameters and compute the likelihood.
    kernel= gedi.kernel_optimization.new_kernel(kernel,np.exp(p))
    ll = gedi.kernel_likelihood.likelihood(kernel,x,y,yerr)
    # The scipy optimizer doesn't play well with infinities.
    return -ll if np.isfinite(ll) else 1e25

# And the gradient of the objective function.
def grad_nll_gedi(p):
    global kernel
    # Update the kernel parameters and compute the likelihood.
    kernel= gedi.kernel_optimization.new_kernel(kernel,np.exp(p))
    return -np.array(gedi.kernel_likelihood.gradient_likelihood(kernel,x,y,yerr))
###############################################################################

### Things to run
for i0, i in enumerate(lista):
    f=open("{0}.txt".format(i),"w")
    sys.stdout = f

    print i
    pontos.append(i)
    print 'pontos', pontos
    x = 10 * np.sort(np.random.rand(2*i))
    yerr = 0.2 * np.ones_like(x)
    y = np.sin(x) + yerr * np.random.randn(len(x))

    av = []
    for _ in range(nrep):
        start= time()
        kernel= gedi.kernel.QuasiPeriodic(15.0,2.0,1.0,10.0)
        print 'Initial gedi kernel =',kernel
        print 'Initial gedi likelihood =',gedi.kernel_likelihood.likelihood(kernel,x,y,yerr)

        # Run the optimization routine.
        p0_gedi = np.log(kernel.pars)
        results_gedi = op.minimize(nll_gedi, p0_gedi, jac=grad_nll_gedi)

        kernel= gedi.kernel_optimization.new_kernel(kernel,np.exp(results_gedi.x))
        print 'Final gedi kernel =',kernel
        print 'Final gedi likelihood =',gedi.kernel_likelihood.likelihood(kernel,x,y,yerr)
        print
        tempo1= time() - start

        av.append(tempo1)

    temposQP.append(sum(av) / float(nrep))
    print 'temposQP', temposQP
    sleep(sleeptime*i0)
###############################################################################
    av = []
    for _ in range(nrep):
        start= time()
        kernel= gedi.kernel.ExpSineSquared(15.0, 2.0, 10.0)* \
                    gedi.kernel.ExpSquared(1.0,1.0)
        print 'Initial gedi kernel =',kernel
        print 'Initial gedi likelihood =',gedi.kernel_likelihood.likelihood(kernel,x,y,yerr)

        # Run the optimization routine.
        p0_gedi = np.log(kernel.pars)
        results_gedi = op.minimize(nll_gedi, p0_gedi, jac=grad_nll_gedi)

        kernel= gedi.kernel_optimization.new_kernel(kernel,np.exp(results_gedi.x))
        print 'Final gedi kernel =',kernel
        print 'Final gedi likelihood =',gedi.kernel_likelihood.likelihood(kernel,x,y,yerr)
        print
        tempo1= time() - start

        av.append(tempo1)

    temposmulti.append(sum(av) / float(nrep))
    print 'temposmult', temposmulti

    sleep(sleeptime*i0)
    av = []
    for _ in range(nrep):

        start = time() # Calculation using george
        kernelg1 = 15.0**2*george.kernels.ExpSine2Kernel(2/2.0**2,10.0)* \
                    george.kernels.ExpSquaredKernel(1.0**2)

        # You need to compute the GP once before starting the optimization.
        gp = george.GP(kernelg1, mean=np.mean(y))
        gp.compute(x,yerr)

        # Print the initial ln-likelihood.
        print 'Initial george kernel', kernelg1
        print 'Initial george likelihood', gp.lnlikelihood(y)

        # Run the optimization routine.
        p0 = gp.kernel.vector
        results = op.minimize(nll, p0, jac=grad_nll)

        # Update the kernel and print the final log-likelihood.
        gp.kernel[:] = results.x
        print 'Final george kernel =',gp.kernel
        print 'Final george likelihood= ', gp.lnlikelihood(y)
        print
        tempog1= time() - start
        av.append(tempog1)

    georgeQP.append(sum(av) / float(nrep))
    print 'georgeQP', georgeQP
    ###########################################################################
    sys.stdout = sys.__stdout__
    f.close()
    sleep(sleeptime*i0)

N = pontos
pl.figure()
pl.loglog(N, temposQP, 'r-')
pl.loglog(N, temposmulti, 'b-o')
pl.loglog(N, georgeQP, 'b--')

pl.xlim(0.9*N[0], 1.1*N[-1])
pl.xlabel('Number of points')
pl.ylabel('Time')
#pl.title('Covariance matrix calculations')
pl.legend(['gedi QP', 'gedi ESS*ES','george ESS*ES'],loc='upper left')
pl.xticks(fontsize = 18);pl.yticks(fontsize=18)
pl.savefig('speedtest_6.png')
#pl.close('all')


__author__ = 'Sukrit'
import bson
import pandas as pd
import numpy as np
#import matplotlib.pyplot as plt
#from scipy.optimize import curve_fit

ELElist = []
with open('../data/Elsevier_journal_list.csv', 'r') as file :
    x = file.readlines()
    for line in x :
        #print line
        line = line.replace('&','and') #converting & to 'and' [UGH]
        ELElist.append(line.rstrip()) #remove whitespaces

import pymongo
client = pymongo.MongoClient("localhost", 27017)
# db name - aminer
db = client.aminer
# collection
db.publications

jlist = []
i = 0
flag = False

for jname in ELElist :
    flag = False
    try :
        if db.publications.find_one(filter = {'publication' : jname},limit = 1) != None :
            flag = True
    except bson.errors.InvalidStringData :
        print "[ERROR] Could not insert value: " + jname
    else :
        if flag == True :
            jlist.append(jname)
            print "[INFO] Value found: " + jname
            i += 1
print i

with open ("../output/both_journal_list.txt","w")as file:
    for line in jlist:
        file.write(line+"\n")

'''
cursor = db.publications.find()
for document in cursor :
    if document['publication'] not in jlist :
        if document['publication'] in ELElist :
            jlist.append(document['publication'])
            print document['publication']
'''

'''
citable_items = list(db.publications.find({"publication" : P})
citable_items_ids = []
for cite in citable_items : citable_items_ids.append(cite['index'])
'''

#print "[DEBUG] Number of papers ", len(papers)
#print papers


import logging
import json
import argparse
from ConfigParser import SafeConfigParser

import panda

logging.basicConfig()
logging.getLogger().setLevel(logging.DEBUG)
logger = logging.getLogger('requests.packages.urllib3')
logger.setLevel(logging.DEBUG)
logger.propagate = True

class ServiceError(Exception):
    pass

class EncodingProfilesSynchronizer(object):

    def __init__(self, service):
        self._service = service

    def run(self, profiles):
        current_profiles = self._fetch_profiles()

        for current_profile in current_profiles:
            profile_name = current_profile['name']
            if profile_name in profiles:
                new_profile = profiles.pop(profile_name)
                self._update_profile(current_profile, new_profile)

        for new_profile in profiles.values():
            self._create_profile(new_profile)

    def _fetch_profiles(self):
        current_profiles = self._service.get('/profiles.json')
        return json.loads(current_profiles)

    def _update_profile(self, current_profile, new_profile):
        payload = current_profile.copy()
        payload.update(new_profile)
        payload.pop('preset_name')
        profile_id = payload.pop('id')

        self._service.put('/profiles/%s.json' % profile_id, payload)
        print "Updated profile '%s'" % current_profile['name']

    def _create_profile(self, new_profile):
        self._service.post('/profiles.json', new_profile)
        print "Created profile '%s'" % new_profile['name']

def get_config_parser(filename):
    config = SafeConfigParser()
    with open(filename) as config_file:
        config.readfp(config_file)

    return config

def load_profiles_from_config_parser(parser):
    profiles = {}

    for profile_name in parser.sections():
        profile = {'name': profile_name}

        for field, value in parser.items(profile_name):
            profile[field] = value

        profiles[profile_name] = profile

    return profiles

def load_profiles_from_file(filename):
    parser = get_config_parser(filename)
    return load_profiles_from_config_parser(parser)

def get_arguments():
    parser = argparse.ArgumentParser(
        description=("Synchronize the profiles in the configuration file "
                     "to the provided PandaStream cloud"))
    parser.add_argument(
        '--api-host',
        dest='api_host',
        action='store',
        default='api.pandastream.com',
        help="The PandaStream API URL (defaults to %(default)s)")
    parser.add_argument(
        '--api-port',
        dest='api_port',
        action='store',
        default='443',
        help=("The PandaStream API port to use. Possible values: 80 and 443 "
              "(defaults to %(default)s)"))
    parser.add_argument(
        'access_key',
        action='store',
        help="The PandaStream API access key")
    parser.add_argument(
        'secret_key',
        action='store',
        help="The PandaStream API secret key")
    parser.add_argument(
        'cloud_id',
        action='store',
        help="The ID of PandaStream cloud to use")
    parser.add_argument(
        '--profiles-file',
        dest='profiles_file',
        action='store',
        default='profiles.cfg',
        help=("The path to the configuration file containing the profiles to "
              "synchronize (defaults to %(default)s)"))

    return parser.parse_args()

def main():
    args = get_arguments()

    service = panda.Panda(
        api_host=args.api_host,
        cloud_id=args.cloud_id,
        access_key=args.access_key,
        secret_key=args.secret_key,
        api_port=args.api_port)

    synchronizer = EncodingProfilesSynchronizer(service)
    profiles = load_profiles_from_file(args.profiles_file)

    try:
        synchronizer.run(profiles)
    except ServiceError, e:
        print "Failed to synchronize profiles: %s" % e

if __name__ == "__main__":
    main()


#!/usr/bin/python
# -*- coding: utf-8 -*-

"""
=========================================================
Linear Regression Example
=========================================================
This example uses the only the first feature of the `diabetes` dataset, in
order to illustrate a two-dimensional plot of this regression technique. The
straight line can be seen in the plot, showing how linear regression attempts
to draw a straight line that will best minimize the residual sum of squares
between the observed responses in the dataset, and the responses predicted by
the linear approximation.

The coefficients, the residual sum of squares and the variance score are also
calculated.

"""
print(__doc__)

# Code source: Jaques Grobler
# License: BSD 3 clause

import matplotlib.pyplot as plt
import numpy as np
from sklearn import datasets, linear_model

# Load the diabetes dataset
diabetes = datasets.load_diabetes()

# Use only one feature
diabetes_X = diabetes.data[:, np.newaxis, 2]

# Split the data into training/testing sets
diabetes_X_train = diabetes_X[:-20]
diabetes_X_test = diabetes_X[-20:]

# Split the targets into training/testing sets
diabetes_y_train = diabetes.target[:-20]
diabetes_y_test = diabetes.target[-20:]

# Create linear regression object
regr = linear_model.LinearRegression()

# Train the model using the training sets
regr.fit(diabetes_X_train, diabetes_y_train)

# The coefficients
print('Coefficients: \n', regr.coef_)
# The mean square error
print("Residual sum of squares: %.2f"
      % np.mean((regr.predict(diabetes_X_test) - diabetes_y_test) ** 2))
# Explained variance score: 1 is perfect prediction
print('Variance score: %.2f' % regr.score(diabetes_X_test, diabetes_y_test))

# Plot outputs
plt.scatter(diabetes_X_test, diabetes_y_test,  color='black')
plt.plot(diabetes_X_test, regr.predict(diabetes_X_test), color='blue',
         linewidth=3)

plt.xticks(())
plt.yticks(())

plt.show()


"""
Tests for TimedeltaIndex methods behaving like their Timedelta counterparts
"""

import numpy as np
import pytest

import pandas as pd
from pandas import Index, Series, Timedelta, TimedeltaIndex, timedelta_range
import pandas.util.testing as tm

class TestVectorizedTimedelta:
    def test_tdi_total_seconds(self):
        # GH#10939
        # test index
        rng = timedelta_range('1 days, 10:11:12.100123456', periods=2,
                              freq='s')
        expt = [1 * 86400 + 10 * 3600 + 11 * 60 + 12 + 100123456. / 1e9,
                1 * 86400 + 10 * 3600 + 11 * 60 + 13 + 100123456. / 1e9]
        tm.assert_almost_equal(rng.total_seconds(), Index(expt))

        # test Series
        ser = Series(rng)
        s_expt = Series(expt, index=[0, 1])
        tm.assert_series_equal(ser.dt.total_seconds(), s_expt)

        # with nat
        ser[1] = np.nan
        s_expt = Series([1 * 86400 + 10 * 3600 + 11 * 60 +
                         12 + 100123456. / 1e9, np.nan], index=[0, 1])
        tm.assert_series_equal(ser.dt.total_seconds(), s_expt)

        # with both nat
        ser = Series([np.nan, np.nan], dtype='timedelta64[ns]')
        tm.assert_series_equal(ser.dt.total_seconds(),
                               Series([np.nan, np.nan], index=[0, 1]))

    def test_tdi_round(self):
        td = pd.timedelta_range(start='16801 days', periods=5, freq='30Min')
        elt = td[1]

        expected_rng = TimedeltaIndex([Timedelta('16801 days 00:00:00'),
                                       Timedelta('16801 days 00:00:00'),
                                       Timedelta('16801 days 01:00:00'),
                                       Timedelta('16801 days 02:00:00'),
                                       Timedelta('16801 days 02:00:00')])
        expected_elt = expected_rng[1]

        tm.assert_index_equal(td.round(freq='H'), expected_rng)
        assert elt.round(freq='H') == expected_elt

        msg = pd._libs.tslibs.frequencies.INVALID_FREQ_ERR_MSG
        with pytest.raises(ValueError, match=msg):
            td.round(freq='foo')
        with pytest.raises(ValueError, match=msg):
            elt.round(freq='foo')

        msg = "<MonthEnd> is a non-fixed frequency"
        with pytest.raises(ValueError, match=msg):
            td.round(freq='M')
        with pytest.raises(ValueError, match=msg):
            elt.round(freq='M')