from .pandas_vb_common import *
from random import shuffle

class Reindexing(object):
    goal_time = 0.2

    def setup(self):
        self.rng = DatetimeIndex(start='1/1/1970', periods=10000, freq='1min')
        self.df = DataFrame(np.random.rand(10000, 10), index=self.rng,
                            columns=range(10))
        self.df['foo'] = 'bar'
        self.rng2 = Index(self.rng[::2])

        self.df2 = DataFrame(index=range(10000),
                             data=np.random.rand(10000, 30), columns=range(30))

        # multi-index
        N = 5000
        K = 200
        level1 = tm.makeStringIndex(N).values.repeat(K)
        level2 = np.tile(tm.makeStringIndex(K).values, N)
        index = MultiIndex.from_arrays([level1, level2])
        self.s1 = Series(np.random.randn((N * K)), index=index)
        self.s2 = self.s1[::2]

    def time_reindex_dates(self):
        self.df.reindex(self.rng2)

    def time_reindex_columns(self):
        self.df2.reindex(columns=self.df.columns[1:5])

    def time_reindex_multiindex(self):
        self.s1.reindex(self.s2.index)

#----------------------------------------------------------------------
# Pad / backfill

class FillMethod(object):
    goal_time = 0.2

    def setup(self):
        self.rng = date_range('1/1/2000', periods=100000, freq='1min')
        self.ts = Series(np.random.randn(len(self.rng)), index=self.rng)
        self.ts2 = self.ts[::2]
        self.ts3 = self.ts2.reindex(self.ts.index)
        self.ts4 = self.ts3.astype('float32')

    def pad(self, source_series, target_index):
        try:
            source_series.reindex(target_index, method='pad')
        except:
            source_series.reindex(target_index, fillMethod='pad')

    def backfill(self, source_series, target_index):
        try:
            source_series.reindex(target_index, method='backfill')
        except:
            source_series.reindex(target_index, fillMethod='backfill')

    def time_backfill_dates(self):
        self.backfill(self.ts2, self.ts.index)

    def time_pad_daterange(self):
        self.pad(self.ts2, self.ts.index)

    def time_backfill(self):
        self.ts3.fillna(method='backfill')

    def time_backfill_float32(self):
        self.ts4.fillna(method='backfill')

    def time_pad(self):
        self.ts3.fillna(method='pad')

    def time_pad_float32(self):
        self.ts4.fillna(method='pad')

#----------------------------------------------------------------------
# align on level

class LevelAlign(object):
    goal_time = 0.2

    def setup(self):
        self.index = MultiIndex(
            levels=[np.arange(10), np.arange(100), np.arange(100)],
            labels=[np.arange(10).repeat(10000),
                    np.tile(np.arange(100).repeat(100), 10),
                    np.tile(np.tile(np.arange(100), 100), 10)])
        random.shuffle(self.index.values)
        self.df = DataFrame(np.random.randn(len(self.index), 4),
                            index=self.index)
        self.df_level = DataFrame(np.random.randn(100, 4),
                                  index=self.index.levels[1])

    def time_align_level(self):
        self.df.align(self.df_level, level=1, copy=False)

    def time_reindex_level(self):
        self.df_level.reindex(self.df.index, level=1)

#----------------------------------------------------------------------
# drop_duplicates

class Duplicates(object):
    goal_time = 0.2

    def setup(self):
        self.N = 10000
        self.K = 10
        self.key1 = tm.makeStringIndex(self.N).values.repeat(self.K)
        self.key2 = tm.makeStringIndex(self.N).values.repeat(self.K)
        self.df = DataFrame({'key1': self.key1, 'key2': self.key2,
                             'value': np.random.randn((self.N * self.K)),})
        self.col_array_list = list(self.df.values.T)

        self.df2 = self.df.copy()
        self.df2.ix[:10000, :] = np.nan

        self.s = Series(np.random.randint(0, 1000, size=10000))
        self.s2 = Series(np.tile(tm.makeStringIndex(1000).values, 10))

        np.random.seed(1234)
        self.N = 1000000
        self.K = 10000
        self.key1 = np.random.randint(0, self.K, size=self.N)
        self.df_int = DataFrame({'key1': self.key1})
        self.df_bool = DataFrame({i: np.random.randint(0, 2, size=self.K,
                                                       dtype=bool)
                                  for i in range(10)})

    def time_frame_drop_dups(self):
        self.df.drop_duplicates(['key1', 'key2'])

    def time_frame_drop_dups_inplace(self):
        self.df.drop_duplicates(['key1', 'key2'], inplace=True)

    def time_frame_drop_dups_na(self):
        self.df2.drop_duplicates(['key1', 'key2'])

    def time_frame_drop_dups_na_inplace(self):
        self.df2.drop_duplicates(['key1', 'key2'], inplace=True)

    def time_series_drop_dups_int(self):
        self.s.drop_duplicates()

    def time_series_drop_dups_string(self):
        self.s2.drop_duplicates()

    def time_frame_drop_dups_int(self):
        self.df_int.drop_duplicates()

    def time_frame_drop_dups_bool(self):
        self.df_bool.drop_duplicates()

#----------------------------------------------------------------------
# blog "pandas escaped the zoo"

class Align(object):
    goal_time = 0.2

    def setup(self):
        n = 50000
        indices = tm.makeStringIndex(n)
        subsample_size = 40000

        def sample(values, k):
            sampler = np.arange(len(values))
            shuffle(sampler)
            return values.take(sampler[:k])

        self.x = Series(np.random.randn(50000), indices)
        self.y = Series(np.random.randn(subsample_size),
                        index=sample(indices, subsample_size))

    def time_align_series_irregular_string(self):
        (self.x + self.y)

class LibFastZip(object):
    goal_time = 0.2

    def setup(self):
        self.N = 10000
        self.K = 10
        self.key1 = tm.makeStringIndex(self.N).values.repeat(self.K)
        self.key2 = tm.makeStringIndex(self.N).values.repeat(self.K)
        self.df = DataFrame({'key1': self.key1, 'key2': self.key2, 'value': np.random.randn((self.N * self.K)), })
        self.col_array_list = list(self.df.values.T)

        self.df2 = self.df.copy()
        self.df2.ix[:10000, :] = np.nan
        self.col_array_list2 = list(self.df2.values.T)

    def time_lib_fast_zip(self):
        lib.fast_zip(self.col_array_list)

    def time_lib_fast_zip_fillna(self):
        lib.fast_zip_fillna(self.col_array_list2)


# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for dnn.py with feature_column_v2."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import shutil
import tempfile

from unittest.mock import patch

from absl.testing import parameterized
import numpy as np
import six
import tensorflow as tf
from tensorflow.core.example import example_pb2
from tensorflow.core.example import feature_pb2
from tensorflow.python.feature_column import feature_column_v2
from tensorflow_estimator.python.estimator.canned import dnn
from tensorflow_estimator.python.estimator.canned import dnn_testing_utils
from tensorflow_estimator.python.estimator.canned import prediction_keys
from tensorflow_estimator.python.estimator.export import export
from tensorflow_estimator.python.estimator.inputs import numpy_io
from tensorflow_estimator.python.estimator.inputs import pandas_io

try:
  # pylint: disable=g-import-not-at-top
  import pandas as pd
  HAS_PANDAS = True
except IOError:
  # Pandas writes a temporary file during import. If it fails, don't use pandas.
  HAS_PANDAS = False
except ImportError:
  HAS_PANDAS = False

def _dnn_classifier_fn(*args, **kwargs):
  return dnn.DNNClassifierV2(*args, **kwargs)

class DNNModelFnV2Test(dnn_testing_utils.BaseDNNModelFnTest, tf.test.TestCase):

  def __init__(self, methodName='runTest'):  # pylint: disable=invalid-name
    tf.test.TestCase.__init__(self, methodName)
    dnn_testing_utils.BaseDNNModelFnTest.__init__(
        self, dnn.dnn_model_fn_v2, fc_impl=feature_column_v2)

class DNNLogitFnV2Test(dnn_testing_utils.BaseDNNLogitFnTest, tf.test.TestCase):

  def __init__(self, methodName='runTest'):  # pylint: disable=invalid-name
    tf.test.TestCase.__init__(self, methodName)
    dnn_testing_utils.BaseDNNLogitFnTest.__init__(
        self, dnn.dnn_logit_fn_builder_v2, fc_impl=feature_column_v2)

class DNNWarmStartingV2Test(dnn_testing_utils.BaseDNNWarmStartingTest,
                            tf.test.TestCase):

  def __init__(self, methodName='runTest'):  # pylint: disable=invalid-name
    tf.test.TestCase.__init__(self, methodName)
    dnn_testing_utils.BaseDNNWarmStartingTest.__init__(
        self, _dnn_classifier_fn, _dnn_regressor_fn, fc_impl=feature_column_v2)

class DNNClassifierEvaluateV2Test(
    dnn_testing_utils.BaseDNNClassifierEvaluateTest, tf.test.TestCase):

  def __init__(self, methodName='runTest'):  # pylint: disable=invalid-name
    tf.test.TestCase.__init__(self, methodName)
    dnn_testing_utils.BaseDNNClassifierEvaluateTest.__init__(
        self, _dnn_classifier_fn, fc_impl=feature_column_v2)

class DNNClassifierPredictV2Test(dnn_testing_utils.BaseDNNClassifierPredictTest,
                                 tf.test.TestCase):

  def __init__(self, methodName='runTest'):  # pylint: disable=invalid-name
    tf.test.TestCase.__init__(self, methodName)
    dnn_testing_utils.BaseDNNClassifierPredictTest.__init__(
        self, _dnn_classifier_fn, fc_impl=feature_column_v2)

class DNNClassifierTrainV2Test(dnn_testing_utils.BaseDNNClassifierTrainTest,
                               tf.test.TestCase):

  def __init__(self, methodName='runTest'):  # pylint: disable=invalid-name
    tf.test.TestCase.__init__(self, methodName)
    dnn_testing_utils.BaseDNNClassifierTrainTest.__init__(
        self, _dnn_classifier_fn, fc_impl=feature_column_v2)

def _dnn_regressor_fn(*args, **kwargs):
  return dnn.DNNRegressorV2(*args, **kwargs)

class DNNRegressorEvaluateV2Test(dnn_testing_utils.BaseDNNRegressorEvaluateTest,
                                 tf.test.TestCase):

  def __init__(self, methodName='runTest'):  # pylint: disable=invalid-name
    tf.test.TestCase.__init__(self, methodName)
    dnn_testing_utils.BaseDNNRegressorEvaluateTest.__init__(
        self, _dnn_regressor_fn, fc_impl=feature_column_v2)

class DNNRegressorPredictV2Test(dnn_testing_utils.BaseDNNRegressorPredictTest,
                                tf.test.TestCase):

  def __init__(self, methodName='runTest'):  # pylint: disable=invalid-name
    tf.test.TestCase.__init__(self, methodName)
    dnn_testing_utils.BaseDNNRegressorPredictTest.__init__(
        self, _dnn_regressor_fn, fc_impl=feature_column_v2)

class DNNRegressorTrainV2Test(dnn_testing_utils.BaseDNNRegressorTrainTest,
                              tf.test.TestCase):

  def __init__(self, methodName='runTest'):  # pylint: disable=invalid-name
    tf.test.TestCase.__init__(self, methodName)
    dnn_testing_utils.BaseDNNRegressorTrainTest.__init__(
        self, _dnn_regressor_fn, fc_impl=feature_column_v2)

def _queue_parsed_features(feature_map):
  tensors_to_enqueue = []
  keys = []
  for key, tensor in six.iteritems(feature_map):
    keys.append(key)
    tensors_to_enqueue.append(tensor)
  queue_dtypes = [x.dtype for x in tensors_to_enqueue]
  input_queue = tf.queue.FIFOQueue(capacity=100, dtypes=queue_dtypes)
  tf.compat.v1.train.queue_runner.add_queue_runner(
      tf.compat.v1.train.queue_runner.QueueRunner(
          input_queue, [input_queue.enqueue(tensors_to_enqueue)]))
  dequeued_tensors = input_queue.dequeue()
  return {keys[i]: dequeued_tensors[i] for i in range(len(dequeued_tensors))}

class DNNRegressorIntegrationTest(tf.test.TestCase, parameterized.TestCase):

  def setUp(self):
    self._model_dir = tempfile.mkdtemp()

  def tearDown(self):
    if self._model_dir:
      tf.compat.v1.summary.FileWriterCache.clear()
      shutil.rmtree(self._model_dir)

  def _test_complete_flow(self, train_input_fn, eval_input_fn, predict_input_fn,
                          input_dimension, label_dimension, batch_size):
    feature_columns = [
        tf.feature_column.numeric_column('x', shape=(input_dimension,))
    ]

    est = dnn.DNNRegressorV2(
        hidden_units=(2, 2),
        feature_columns=feature_columns,
        label_dimension=label_dimension,
        model_dir=self._model_dir)

    # TRAIN
    num_steps = 10
    est.train(train_input_fn, steps=num_steps)

    # EVALUATE
    scores = est.evaluate(eval_input_fn)
    self.assertEqual(num_steps, scores[tf.compat.v1.GraphKeys.GLOBAL_STEP])
    self.assertIn('loss', six.iterkeys(scores))

    # PREDICT
    predictions = np.array([
        x[prediction_keys.PredictionKeys.PREDICTIONS]
        for x in est.predict(predict_input_fn)
    ])
    self.assertAllEqual((batch_size, label_dimension), predictions.shape)

    # EXPORT
    feature_spec = tf.feature_column.make_parse_example_spec(feature_columns)
    serving_input_receiver_fn = export.build_parsing_serving_input_receiver_fn(
        feature_spec)
    export_dir = est.export_saved_model(tempfile.mkdtemp(),
                                        serving_input_receiver_fn)
    self.assertTrue(tf.compat.v1.gfile.Exists(export_dir))

  def test_numpy_input_fn(self):
    """Tests complete flow with numpy_input_fn."""
    label_dimension = 2
    batch_size = 10
    data = np.linspace(0., 2., batch_size * label_dimension, dtype=np.float32)
    data = data.reshape(batch_size, label_dimension)
    # learn y = x
    train_input_fn = numpy_io.numpy_input_fn(
        x={'x': data},
        y=data,
        batch_size=batch_size,
        num_epochs=None,
        shuffle=True)
    eval_input_fn = numpy_io.numpy_input_fn(
        x={'x': data}, y=data, batch_size=batch_size, shuffle=False)
    predict_input_fn = numpy_io.numpy_input_fn(
        x={'x': data}, batch_size=batch_size, shuffle=False)

    self._test_complete_flow(
        train_input_fn=train_input_fn,
        eval_input_fn=eval_input_fn,
        predict_input_fn=predict_input_fn,
        input_dimension=label_dimension,
        label_dimension=label_dimension,
        batch_size=batch_size)

  def test_pandas_input_fn(self):
    """Tests complete flow with pandas_input_fn."""
    if not HAS_PANDAS:
      return
    label_dimension = 1
    batch_size = 10
    data = np.linspace(0., 2., batch_size, dtype=np.float32)
    x = pd.DataFrame({'x': data})
    y = pd.Series(data)
    train_input_fn = pandas_io.pandas_input_fn(
        x=x, y=y, batch_size=batch_size, num_epochs=None, shuffle=True)
    eval_input_fn = pandas_io.pandas_input_fn(
        x=x, y=y, batch_size=batch_size, shuffle=False)
    predict_input_fn = pandas_io.pandas_input_fn(
        x=x, batch_size=batch_size, shuffle=False)

    self._test_complete_flow(
        train_input_fn=train_input_fn,
        eval_input_fn=eval_input_fn,
        predict_input_fn=predict_input_fn,
        input_dimension=label_dimension,
        label_dimension=label_dimension,
        batch_size=batch_size)

  def test_input_fn_from_parse_example(self):
    """Tests complete flow with input_fn constructed from parse_example."""
    label_dimension = 2
    batch_size = 10
    data = np.linspace(0., 2., batch_size * label_dimension, dtype=np.float32)
    data = data.reshape(batch_size, label_dimension)

    serialized_examples = []
    for datum in data:
      example = example_pb2.Example(
          features=feature_pb2.Features(
              feature={
                  'x':
                      feature_pb2.Feature(
                          float_list=feature_pb2.FloatList(value=datum)),
                  'y':
                      feature_pb2.Feature(
                          float_list=feature_pb2.FloatList(value=datum)),
              }))
      serialized_examples.append(example.SerializeToString())

    feature_spec = {
        'x': tf.io.FixedLenFeature([label_dimension], tf.dtypes.float32),
        'y': tf.io.FixedLenFeature([label_dimension], tf.dtypes.float32),
    }

    def _train_input_fn():
      feature_map = tf.compat.v1.io.parse_example(serialized_examples,
                                                  feature_spec)
      features = _queue_parsed_features(feature_map)
      labels = features.pop('y')
      return features, labels

    def _eval_input_fn():
      feature_map = tf.compat.v1.io.parse_example(
          tf.compat.v1.train.limit_epochs(serialized_examples, num_epochs=1),
          feature_spec)
      features = _queue_parsed_features(feature_map)
      labels = features.pop('y')
      return features, labels

    def _predict_input_fn():
      feature_map = tf.compat.v1.io.parse_example(
          tf.compat.v1.train.limit_epochs(serialized_examples, num_epochs=1),
          feature_spec)
      features = _queue_parsed_features(feature_map)
      features.pop('y')
      return features, None

    self._test_complete_flow(
        train_input_fn=_train_input_fn,
        eval_input_fn=_eval_input_fn,
        predict_input_fn=_predict_input_fn,
        input_dimension=label_dimension,
        label_dimension=label_dimension,
        batch_size=batch_size)

class DNNClassifierIntegrationTest(tf.test.TestCase):

  def setUp(self):
    self._model_dir = tempfile.mkdtemp()

  def tearDown(self):
    if self._model_dir:
      tf.compat.v1.summary.FileWriterCache.clear()
      shutil.rmtree(self._model_dir)

  def _as_label(self, data_in_float):
    return np.rint(data_in_float).astype(np.int64)

  def _test_complete_flow(self, train_input_fn, eval_input_fn, predict_input_fn,
                          input_dimension, n_classes, batch_size):
    feature_columns = [
        tf.feature_column.numeric_column('x', shape=(input_dimension,))
    ]

    est = dnn.DNNClassifierV2(
        hidden_units=(2, 2),
        feature_columns=feature_columns,
        n_classes=n_classes,
        model_dir=self._model_dir)

    # TRAIN
    num_steps = 10
    est.train(train_input_fn, steps=num_steps)

    # EVALUATE
    scores = est.evaluate(eval_input_fn)
    self.assertEqual(num_steps, scores[tf.compat.v1.GraphKeys.GLOBAL_STEP])
    self.assertIn('loss', six.iterkeys(scores))

    # PREDICT
    predicted_proba = np.array([
        x[prediction_keys.PredictionKeys.PROBABILITIES]
        for x in est.predict(predict_input_fn)
    ])
    self.assertAllEqual((batch_size, n_classes), predicted_proba.shape)

    # EXPORT
    feature_spec = tf.feature_column.make_parse_example_spec(feature_columns)
    serving_input_receiver_fn = export.build_parsing_serving_input_receiver_fn(
        feature_spec)
    export_dir = est.export_saved_model(tempfile.mkdtemp(),
                                        serving_input_receiver_fn)
    self.assertTrue(tf.compat.v1.gfile.Exists(export_dir))

  def test_numpy_input_fn(self):
    """Tests complete flow with numpy_input_fn."""
    n_classes = 3
    input_dimension = 2
    batch_size = 10
    data = np.linspace(
        0., n_classes - 1., batch_size * input_dimension, dtype=np.float32)
    x_data = data.reshape(batch_size, input_dimension)
    y_data = np.reshape(self._as_label(data[:batch_size]), (batch_size, 1))
    # learn y = x
    train_input_fn = numpy_io.numpy_input_fn(
        x={'x': x_data},
        y=y_data,
        batch_size=batch_size,
        num_epochs=None,
        shuffle=True)
    eval_input_fn = numpy_io.numpy_input_fn(
        x={'x': x_data}, y=y_data, batch_size=batch_size, shuffle=False)
    predict_input_fn = numpy_io.numpy_input_fn(
        x={'x': x_data}, batch_size=batch_size, shuffle=False)

    self._test_complete_flow(
        train_input_fn=train_input_fn,
        eval_input_fn=eval_input_fn,
        predict_input_fn=predict_input_fn,
        input_dimension=input_dimension,
        n_classes=n_classes,
        batch_size=batch_size)

  def test_pandas_input_fn(self):
    """Tests complete flow with pandas_input_fn."""
    if not HAS_PANDAS:
      return
    input_dimension = 1
    n_classes = 3
    batch_size = 10
    data = np.linspace(0., n_classes - 1., batch_size, dtype=np.float32)
    x = pd.DataFrame({'x': data})
    y = pd.Series(self._as_label(data))
    train_input_fn = pandas_io.pandas_input_fn(
        x=x, y=y, batch_size=batch_size, num_epochs=None, shuffle=True)
    eval_input_fn = pandas_io.pandas_input_fn(
        x=x, y=y, batch_size=batch_size, shuffle=False)
    predict_input_fn = pandas_io.pandas_input_fn(
        x=x, batch_size=batch_size, shuffle=False)

    self._test_complete_flow(
        train_input_fn=train_input_fn,
        eval_input_fn=eval_input_fn,
        predict_input_fn=predict_input_fn,
        input_dimension=input_dimension,
        n_classes=n_classes,
        batch_size=batch_size)

  def test_input_fn_from_parse_example(self):
    """Tests complete flow with input_fn constructed from parse_example."""
    input_dimension = 2
    n_classes = 3
    batch_size = 10
    data = np.linspace(
        0., n_classes - 1., batch_size * input_dimension, dtype=np.float32)
    data = data.reshape(batch_size, input_dimension)

    serialized_examples = []
    for datum in data:
      example = example_pb2.Example(
          features=feature_pb2.Features(
              feature={
                  'x':
                      feature_pb2.Feature(
                          float_list=feature_pb2.FloatList(value=datum)),
                  'y':
                      feature_pb2.Feature(
                          int64_list=feature_pb2.Int64List(
                              value=self._as_label(datum[:1]))),
              }))
      serialized_examples.append(example.SerializeToString())

    feature_spec = {
        'x': tf.io.FixedLenFeature([input_dimension], tf.dtypes.float32),
        'y': tf.io.FixedLenFeature([1], tf.dtypes.int64),
    }

    def _train_input_fn():
      feature_map = tf.compat.v1.io.parse_example(serialized_examples,
                                                  feature_spec)
      features = _queue_parsed_features(feature_map)
      labels = features.pop('y')
      return features, labels

    def _eval_input_fn():
      feature_map = tf.compat.v1.io.parse_example(
          tf.compat.v1.train.limit_epochs(serialized_examples, num_epochs=1),
          feature_spec)
      features = _queue_parsed_features(feature_map)
      labels = features.pop('y')
      return features, labels

    def _predict_input_fn():
      feature_map = tf.compat.v1.io.parse_example(
          tf.compat.v1.train.limit_epochs(serialized_examples, num_epochs=1),
          feature_spec)
      features = _queue_parsed_features(feature_map)
      features.pop('y')
      return features, None

    self._test_complete_flow(
        train_input_fn=_train_input_fn,
        eval_input_fn=_eval_input_fn,
        predict_input_fn=_predict_input_fn,
        input_dimension=input_dimension,
        n_classes=n_classes,
        batch_size=batch_size)

class DNNTrainingMode(tf.test.TestCase):
  """Tests that training mode propagates to feature columns correctly."""

  def setUp(self):
    self._model_dir = tempfile.mkdtemp()
    self._label_dimension = 1
    self._batch_size = 10

  def tearDown(self):
    if self._model_dir:
      tf.compat.v1.summary.FileWriterCache.clear()
      shutil.rmtree(self._model_dir)

  def _create_data(self):
    data = np.linspace(
        0., 2., self._batch_size * self._label_dimension, dtype=np.float32)
    return data.reshape(self._batch_size, self._label_dimension)

  def _get_estimator(self):
    feature_columns = [
        tf.feature_column.numeric_column('x', shape=(self._label_dimension,))
    ]
    return dnn.DNNRegressorV2(
        hidden_units=(2, 2),
        feature_columns=feature_columns,
        label_dimension=self._label_dimension,
        model_dir=self._model_dir)

  def test_train_vs_eval_mode(self):
    data = self._create_data()
    train_input_fn = numpy_io.numpy_input_fn(
        x={'x': data},
        y=data,
        batch_size=self._batch_size,
        num_epochs=None,
        shuffle=True)
    eval_input_fn = numpy_io.numpy_input_fn(
        x={'x': data}, y=data, batch_size=self._batch_size, shuffle=False)
    est = self._get_estimator()
    with patch.object(
        tf.compat.v2.keras.layers.DenseFeatures, 'call',
        return_value=data) as mock_dense_features_call:
      est.train(train_input_fn, steps=10)
      est.evaluate(eval_input_fn)
    train_args, eval_args = mock_dense_features_call.call_args_list
    # DenseFeature should have been called with training = True in train.
    _, train_training_kwarg = train_args
    self.assertTrue(train_training_kwarg['training'])
    # DenseFeature should have been called with training = False in eval.
    _, eval_training_kwarg = eval_args
    self.assertFalse(eval_training_kwarg['training'])

if __name__ == '__main__':
  tf.test.main()


# -*- coding: utf-8 -*-
"""
[Python 2.7 (Mayavi is not yet compatible with Python 3+)]
Created on Wed Dec 16 22:44:15 2015
@author: Ryan Stauffer
https://github.com/ryanpstauffer/market-vis

[This module referenced http://www.theodor.io/scraping-google-finance-data-using-pandas/]
Market Visualization Prototype
Quotes Module
"""

from datetime import datetime, date
import pandas as pd
import json
import urllib
import urllib2
import os

def getIntradayData(ticker, interval_seconds=61, num_days=10):
    # Specify URL string based on function inputs.
    urlString = 'http://www.google.com/finance/getprices?q={0}'.format(ticker.upper())
    urlString += "&i={0}&p={1}d&f=d,c".format(interval_seconds,num_days)

    # Request the text, and split by each line
    r = urllib2.urlopen(urllib2.Request(urlString)).read()
    r = r.splitlines()

    # Split each line by a comma, starting at the 8th line
    r = [line.split(',') for line in r[7:]]

    # Save data in Pandas DataFrame
    df = pd.DataFrame(r, columns=['Datetime',ticker])

    # Convert UNIX to Datetime format
    df['Datetime'] = df['Datetime'].apply(lambda x: datetime.fromtimestamp(int(x[1:])))
    df.index = df['Datetime']

    return df[ticker]

def getDailyData(ticker, startDate, endDate=date.today()):
    ''' Daily quotes from Google Finance API. Date format='yyyy-mm-dd' '''
    ticker = ticker.upper()
    urlString = "http://www.google.com/finance/historical?q={0}".format(ticker)
    urlString += "&startdate={0}&enddate={1}&output=csv".format(
                      startDate.strftime('%b %d, %Y'),endDate.strftime('%b %d, %Y'))

    #Convert URL output to dataframe
    df = pd.read_csv(urllib.urlopen(urlString))

    # Convert strings to Datetime format
    df[df.columns[0]] = df[df.columns[0]].apply(lambda x: datetime.strptime(x, '%d-%b-%y'))

    #Index by date
    df.index = df[df.columns[0]]
    df.drop(df.columns[0], axis=1, inplace=True)

    return df

def getLastPrice(ticker):
    '''Returns last price and date time of a given ticker (from Google Finance API)'''
    # Specify URL string based on function inputs.
    urlString = 'http://www.google.com/finance/info?client=ig&q={0}'.format(ticker.upper())

    # Request the text, and split by each line
    r = urllib2.urlopen(urllib2.Request(urlString)).read()
    obj = json.loads(r[3:])
    print(obj)

    price = float(obj[0]['l'])

    return price

def buildDailyPriceData(tickerList, startDate, endDate):
    print('Pulling Market Data for S&P 500 from {0} to {1}'.format(startDate.strftime('%Y%m%d'), endDate.strftime('%Y%m%d')))
    #Build SP500 daily price data (for saving)
    firstTicker = tickerList[0]
    print(firstTicker)
    firstTickerData = getDailyData(firstTicker, startDate, endDate)
    firstTickerData.rename(columns={'Close' : firstTicker}, inplace = True)
    df = firstTickerData[firstTicker]

    for ticker in tickerList[1:]:
        print(ticker)
        newTicker = getDailyData(ticker, startDate, endDate)
        if not newTicker.empty:
            newTicker.rename(columns={'Close' : ticker}, inplace = True)
            df = pd.concat([df, newTicker[ticker]], axis=1, join='outer')

    #Google returns data w/ most recent at the top, this puts data in chrono order
    stockPrices = df.sort_index()

    print('Pulled data for {0} stocks from {1} to {2}'.format(len(stockPrices.columns), startDate.strftime('%Y%m%d'), endDate.strftime('%Y%m%d')))

    return stockPrices

def buildDummyData():
    '''Builds Daily Price Data from a backup .csv file
    Used for offline testing purposes
    '''

    #Select Dates
    startDate = datetime.strptime('20120101', '%Y%m%d')
    endDate = datetime.strptime('20130101', '%Y%m%d')

    #Load dataset from .csv
    print("Pulling Market Data from .csv")
    dataLoc = os.path.join(os.path.dirname(__file__),"Resources/SP500_daily_price_data.csv")
    df = pd.read_csv(dataLoc)

    #Convert strings to Datetime format
    df[df.columns[0]] = df[df.columns[0]].apply(lambda x: datetime.strptime(x, '%Y-%m-%d'))
    df.index = df[df.columns[0]]
    df.drop(df.columns[0], axis=1, inplace=True)

    #Build Price Table
    stockPrices = df[startDate:endDate]

    print('Pulled data for {0} stocks from {1} to {2}'.format(len(stockPrices.columns), startDate.strftime('%Y%m%d'), endDate.strftime('%Y%m%d')))
    return stockPrices

def createIndexedPricing(stockPrices, startingIndexValue):
    '''Takes a stock prices tables and converts to indexed pricing
    (i.e. all prices are relative based on a common starting index value)
    Inputs:
    stockPrices => a panda DataFrame
    startingIndexValue => the value that all prices will start at
    '''
    #Build Returns Table
    stockReturns = stockPrices.pct_change(1)

    #Build Indexed Price Table (indexed to 100)
    indexedPrices = stockReturns + 1
    indexedPrices.iloc[0] = startingIndexValue
    indexedPrices = indexedPrices.cumprod(axis=0)

    return indexedPrices


"""Classes to handle plotting during the training."""
from __future__ import print_function, division
import math
import cPickle as pickle
from collections import OrderedDict
import numpy as np
import matplotlib.pyplot as plt
import time

GROWTH_BY = 500

class History(object):
    def __init__(self):
        self.line_groups = OrderedDict()

    @staticmethod
    def from_string(s):
        return pickle.loads(s)

    def to_string(self):
        return pickle.dumps(self, protocol=-1)

    @staticmethod
    def load_from_filepath(fp):
        #return json.loads(open(, "r").read())
        with open(fp, "r") as f:
            history = pickle.load(f)
        return history

    def save_to_filepath(self, fp):
        with open(fp, "w") as f:
            pickle.dump(self, f, protocol=-1)

    def add_group(self, group_name, line_names, increasing=True):
        self.line_groups[group_name] = LineGroup(group_name, line_names, increasing=increasing)

    def add_value(self, group_name, line_name, x, y, average=False):
        self.line_groups[group_name].lines[line_name].append(x, y, average=average)

    def get_group_names(self):
        return list(self.line_groups.iterkeys())

    def get_groups_increasing(self):
        return [group.increasing for group in self.line_groups.itervalues()]

    def get_max_x(self):
        return max([group.get_max_x() for group in self.line_groups.itervalues()])

    def get_recent_average(self, group_name, line_name, nb_points):
        ys = self.line_groups[group_name].lines[line_name].ys[-nb_points:]
        return np.average(ys)

class LineGroup(object):
    def __init__(self, group_name, line_names, increasing=True):
        self.group_name = group_name
        self.lines = OrderedDict([(name, Line()) for name in line_names])
        self.increasing = increasing
        self.xlim = (None, None)

    def get_line_names(self):
        return list(self.lines.iterkeys())

    def get_line_xs(self):
        #return [line.xs for line in self.lines.itervalues()]
        """
        for key, line in self.lines.items():
            if not hasattr(line, "last_index"):
                print(self.group_name, key, "no last index")
            else:
                print(self.group_name, key, "OK")
            print(type(line.xs), type(line.ys), type(line.counts), type(line.datetimes))
        """
        return [line.get_xs() for line in self.lines.itervalues()]

    def get_line_ys(self):
        #return [line.ys for line in self.lines.itervalues()]
        return [line.get_ys() for line in self.lines.itervalues()]

    def get_max_x(self):
        #return max([max(line.xs) if len(line.xs) > 0 else 0 for line in self.lines.itervalues()])
        return max([np.maximum(line.get_xs()) if line.last_index > -1 else 0 for line in self.lines.itervalues()])

"""
class Line(object):
    def __init__(self, xs=None, ys=None, counts=None, datetimes=None):
        self.xs = xs if xs is not None else []
        self.ys = ys if ys is not None else []
        self.counts = counts if counts is not None else []
        self.datetimes = datetimes if datetimes is not None else []
        self.last_index = -1

    def append(self, x, y, average=False):
        # legacy (for loading from pickle)
        #if not hasattr(self, "counts"):
        #    self.counts = [1] * len(self.xs)
        # ---

        if not average or len(self.xs) == 0 or self.xs[-1] != x:
            self.xs.append(x)
            self.ys.append(float(y)) # float to get rid of numpy
            self.counts.append(1)
            self.datetimes.append(time.time())
        else:
            count = self.counts[-1]
            self.ys[-1] = ((self.ys[-1] * count) + y) / (count+1)
            self.counts[-1] += 1
            self.datetimes[-1] = time.time()
"""

class Line(object):
    def __init__(self, xs=None, ys=None, counts=None, datetimes=None):
        zeros = np.tile(np.array([0], dtype=np.int32), GROWTH_BY)
        self.xs = xs if xs is not None else np.copy(zeros)
        self.ys = ys if ys is not None else zeros.astype(np.float32)
        self.counts = counts if counts is not None else zeros.astype(np.uint16)
        self.datetimes = datetimes if datetimes is not None else zeros.astype(np.uint64)
        self.last_index = -1

    # for legacy as functions, replace with properties
    def get_xs(self):
        # legacy
        if isinstance(self.xs, list):
            self._legacy_convert_from_list_to_np()

        return self.xs[0:self.last_index+1]

    def get_ys(self):
        return self.ys[0:self.last_index+1]

    def get_counts(self):
        return self.counts[0:self.last_index+1]

    def get_datetimes(self):
        return self.datetimes[0:self.last_index+1]

    def _legacy_convert_from_list_to_np(self):
        #print("is list!")
        print("[plotting] Converting from list to numpy...")
        self.last_index = len(self.xs) - 1
        self.xs = np.array(self.xs, dtype=np.int32)
        self.ys = np.array(self.ys, dtype=np.float32)
        self.counts = np.array(self.counts, dtype=np.uint16)
        self.datetimes = np.array([int(dt*1000) for dt in self.datetimes], dtype=np.uint64)

    def append(self, x, y, average=False):
        # legacy (for loading from pickle)
        #if not hasattr(self, "counts"):
        #    self.counts = [1] * len(self.xs)
        # ---

        #legacy
        if isinstance(self.xs, list):
            self._legacy_convert_from_list_to_np()

        if (self.last_index+1) == self.xs.shape[0]:
            #print("growing from %d by %d..." % (self.xs.shape[0], GROWTH_BY), self.xs.shape, self.ys.shape, self.counts.shape, self.datetimes.shape)
            zeros = np.tile(np.array([0], dtype=np.int32), GROWTH_BY)
            self.xs = np.append(self.xs, np.copy(zeros))
            self.ys = np.append(self.ys, zeros.astype(np.float32))
            self.counts = np.append(self.counts, zeros.astype(np.uint16))
            self.datetimes = np.append(self.datetimes, zeros.astype(np.uint64))
            #print("growing done", self.xs.shape, self.ys.shape, self.counts.shape, self.datetimes.shape)

        first_entry = (self.last_index == -1)
        if not average or first_entry or self.xs[self.last_index] != x:
            idx = self.last_index + 1
            self.xs[idx] = x
            self.ys[idx] = y
            self.counts[idx] = 1
            self.datetimes[idx] = int(time.time()*1000)
            self.last_index = idx
        else:
            idx = self.last_index
            count = self.counts[idx]
            self.ys[idx] = ((self.ys[idx] * count) + y) / (count+1)
            self.counts[idx] = count + 1
            self.datetimes[idx] = int(time.time()*1000)

        #print("added", x, y, average)
        #print(self.xs[self.last_index-10:self.last_index+10+1])
        #print(self.ys[self.last_index-10:self.last_index+10+1])
        #print(self.counts[self.last_index-10:self.last_index+10+1])
        #print(self.datetimes[self.last_index-10:self.last_index+10+1])

class LossPlotter(object):
    def __init__(self, titles, increasing, save_to_fp):
        assert len(titles) == len(increasing)
        n_plots = len(titles)
        self.titles = titles
        self.increasing = dict([(title, incr) for title, incr in zip(titles, increasing)])
        self.xlim = dict([(title, (None, None)) for title in titles])
        self.colors = ["red", "blue", "cyan", "magenta", "orange", "black"]

        self.nb_points_max = 500
        self.save_to_fp = save_to_fp
        self.start_batch_idx = 0
        self.autolimit_y = False
        self.autolimit_y_multiplier = 5

        #self.fig, self.axes = plt.subplots(nrows=2, ncols=2, figsize=(20, 20))
        nrows = max(1, int(math.sqrt(n_plots)))
        ncols = int(math.ceil(n_plots / nrows))
        width = ncols * 10
        height = nrows * 10

        self.fig, self.axes = plt.subplots(nrows=nrows, ncols=ncols, figsize=(width, height))

        if nrows == 1 and ncols == 1:
            self.axes = [self.axes]
        else:
            self.axes = self.axes.flat

        title_to_ax = dict()
        for idx, (title, ax) in enumerate(zip(self.titles, self.axes)):
            title_to_ax[title] = ax
        self.title_to_ax = title_to_ax

        self.fig.tight_layout()
        self.fig.subplots_adjust(left=0.05)

    def plot(self, history):
        for plot_idx, title in enumerate(self.titles):
            ax = self.title_to_ax[title]
            group_name = title
            group_increasing = self.increasing[title]
            group = history.line_groups[title]
            line_names = group.get_line_names()
            #print("getting line x/y...", time.time())
            line_xs = group.get_line_xs()
            line_ys = group.get_line_ys()
            #print("getting line x/y FIN", time.time())

            """
            print("title", title)
            print("line_names", line_names)
            for i, xx in enumerate(line_xs):
                print("line_xs i: ", xx)
            for i, yy in enumerate(line_ys):
                print("line_ys i: ", yy)
            """
            if any([len(xx) > 0 for xx in line_xs]):
                xs_min = min([min(xx) for xx in line_xs if len(xx) > 0])
                xs_max = max([max(xx) for xx in line_xs if len(xx) > 0])
                xlim = self.xlim[title]
                xlim = [
                    max(xs_min, self.start_batch_idx) if xlim[0] is None else min(xlim[0], xs_max-1),
                    xs_max+1 if xlim[1] is None else xlim[1]
                ]
                if xlim[0] < 0:
                    xlim[0] = max(xs_max - abs(xlim[0]), 0)
                if xlim[1] < 0:
                    xlim[1] = max(xs_max - abs(xlim[1]), 1)
            else:
                # none of the lines has any value, so just use dummy values
                # to avoid min/max of empty sequence errors
                xlim = [
                    0 if self.xlim[title][0] is None else self.xlim[title][0],
                    1 if self.xlim[title][1] is None else self.xlim[title][1]
                ]

            self._plot_group(ax, group_name, group_increasing, line_names, line_xs, line_ys, xlim)
        self.fig.savefig(self.save_to_fp)

    # this seems to be slow sometimes
    def _line_to_xy(self, line_x, line_y, xlim, limit_y_min=None, limit_y_max=None):
        def _add_point(points_x, points_y, curr_sum, counter):
            points_x.append(batch_idx)
            y = curr_sum / counter
            if limit_y_min is not None and limit_y_max is not None:
                y = np.clip(y, limit_y_min, limit_y_max)
            elif limit_y_min is not None:
                y = max(y, limit_y_min)
            elif limit_y_max is not None:
                y = min(y, limit_y_max)
            points_y.append(y)

        nb_points = 0
        for i in range(len(line_x)):
            batch_idx = line_x[i]
            if xlim[0] <= batch_idx < xlim[1]:
                nb_points += 1

        point_every = max(1, int(nb_points / self.nb_points_max))
        points_x = []
        points_y = []
        curr_sum = 0
        counter = 0
        for i in range(len(line_x)):
            batch_idx = line_x[i]
            if xlim[0] <= batch_idx < xlim[1]:
                curr_sum += line_y[i]
                counter += 1
                if counter >= point_every:
                    _add_point(points_x, points_y, curr_sum, counter)
                    counter = 0
                    curr_sum = 0
        if counter > 0:
            _add_point(points_x, points_y, curr_sum, counter)

        return points_x, points_y

    def _plot_group(self, ax, group_name, group_increasing, line_names, line_xs, line_ys, xlim):
        ax.cla()
        ax.grid()

        if self.autolimit_y and any([len(line_xs) > 0 for line_xs in line_xs]):
            min_x = min([np.min(line_x) for line_x in line_xs])
            max_x = max([np.max(line_x) for line_x in line_xs])
            min_y = min([np.min(line_y) for line_y in line_ys])
            max_y = max([np.max(line_y) for line_y in line_ys])

            if group_increasing:
                if max_y > 0:
                    limit_y_max = None
                    limit_y_min = max_y / self.autolimit_y_multiplier
                    if min_y > limit_y_min:
                        limit_y_min = None
            else:
                if min_y > 0:
                    limit_y_max = min_y * self.autolimit_y_multiplier
                    limit_y_min = None
                    if max_y < limit_y_max:
                        limit_y_max = None

            if limit_y_min is not None:
                ax.plot((min_x, max_x), (limit_y_min, limit_y_min), c="purple")

            if limit_y_max is not None:
                ax.plot((min_x, max_x), (limit_y_max, limit_y_max), c="purple")

            # y achse range begrenzen
            yaxmin = min_y if limit_y_min is None else limit_y_min
            yaxmax = max_y if limit_y_max is None else limit_y_max
            yrange = yaxmax - yaxmin
            yaxmin = yaxmin - (0.05 * yrange)
            yaxmax = yaxmax + (0.05 * yrange)
            ax.set_ylim([yaxmin, yaxmax])
        else:
            limit_y_min = None
            limit_y_max = None

        for line_name, line_x, line_y, line_col in zip(line_names, line_xs, line_ys, self.colors):
            #print("line to xy...", time.time())
            x, y = self._line_to_xy(line_x, line_y, xlim, limit_y_min=limit_y_min, limit_y_max=limit_y_max)
            #print("line to xy FIN", time.time())
            #print("plotting ax...", time.time())
            ax.plot(x, y, color=line_col, linewidth=1.0)
            #print("plotting ax FIN", time.time())

        ax.set_title(group_name)


#! /usr/bin/env python

import numpy as np
import matplotlib.pyplot as plt
from DataTools import writeDataToFile
import argparse

parser = argparse.ArgumentParser()
parser.add_argument('--time-step',dest='time_step',required=False)
parser.add_argument('--output-file',dest='fn_out',required=False)
args = parser.parse_args()

# Parameters of potential
m = 1.0
k = (2.0*np.pi)**2
angular_freq = np.sqrt(k/m)
freq = angular_freq/(2.0*np.pi)
period = 1.0/freq

# MD Parameters
if(args.time_step):
    time_step = np.float64(args.time_step)
else:
    time_step = 0.01*period
if(args.fn_out):
    fn_out = args.fn_out
else:
    fn_out = 'results.data'
showPlots = False
#num_periods = 20
#num_steps = np.int(np.rint( (num_periods*period)/time_step   ))
num_steps = 10000
# initial postion and velocity at t=0
initial_position = 2.0
initial_velocity = 0.0

def getPotentialEnergy(x):
    potential_ener = 0.5*k*x**2
    return potential_ener
#-------------------------------

def getForce(x):
    force = -k*x
    return force
#-------------------------------

def getAccleration(x):
    return getForce(x)/m
#-------------------------------

def getPotentialAndForce(x):
    return ( getPotentialEnergy(x), getForce(x) )
#-------------------------------

def getKineticEnergy(v):
    kinetic_ener = 0.5*m*v**2
    return kinetic_ener
#-------------------------------

def getTotalEnergy(x,v):
    return getPotentialEnergy(x)+getKineticEnergy(v)
#-------------------------------

# analytical solution:
phi = np.arctan(-initial_velocity/(initial_position*angular_freq))
amplitude =  initial_position/np.cos(phi)
conserved_energy = getPotentialEnergy(amplitude)

# ----------------------
times = []
positions = []
velocites = []
pot_energies = []
kin_energies = []
tot_energies = []

time = 0.0
curr_position = initial_position
prev_position = curr_position-initial_velocity*time_step + 0.5*getAccleration(curr_position)*time_step**2
curr_velocity = initial_velocity

for i in range(num_steps):
    if (i+1) % (num_steps/10) == 0:
        print 'MD step {0:6d} of {1:6d}'.format(i+1,num_steps)
    # get force at t
    accleration = getAccleration(curr_position)
    # get new position at t+dt
    new_position = 2.0*curr_position - prev_position + accleration*time_step**2
    # get velocity at t
    curr_velocity = (new_position - prev_position) / (2.0*time_step)
    # get energies at t
    curr_pot_ener = getPotentialEnergy(curr_position)
    curr_kin_ener = getKineticEnergy(curr_velocity)
    curr_tot_ener = curr_pot_ener + curr_kin_ener
    #
    times.append( time )
    positions.append( curr_position )
    velocites.append( curr_velocity )
    pot_energies.append( curr_pot_ener )
    kin_energies.append( curr_kin_ener )
    tot_energies.append( curr_tot_ener )
    #
    prev_position = curr_position
    curr_position = new_position
    time += time_step
    #
#----------------------------------------

times = np.array(times)
positions = np.array(positions)
velocites = np.array(velocites)
pot_energies = np.array(pot_energies)
kin_energies = np.array(kin_energies)
tot_energies  = np.array(tot_energies)

positions_analytical = amplitude*np.cos(angular_freq*times+phi)
velocites_analytical = -angular_freq*amplitude*np.sin(angular_freq*times+phi)

writeDataToFile(fn_out,
                [times,positions,velocites,pot_energies,kin_energies,tot_energies,positions_analytical,velocites_analytical],
                ['time','pos','vel','pot_ene','kin_ene','tot_ene','pos_an','vel_an'],
                constantsNames=['time_step','period','amplitude','k','m','phi','conserved_energy'],
                constantsValues=[time_step,period,amplitude,k,m,phi,conserved_energy],
                dataFormat='%15.8f')

if showPlots:
    plt.figure(1)
    plt.plot(times,tot_energies)
    plt.plot(times,pot_energies)
    plt.plot(times,kin_energies)
    plt.show()

    plt.figure(2)
    plt.plot(times,pot_energies)
    plt.show()

    plt.figure(3)
    plt.plot(times,kin_energies)
    plt.show()

    plt.figure(4)
    plt.plot(times,velocites)
    plt.show()

    plt.figure(5)
    plt.plot(times,positions)
    plt.plot(times,positions_analytical)
    plt.show()

    plt.figure(6)
    plt.plot(times,positions-positions_analytical)
    plt.show()
#


# -*- coding: utf-8 -*-
'''
Definitions of the grid.
'''

from __future__ import (absolute_import, division, print_function)

import numpy as np

import functions
import gll

class OneDimensionalGrid(object):
    """Contains the grid properties"""

    def __init__(self, param):
        """Init"""
        self.param = param
        self.z = np.zeros(param.nGlob)
        self.rho = np.zeros((param.nSpec, param.nGLL))
        self.mu = np.zeros((param.nSpec, param.nGLL))
        self.ticks = np.zeros(param.nSpec + 1)

        if param.gridType == 'homogeneous':
            self.ticks = np.linspace(0, param.length, param.nSpec + 1)
            self.rho.fill(param.meanRho)
            self.mu.fill(param.meanMu)

            self.z[1:param.nGLJ] = functions.project_inverse(
                param.ksiGLJ[1:param.nGLJ],
                0,
                self.ticks)

            ksiGLL = param.ksiGLL[1:]
            for i in range(param.nGLL, param.nGlob, param.N):
                self.z[i:i + param.N] = functions.project_inverse(ksiGLL,
                                                                  i // param.N,
                                                                  self.ticks)

            self.z[-1] = self.ticks[-1]

        elif param.gridType == 'gradient':
            msg = "typeOfGrid == 'gradient' has not been implemented yet"
            raise NotImplementedError(msg)
        elif param.gridType == 'miscellaneous':
            msg = "typeOfGrid == 'miscellaneous' has not been implemented yet"
            raise NotImplementedError(msg)
        elif param.gridType == 'file':
            self.z, self.rho, self.mu = np.loadtxt(param.gridFile, unpack=True)
            self.ticks = np.loadtxt(param.ticksFile)
        else:
            raise ValueError('Unknown grid type: %s' % (param.gridType, ))
        # Jacobians at the GLL (and GLJ for the first element in axisym)
        # points (arrays nSpec*(N+1) elements)
        self.dXdKsi = gll.jacobian(self.ticks, param)
        self.dKsiDx = gll.jacobian_inverse(self.ticks, param)

    def plot(self):
        """Plot the grid
        my_ticks gives the abscissa of the borders
        TODO I should test : _the types of the parameters
                             _their sizes"""
        import matplotlib.pyplot as plt
        from matplotlib.ticker import FixedLocator

        fig, ax = plt.subplots(2, 1, sharex=True)

        ax[0].plot(self.z[self.param.ibool].flat, self.rho.flat, 'b+')
        ax[0].set_title(r'$\rho(z)$')
        ax[0].xaxis.set_minor_locator(FixedLocator(self.ticks))
        ax[0].xaxis.grid(True, which='minor', alpha=0.5)
        ax[0].yaxis.grid(True)

        ax[1].plot(self.z[self.param.ibool].flat, self.mu.flat, 'r+')
        ax[1].set_title(r'$\mu(z)$')
        ax[1].xaxis.set_minor_locator(FixedLocator(self.ticks))
        ax[1].xaxis.grid(True, which='minor', alpha=0.5)
        ax[1].yaxis.grid(True)

        plt.suptitle('Grid')

        plt.show()


"""This test for the LFW require medium-size data dowloading and processing

If the data has not been already downloaded by running the examples,
the tests won't run (skipped).

If the test are run, the first execution will be long (typically a bit
more than a couple of minutes) but as the dataset loader is leveraging
joblib, successive runs will be fast (less than 200ms).
"""

import random
import os
import shutil
import tempfile
import numpy as np
from sklearn.externals import six
try:
    try:
        from scipy.misc import imsave
    except ImportError:
        from scipy.misc.pilutil import imsave
except ImportError:
    imsave = None

from sklearn.datasets import load_lfw_pairs
from sklearn.datasets import load_lfw_people
from sklearn.datasets import fetch_lfw_pairs
from sklearn.datasets import fetch_lfw_people

from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_warns_message
from sklearn.utils.testing import SkipTest
from sklearn.utils.testing import raises

SCIKIT_LEARN_DATA = tempfile.mkdtemp(prefix="scikit_learn_lfw_test_")
SCIKIT_LEARN_EMPTY_DATA = tempfile.mkdtemp(prefix="scikit_learn_empty_test_")

LFW_HOME = os.path.join(SCIKIT_LEARN_DATA, 'lfw_home')
FAKE_NAMES = [
    'Abdelatif_Smith',
    'Abhati_Kepler',
    'Camara_Alvaro',
    'Chen_Dupont',
    'John_Lee',
    'Lin_Bauman',
    'Onur_Lopez',
]

def setup_module():
    """Test fixture run once and common to all tests of this module"""
    if imsave is None:
        raise SkipTest("PIL not installed.")

    if not os.path.exists(LFW_HOME):
        os.makedirs(LFW_HOME)

    random_state = random.Random(42)
    np_rng = np.random.RandomState(42)

    # generate some random jpeg files for each person
    counts = {}
    for name in FAKE_NAMES:
        folder_name = os.path.join(LFW_HOME, 'lfw_funneled', name)
        if not os.path.exists(folder_name):
            os.makedirs(folder_name)

        n_faces = np_rng.randint(1, 5)
        counts[name] = n_faces
        for i in range(n_faces):
            file_path = os.path.join(folder_name, name + '_%04d.jpg' % i)
            uniface = np_rng.randint(0, 255, size=(250, 250, 3))
            try:
                imsave(file_path, uniface)
            except ImportError:
                raise SkipTest("PIL not installed")

    # add some random file pollution to test robustness
    with open(os.path.join(LFW_HOME, 'lfw_funneled', '.test.swp'), 'wb') as f:
        f.write(six.b('Text file to be ignored by the dataset loader.'))

    # generate some pairing metadata files using the same format as LFW
    with open(os.path.join(LFW_HOME, 'pairsDevTrain.txt'), 'wb') as f:
        f.write(six.b("10\n"))
        more_than_two = [name for name, count in six.iteritems(counts)
                         if count >= 2]
        for i in range(5):
            name = random_state.choice(more_than_two)
            first, second = random_state.sample(range(counts[name]), 2)
            f.write(six.b('%s\t%d\t%d\n' % (name, first, second)))

        for i in range(5):
            first_name, second_name = random_state.sample(FAKE_NAMES, 2)
            first_index = random_state.choice(np.arange(counts[first_name]))
            second_index = random_state.choice(np.arange(counts[second_name]))
            f.write(six.b('%s\t%d\t%s\t%d\n' % (first_name, first_index,
                                                second_name, second_index)))

    with open(os.path.join(LFW_HOME, 'pairsDevTest.txt'), 'wb') as f:
        f.write(six.b("Fake place holder that won't be tested"))

    with open(os.path.join(LFW_HOME, 'pairs.txt'), 'wb') as f:
        f.write(six.b("Fake place holder that won't be tested"))

def teardown_module():
    """Test fixture (clean up) run once after all tests of this module"""
    if os.path.isdir(SCIKIT_LEARN_DATA):
        shutil.rmtree(SCIKIT_LEARN_DATA)
    if os.path.isdir(SCIKIT_LEARN_EMPTY_DATA):
        shutil.rmtree(SCIKIT_LEARN_EMPTY_DATA)

@raises(IOError)
def test_load_empty_lfw_people():
    fetch_lfw_people(data_home=SCIKIT_LEARN_EMPTY_DATA, download_if_missing=False)

def test_load_lfw_people_deprecation():
    msg = ("Function 'load_lfw_people' has been deprecated in 0.17 and will be "
           "removed in 0.19."
           "Use fetch_lfw_people(download_if_missing=False) instead.")
    assert_warns_message(DeprecationWarning, msg, load_lfw_people,
                         data_home=SCIKIT_LEARN_DATA)

def test_load_fake_lfw_people():
    lfw_people = fetch_lfw_people(data_home=SCIKIT_LEARN_DATA,
                                  min_faces_per_person=3, download_if_missing=False)

    # The data is croped around the center as a rectangular bounding box
    # arounthe the face. Colors are converted to gray levels:
    assert_equal(lfw_people.images.shape, (10, 62, 47))
    assert_equal(lfw_people.data.shape, (10, 2914))

    # the target is array of person integer ids
    assert_array_equal(lfw_people.target, [2, 0, 1, 0, 2, 0, 2, 1, 1, 2])

    # names of the persons can be found using the target_names array
    expected_classes = ['Abdelatif Smith', 'Abhati Kepler', 'Onur Lopez']
    assert_array_equal(lfw_people.target_names, expected_classes)

    # It is possible to ask for the original data without any croping or color
    # conversion and not limit on the number of picture per person
    lfw_people = fetch_lfw_people(data_home=SCIKIT_LEARN_DATA,
                                  resize=None, slice_=None, color=True, download_if_missing=False)
    assert_equal(lfw_people.images.shape, (17, 250, 250, 3))

    # the ids and class names are the same as previously
    assert_array_equal(lfw_people.target,
                       [0, 0, 1, 6, 5, 6, 3, 6, 0, 3, 6, 1, 2, 4, 5, 1, 2])
    assert_array_equal(lfw_people.target_names,
                       ['Abdelatif Smith', 'Abhati Kepler', 'Camara Alvaro',
                        'Chen Dupont', 'John Lee', 'Lin Bauman', 'Onur Lopez'])

@raises(ValueError)
def test_load_fake_lfw_people_too_restrictive():
    fetch_lfw_people(data_home=SCIKIT_LEARN_DATA, min_faces_per_person=100, download_if_missing=False)

@raises(IOError)
def test_load_empty_lfw_pairs():
    fetch_lfw_pairs(data_home=SCIKIT_LEARN_EMPTY_DATA, download_if_missing=False)

def test_load_lfw_pairs_deprecation():
    msg = ("Function 'load_lfw_pairs' has been deprecated in 0.17 and will be "
           "removed in 0.19."
           "Use fetch_lfw_pairs(download_if_missing=False) instead.")
    assert_warns_message(DeprecationWarning, msg, load_lfw_pairs,
                         data_home=SCIKIT_LEARN_DATA)

def test_load_fake_lfw_pairs():
    lfw_pairs_train = fetch_lfw_pairs(data_home=SCIKIT_LEARN_DATA, download_if_missing=False)

    # The data is croped around the center as a rectangular bounding box
    # arounthe the face. Colors are converted to gray levels:
    assert_equal(lfw_pairs_train.pairs.shape, (10, 2, 62, 47))

    # the target is whether the person is the same or not
    assert_array_equal(lfw_pairs_train.target, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0])

    # names of the persons can be found using the target_names array
    expected_classes = ['Different persons', 'Same person']
    assert_array_equal(lfw_pairs_train.target_names, expected_classes)

    # It is possible to ask for the original data without any croping or color
    # conversion
    lfw_pairs_train = fetch_lfw_pairs(data_home=SCIKIT_LEARN_DATA,
                                      resize=None, slice_=None, color=True, download_if_missing=False)
    assert_equal(lfw_pairs_train.pairs.shape, (10, 2, 250, 250, 3))

    # the ids and class names are the same as previously
    assert_array_equal(lfw_pairs_train.target, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0])
    assert_array_equal(lfw_pairs_train.target_names, expected_classes)


"""
================
Confusion matrix
================

Example of confusion matrix usage to evaluate the quality
of the output of a classifier on the iris data set. The
diagonal elements represent the number of points for which
the predicted label is equal to the true label, while
off-diagonal elements are those that are mislabeled by the
classifier. The higher the diagonal values of the confusion
matrix the better, indicating many correct predictions.

The figures show the confusion matrix with and without
normalization by class support size (number of elements
in each class). This kind of normalization can be
interesting in case of class imbalance to have a more
visual interpretation of which class is being misclassified.

Here the results are not as good as they could be as our
choice for the regularization parameter C was not the best.
In real life applications this parameter is usually chosen
using :ref:`grid_search`.

"""

print(__doc__)

import numpy as np
import matplotlib.pyplot as plt

from sklearn import svm, datasets
from sklearn.model_selection import train_test_split
from sklearn.metrics import ConfusionMatrixDisplay

# import some data to play with
iris = datasets.load_iris()
X = iris.data
y = iris.target
class_names = iris.target_names

# Split the data into a training set and a test set
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)

# Run classifier, using a model that is too regularized (C too low) to see
# the impact on the results
classifier = svm.SVC(kernel='linear', C=0.01).fit(X_train, y_train)

np.set_printoptions(precision=2)

# Plot non-normalized confusion matrix
titles_options = [("Confusion matrix, without normalization", None),
                  ("Normalized confusion matrix", 'true')]
for title, normalize in titles_options:
    disp = ConfusionMatrixDisplay.from_estimator(
        classifier, X_test, y_test, display_labels=class_names,
        cmap=plt.cm.Blues, normalize=normalize
    )
    disp.ax_.set_title(title)

    print(title)
    print(disp.confusion_matrix)

plt.show()


import pandas as pd
import os
import time
from datetime import datetime

# path = "X:/Backups/intraQuarter" # for Windows with X files :)
# if git clone'ed then use relative path,
# assuming you extracted the downloaded zip into this project's folder:
path = "intraQuarter"

def Key_Stats(gather="Total Debt/Equity (mrq)"):
  statspath = path+'/_KeyStats'
  stock_list = [x[0] for x in os.walk(statspath)]
  df = pd.DataFrame(columns = ['Date','Unix','Ticker','DE Ratio'])
  for each_dir in stock_list[1:]:
    each_file = os.listdir(each_dir)
    # ticker = each_dir.split("\\")[1] # Windows only
    # ticker = each_dir.split("/")[1] # this didn't work so do this:
    ticker = os.path.basename(os.path.normpath(each_dir))
    # print(ticker) # uncomment to verify
    if len(each_file) > 0:
      for file in each_file:
        date_stamp = datetime.strptime(file, '%Y%m%d%H%M%S.html')
        unix_time = time.mktime(date_stamp.timetuple())
        full_file_path = each_dir+'/'+file
        source = open(full_file_path,'r').read()
        try:
          value = float(source.split(gather+':	')[1].split('	')[0])
          # print("value=%s"%value) # uncomment to see what's up
          df = df.append({'Date':date_stamp,'Unix':unix_time,'Ticker':ticker,'DE Ratio':value,}, ignore_index = True)
        except Exception as e:
          pass
  save = gather.replace(' ','').replace(')','').replace('(','').replace('/','')+('.csv')
  print(save)
  df.to_csv(save)

Key_Stats()


""" Automatic refining of astrometry calibration. The initial astrometric calibration is needed, which will be
    refined by using all stars from a given night.
"""

from __future__ import print_function, division, absolute_import

import os
import sys
import copy
import shutil
import random
import argparse

import numpy as np
import scipy.optimize
import matplotlib.pyplot as plt

import RMS.ConfigReader as cr
from RMS.Formats import Platepar
from RMS.Formats import CALSTARS
from RMS.Formats import StarCatalog
from RMS.Formats import FFfile
from RMS.Astrometry.ApplyAstrometry import raDecToXYPP, xyToRaDecPP, rotationWrtHorizon, getFOVSelectionRadius
from RMS.Astrometry.Conversions import date2JD, jd2Date, raDec2AltAz
from RMS.Astrometry.FFTalign import alignPlatepar
from RMS.Math import angularSeparation

# Import Cython functions
import pyximport
pyximport.install(setup_args={'include_dirs':[np.get_include()]})
from RMS.Astrometry.CyFunctions import matchStars, subsetCatalog

def computeMinimizationTolerances(config, platepar, star_dict_len):
    """ Compute tolerances for minimization. """

    # Calculate the function tolerance, so the desired precision can be reached (the number is calculated
    # in the same regard as the cost function)
    fatol = (config.dist_check_threshold**2)/np.sqrt(star_dict_len*config.min_matched_stars + 1)

    # Parameter estimation tolerance for angular values
    fov_w = platepar.X_res/platepar.F_scale
    xatol_ang = config.dist_check_threshold*fov_w/platepar.X_res

    return fatol, xatol_ang

def matchStarsResiduals(config, platepar, catalog_stars, star_dict, match_radius, ret_nmatch=False, \
    sky_coords=False, lim_mag=None, verbose=False):
    """ Match the image and catalog stars with the given astrometry solution and estimate the residuals
        between them.

    Arguments:
        config: [Config structure]
        platepar: [Platepar structure] Astrometry parameters.
        catalog_stars: [ndarray] An array of catalog stars (ra, dec, mag).
        star_dict: [ndarray] A dictionary where the keys are JDs when the stars were recorded and values are
            2D list of stars, each entry is (X, Y, bg_level, level, fwhm).
        match_radius: [float] Maximum radius for star matching (pixels).
        min_matched_stars: [int] Minimum number of matched stars on the image for the image to be accepted.
    Keyword arguments:
        ret_nmatch: [bool] If True, the function returns the number of matched stars and the average
            deviation. False by default.
        sky_coords: [bool] If True, sky coordinate residuals in RA, dec will be used to compute the cost,
            function, not image coordinates.
        lim_mag: [float] Override the limiting magnitude from config. None by default.
        verbose: [bool] Print results. True by default.
    Return:
        cost: [float] The cost function which weights the number of matched stars and the average deviation.
    """

    if lim_mag is None:
        lim_mag = config.catalog_mag_limit

    # Estimate the FOV radius
    fov_radius = getFOVSelectionRadius(platepar)

    # Dictionary containing the matched stars, the keys are JDs of every image
    matched_stars = {}

    # Go through every FF image and its stars
    for jd in star_dict:

        # Estimate RA,dec of the centre of the FOV
        _, RA_c, dec_c, _ = xyToRaDecPP([jd2Date(jd)], [platepar.X_res/2], [platepar.Y_res/2], [1], \
            platepar, extinction_correction=False)

        RA_c = RA_c[0]
        dec_c = dec_c[0]

        # Get stars from the catalog around the defined center in a given radius
        _, extracted_catalog = subsetCatalog(catalog_stars, RA_c, dec_c, jd, platepar.lat, platepar.lon, \
            fov_radius, lim_mag)
        ra_catalog, dec_catalog, mag_catalog = extracted_catalog.T

        # Extract stars for the given Julian date
        stars_list = star_dict[jd]
        stars_list = np.array(stars_list)

        # Convert all catalog stars to image coordinates
        cat_x_array, cat_y_array = raDecToXYPP(ra_catalog, dec_catalog, jd, platepar)

        # Take only those stars which are within the FOV
        x_indices = np.argwhere((cat_x_array >= 0) & (cat_x_array < platepar.X_res))
        y_indices = np.argwhere((cat_y_array >= 0) & (cat_y_array < platepar.Y_res))
        cat_good_indices = np.intersect1d(x_indices, y_indices).astype(np.uint32)

        # cat_x_array = cat_x_array[good_indices]
        # cat_y_array = cat_y_array[good_indices]

        # # Plot image stars
        # im_y, im_x, _, _ = stars_list.T
        # plt.scatter(im_y, im_x, facecolors='none', edgecolor='g')

        # # Plot catalog stars
        # plt.scatter(cat_y_array[cat_good_indices], cat_x_array[cat_good_indices], c='r', s=20, marker='+')

        # plt.show()

        # Match image and catalog stars
        matched_indices = matchStars(stars_list, cat_x_array, cat_y_array, cat_good_indices, match_radius)

        # Skip this image is no stars were matched
        if len(matched_indices) < config.min_matched_stars:
            continue

        matched_indices = np.array(matched_indices)
        matched_img_inds, matched_cat_inds, dist_list = matched_indices.T

        # Extract data from matched stars
        matched_img_stars = stars_list[matched_img_inds.astype(np.int)]
        matched_cat_stars = extracted_catalog[matched_cat_inds.astype(np.int)]

        # Put the matched stars to a dictionary
        matched_stars[jd] = [matched_img_stars, matched_cat_stars, dist_list]

        # # Plot matched stars
        # im_y, im_x, _, _ = matched_img_stars.T
        # cat_y = cat_y_array[matched_cat_inds.astype(np.int)]
        # cat_x = cat_x_array[matched_cat_inds.astype(np.int)]

        # plt.scatter(im_x, im_y, c='r', s=5)
        # plt.scatter(cat_x, cat_y, facecolors='none', edgecolor='g')

        # plt.xlim([0, platepar.X_res])
        # plt.ylim([platepar.Y_res, 0])

        # plt.show()

    # If residuals on the image should be computed
    if not sky_coords:

        unit_label = 'px'

        # Extract all distances
        global_dist_list = []
        # level_list = []
        # mag_list = []
        for jd in matched_stars:
            # matched_img_stars, matched_cat_stars, dist_list = matched_stars[jd]

            _, _, dist_list = matched_stars[jd]

            global_dist_list += dist_list.tolist()

            # # TEST
            # level_list += matched_img_stars[:, 3].tolist()
            # mag_list += matched_cat_stars[:, 2].tolist()

        # # Plot levels vs. magnitudes
        # plt.scatter(mag_list, np.log10(level_list))
        # plt.xlabel('Magnitude')
        # plt.ylabel('Log10 level')
        # plt.show()

    # Compute the residuals on the sky
    else:

        unit_label = 'arcmin'

        global_dist_list = []

        # Go through all matched stars
        for jd in matched_stars:

            matched_img_stars, matched_cat_stars, dist_list = matched_stars[jd]

            # Go through all stars on the image
            for img_star_entry, cat_star_entry in zip(matched_img_stars, matched_cat_stars):

                # Extract star coords
                star_y = img_star_entry[0]
                star_x = img_star_entry[1]
                cat_ra = cat_star_entry[0]
                cat_dec = cat_star_entry[1]

                # Convert image coordinates to RA/Dec
                _, star_ra, star_dec, _ = xyToRaDecPP([jd2Date(jd)], [star_x], [star_y], [1], \
                    platepar, extinction_correction=False)

                # Compute angular distance between the predicted and the catalog position
                ang_dist = np.degrees(angularSeparation(np.radians(cat_ra), np.radians(cat_dec), \
                    np.radians(star_ra[0]), np.radians(star_dec[0])))

                # Store the angular separation in arc minutes
                global_dist_list.append(ang_dist*60)

    # Number of matched stars
    n_matched = len(global_dist_list)

    if n_matched == 0:

        if verbose:
            print('No matched stars with radius {:.1f} px!'.format(match_radius))

        if ret_nmatch:
            return 0, 9999.0, 9999.0, {}

        else:
            return 9999.0

    # Calculate the average distance
    avg_dist = np.median(global_dist_list)

    cost = (avg_dist**2)*(1.0/np.sqrt(n_matched + 1))

    if verbose:

        print()
        print("Matched {:d} stars with radius of {:.1f} px".format(n_matched, match_radius))
        print("    Average distance = {:.3f} {:s}".format(avg_dist, unit_label))
        print("    Cost function    = {:.5f}".format(cost))

    if ret_nmatch:
        return n_matched, avg_dist, cost, matched_stars

    else:
        return cost

def checkFitGoodness(config, platepar, catalog_stars, star_dict, match_radius, verbose=False):
    """ Checks if the platepar is 'good enough', given the extracted star positions. Returns True if the
        fit is deemed good, False otherwise. The goodness of fit is determined by 2 criteria: the average
        star residual (in pixels) has to be below a certain threshold, and an average number of matched stars
        per image has to be above a predefined threshold as well.

    Arguments:
        config: [Config structure]
        platepar: [Platepar structure] Initial astrometry parameters.
        catalog_stars: [ndarray] An array of catalog stars (ra, dec, mag).
        star_dict: [ndarray] A dictionary where the keys are JDs when the stars were recorded and values are
            2D list of stars, each entry is (X, Y, bg_level, level).
        match_radius: [float] Maximum radius for star matching (pixels).
    Keyword arguments:
        verbose: [bool] If True, fit status will be printed on the screen. False by default.
    Return:
        [bool] True if the platepar is good, False otherwise.
    """

    if verbose:
        print()
        print("CHECK FIT GOODNESS:")

    # Match the stars and calculate the residuals
    n_matched, avg_dist, cost, matched_stars = matchStarsResiduals(config, platepar, catalog_stars, \
        star_dict, match_radius, ret_nmatch=True, verbose=verbose)

    # ### Plot zenith distance vs. residual

    # # Go through all images
    # for jd in matched_stars:
    #     _, cat_stars, dists = matched_stars[jd]

    #     # Extract RA/Dec
    #     ra, dec, _ = cat_stars.T

    #     zangle_list = []
    #     for ra_t, dec_t in zip(ra, dec):

    #         # Compute zenith distance
    #         azim, elev = raDec2AltAz(ra_t, dec_t, jd, platepar.lat, platepar.lon)

    #         zangle = 90 - elev

    #         zangle_list.append(zangle)

    #     # Plot zangle vs. distance
    #     plt.scatter(zangle_list, dists, c='k', s=0.1)

    # plt.xlabel('Zenith angle')
    # plt.ylabel('Residual (px)')
    # plt.show()

    # ###

    # Check that the average distance is within the threshold
    if avg_dist <= config.dist_check_threshold:

        if verbose:
            print()
            print('The minimum residual is satisfied!')

        # Check that the minimum number of stars is matched per every image
        if n_matched >= len(star_dict)*1:

            return True

        else:
            if verbose:
                print('But there are not enough stars on every image, recalibrating...')

    return False

def _calcImageResidualsAstro(params, config, platepar, catalog_stars, star_dict, match_radius):
    """ Calculates the differences between the stars on the image and catalog stars in image coordinates with
        the given astrometrical solution.

    Arguments:
        params: [list] Fit parameters - reference RA, Dec, position angle, and scale.
        config: [Config]
        platepar: [Platepar]
        catalog_stars: [list] List of (ra, dec, mag) entries (angles in degrees).
        star_dict: [dict] Dictionary which contains the JD, and a list of (X, Y, bg_intens, intens) of the
            stars on the image.
        match_radius: [float] Star match radius (px).

    Return:
        [float] The average pixel residual (difference between image and catalog positions) normalized
            by the square root of the total number of matched stars.
    """

    # Make a copy of the platepar
    pp = copy.deepcopy(platepar)

    # Extract fitting parameters
    ra_ref, dec_ref, pos_angle_ref, F_scale = params

    # Set the fitting parameters to the platepar clone
    pp.RA_d = ra_ref
    pp.dec_d = dec_ref
    pp.pos_angle_ref = pos_angle_ref
    pp.F_scale = F_scale

    # Match stars and calculate image residuals
    return matchStarsResiduals(config, pp, catalog_stars, star_dict, match_radius, verbose=False)

def starListToDict(config, calstars_list, max_ffs=None):
    """ Converts the list of calstars into dictionary where the keys are FF file JD and the values is
        a list of (X, Y, bg_intens, intens) of stars.
    """

    # Convert the list to a dictionary
    calstars = {ff_file: star_data for ff_file, star_data in calstars_list}

    # Dictionary which will contain the JD, and a list of (X, Y, bg_intens, intens) of the stars
    star_dict = {}

    # Take only those files with enough stars on them
    for ff_name in calstars:

        stars_list = calstars[ff_name]

        # Check if there are enough stars on the image
        if len(stars_list) >= config.ff_min_stars:

            # Calculate the JD time of the FF file
            dt = FFfile.getMiddleTimeFF(ff_name, config.fps, ret_milliseconds=True)
            jd = date2JD(*dt)

            # Add the time and the stars to the dict
            star_dict[jd] = stars_list

    if max_ffs is not None:

        # Limit the number of FF files used
        if len(star_dict) > max_ffs:

            # Randomly choose calstars_files_N image files from the whole list
            rand_keys = random.sample(list(star_dict), max_ffs)
            star_dict = {key: star_dict[key] for key in rand_keys}

    return star_dict

def autoCheckFit(config, platepar, calstars_list, _fft_refinement=False):
    """ Attempts to refine the astrometry fit with the given stars and and initial astrometry parameters.
    Arguments:
        config: [Config structure]
        platepar: [Platepar structure] Initial astrometry parameters.
        calstars_list: [list] A list containing stars extracted from FF files. See RMS.Formats.CALSTARS for
            more details.
    Keyword arguments:
        _fft_refinement: [bool] Internal flag indicating that autoCF is running the second time recursively
            after FFT platepar adjustment.

    Return:
        (platepar, fit_status):
            platepar: [Platepar structure] Estimated/refined platepar.
            fit_status: [bool] True if fit was successfuly, False if not.
    """

    def _handleFailure(config, platepar, calstars_list, catalog_stars, _fft_refinement):
        """ Run FFT alignment before giving up on ACF. """

        if not _fft_refinement:

            print()
            print("-------------------------------------------------------------------------------")
            print('The initial platepar is bad, trying to refine it using FFT phase correlation...')
            print()

            # Prepare data for FFT image registration

            calstars_dict = {ff_file: star_data for ff_file, star_data in calstars_list}

            # Extract star list from CALSTARS file from FF file with most stars
            max_len_ff = max(calstars_dict, key=lambda k: len(calstars_dict[k]))

            # Take only X, Y (change order so X is first)
            calstars_coords = np.array(calstars_dict[max_len_ff])[:, :2]
            calstars_coords[:, [0, 1]] = calstars_coords[:, [1, 0]]

            # Get the time of the FF file
            calstars_time = FFfile.getMiddleTimeFF(max_len_ff, config.fps, ret_milliseconds=True)

            # Try aligning the platepar using FFT image registration
            platepar_refined = alignPlatepar(config, platepar, calstars_time, calstars_coords)

            print()

            ### If there are still not enough stars matched, try FFT again ###
            min_radius = 10

            # Prepare star dictionary to check the match
            dt = FFfile.getMiddleTimeFF(max_len_ff, config.fps, ret_milliseconds=True)
            jd = date2JD(*dt)
            star_dict_temp = {}
            star_dict_temp[jd] = calstars_dict[max_len_ff]

            # Check the number of matched stars
            n_matched, _, _, _ = matchStarsResiduals(config, platepar_refined, catalog_stars, \
                star_dict_temp, min_radius, ret_nmatch=True, verbose=True)

            # Realign again if necessary
            if n_matched < config.min_matched_stars:
                print()
                print("-------------------------------------------------------------------------------")
                print('Doing a second FFT pass as the number of matched stars was too small...')
                print()
                platepar_refined = alignPlatepar(config, platepar_refined, calstars_time, calstars_coords)
                print()

            ### ###

            # Redo autoCF
            return autoCheckFit(config, platepar_refined, calstars_list, _fft_refinement=True)

        else:
            print('Auto Check Fit failed completely, please redo the plate manually!')
            return platepar, False

    if _fft_refinement:
        print('Second ACF run with an updated platepar via FFT phase correlation...')

    # Load catalog stars (overwrite the mag band ratios if specific catalog is used)
    catalog_stars, _, config.star_catalog_band_ratios = StarCatalog.readStarCatalog(config.star_catalog_path, \
        config.star_catalog_file, lim_mag=config.catalog_mag_limit, \
        mag_band_ratios=config.star_catalog_band_ratios)

    # Dictionary which will contain the JD, and a list of (X, Y, bg_intens, intens) of the stars
    star_dict = starListToDict(config, calstars_list, max_ffs=config.calstars_files_N)

    # There has to be a minimum of 200 FF files for star fitting
    if len(star_dict) < config.calstars_files_N:
        print('Not enough FF files in CALSTARS for ACF!')
        return platepar, False

    # Calculate the total number of calibration stars used
    total_calstars = sum([len(star_dict[key]) for key in star_dict])
    print('Total calstars:', total_calstars)

    if total_calstars < config.calstars_min_stars:
        print('Not enough calibration stars, need at least', config.calstars_min_stars)
        return platepar, False

    print()

    # A list of matching radiuses to try
    min_radius = 0.5
    radius_list = [10, 5, 3, 1.5, min_radius]

    # Calculate the function tolerance, so the desired precision can be reached (the number is calculated
    # in the same regard as the cost function)
    fatol, xatol_ang = computeMinimizationTolerances(config, platepar, len(star_dict))

    ### If the initial match is good enough, do only quick recalibratoin ###

    # Match the stars and calculate the residuals
    n_matched, avg_dist, cost, _ = matchStarsResiduals(config, platepar, catalog_stars, star_dict, \
        min_radius, ret_nmatch=True)

    if n_matched >= config.calstars_files_N:

        # Check if the average distance with the tightest radius is close
        if avg_dist < config.dist_check_quick_threshold:

            print("Using quick fit with smaller radiia...")

            # Use a reduced set of initial radius values
            radius_list = [1.5, min_radius]

    ##########

    # Match increasingly smaller search radiia around image stars
    for i, match_radius in enumerate(radius_list):

        # Match the stars and calculate the residuals
        n_matched, avg_dist, cost, _ = matchStarsResiduals(config, platepar, catalog_stars, star_dict, \
            match_radius, ret_nmatch=True)

        print()
        print("-------------------------------------------------------------")
        print("Refining camera pointing with max pixel deviation = {:.1f} px".format(match_radius))
        print("Initial values:")
        print("    Matched stars     = {:>6d}".format(n_matched))
        print("    Average deviation = {:>6.2f} px".format(avg_dist))

        # The initial number of matched stars has to be at least the number of FF imaages, otherwise it means
        #   that the initial platepar is no good
        if n_matched < config.calstars_files_N:
            print("The total number of initially matched stars is too small! Please manually redo the plate or make sure there are enough calibration stars.")

            # Try to refine the platepar with FFT phase correlation and redo the ACF
            return _handleFailure(config, platepar, calstars_list, catalog_stars, _fft_refinement)

        # Check if the platepar is good enough and do not estimate further parameters
        if checkFitGoodness(config, platepar, catalog_stars, star_dict, min_radius, verbose=True):

            # Print out notice only if the platepar is good right away
            if i == 0:
                print("Initial platepar is good enough!")

            return platepar, True

        # Initial parameters for the astrometric fit
        p0 = [platepar.RA_d, platepar.dec_d, platepar.pos_angle_ref, platepar.F_scale]

        # Fit the astrometric parameters
        res = scipy.optimize.minimize(_calcImageResidualsAstro, p0, args=(config, platepar, catalog_stars, \
            star_dict, match_radius), method='Nelder-Mead', \
            options={'fatol': fatol, 'xatol': xatol_ang})

        print(res)

        # If the fit was not successful, stop further fitting
        if not res.success:

            # Try to refine the platepar with FFT phase correlation and redo the ACF
            return _handleFailure(config, platepar, calstars_list, catalog_stars, _fft_refinement)

        else:
            # If the fit was successful, use the new parameters from now on
            ra_ref, dec_ref, pos_angle_ref, F_scale = res.x

            platepar.RA_d = ra_ref
            platepar.dec_d = dec_ref
            platepar.pos_angle_ref = pos_angle_ref
            platepar.F_scale = F_scale

        # Check if the platepar is good enough and do not estimate further parameters
        if checkFitGoodness(config, platepar, catalog_stars, star_dict, min_radius, verbose=True):
            return platepar, True

    # Match the stars and calculate the residuals
    n_matched, avg_dist, cost, matched_stars = matchStarsResiduals(config, platepar, catalog_stars, \
        star_dict, min_radius, ret_nmatch=True)

    print("FINAL SOLUTION with radius {:.1} px:".format(min_radius))
    print("    Matched stars     = {:>6d}".format(n_matched))
    print("    Average deviation = {:>6.2f} px".format(avg_dist))

    # Mark the platepar to indicate that it was automatically refined with CheckFit
    platepar.auto_check_fit_refined = True

    # Recompute alt/az of the FOV centre
    platepar.az_centre, platepar.alt_centre = raDec2AltAz(platepar.RA_d, platepar.dec_d, platepar.JD, \
        platepar.lat, platepar.lon)

    # Recompute the rotation wrt horizon
    platepar.rotation_from_horiz = rotationWrtHorizon(platepar)

    return platepar, True

if __name__ == "__main__":

    ### COMMAND LINE ARGUMENTS

    # Init the command line arguments parser
    arg_parser = argparse.ArgumentParser(description="Check if the calibration file matches the stars, and improve it.")

    arg_parser.add_argument('dir_path', nargs=1, metavar='DIR_PATH', type=str, \
        help='Path to the folder with FF or image files. This folder also has to contain the platepar file.')

    arg_parser.add_argument('-c', '--config', nargs=1, metavar='CONFIG_PATH', type=str, \
        help="Path to a config file which will be used instead of the default one.")

    # Parse the command line arguments
    cml_args = arg_parser.parse_args()

    #########################

    dir_path = cml_args.dir_path[0]

    # Check if the given directory is OK
    if not os.path.exists(dir_path):
        print('No such directory:', dir_path)
        sys.exit()

    # Load the config file
    config = cr.loadConfigFromDirectory(cml_args.config, dir_path)

    # Get a list of files in the night folder
    file_list = os.listdir(dir_path)

    # Find and load the platepar file
    if config.platepar_name in file_list:

        # Load the platepar
        platepar = Platepar.Platepar()
        platepar.read(os.path.join(dir_path, config.platepar_name), use_flat=config.use_flat)

    else:
        print('Cannot find the platepar file in the night directory: ', config.platepar_name)
        sys.exit()

    # Find the CALSTARS file in the given folder
    calstars_file = None
    for calstars_file in file_list:
        if ('CALSTARS' in calstars_file) and ('.txt' in calstars_file):
            break

    if calstars_file is None:
        print('CALSTARS file could not be found in the given directory!')
        sys.exit()

    # Load the calstars file
    calstars_list = CALSTARS.readCALSTARS(dir_path, calstars_file)

    print('CALSTARS file: ' + calstars_file + ' loaded!')

    # Run the automatic astrometry fit
    pp, fit_status = autoCheckFit(config, platepar, calstars_list)

    # If the fit suceeded, save the platepar
    if fit_status:

        print('ACF sucessful!')

        # Save the old platepar
        shutil.move(os.path.join(dir_path, config.platepar_name), os.path.join(dir_path,
            config.platepar_name + '.old'))

        # Save the new platepar
        pp.write(os.path.join(dir_path, config.platepar_name))


#!/usr/bin/python
# -*- coding: utf-8 -*-

r"""
=========================================================
Gaussian Processes regression: basic introductory example
=========================================================

A simple one-dimensional regression exercise computed in two different ways:

1. A noise-free case with a cubic correlation model
2. A noisy case with a squared Euclidean correlation model

In both cases, the model parameters are estimated using the maximum
likelihood principle.

The figures illustrate the interpolating property of the Gaussian Process
model as well as its probabilistic nature in the form of a pointwise 95%
confidence interval.

Note that the parameter ``nugget`` is applied as a Tikhonov regularization
of the assumed covariance between the training points.  In the special case
of the squared euclidean correlation model, nugget is mathematically equivalent
to a normalized variance:  That is

.. math::
   \mathrm{nugget}_i = \left[\frac{\sigma_i}{y_i}\right]^2

"""
print(__doc__)

# Author: Vincent Dubourg <vincent.dubourg@gmail.com>
#         Jake Vanderplas <vanderplas@astro.washington.edu>
# Licence: BSD 3 clause

import numpy as np
from sklearn.gaussian_process import GaussianProcess
from matplotlib import pyplot as pl

np.random.seed(1)

def f(x):
    """The function to predict."""
    return x * np.sin(x)

#----------------------------------------------------------------------
#  First the noiseless case
X = np.atleast_2d([1., 3., 5., 6., 7., 8.]).T

# Observations
y = f(X).ravel()

# Mesh the input space for evaluations of the real function, the prediction and
# its MSE
x = np.atleast_2d(np.linspace(0, 10, 1000)).T

# Instanciate a Gaussian Process model
gp = GaussianProcess(corr='cubic', theta0=1e-2, thetaL=1e-4, thetaU=1e-1,
                     random_start=100)

# Fit to data using Maximum Likelihood Estimation of the parameters
gp.fit(X, y)

# Make the prediction on the meshed x-axis (ask for MSE as well)
y_pred, MSE = gp.predict(x, eval_MSE=True)
sigma = np.sqrt(MSE)

# Plot the function, the prediction and the 95% confidence interval based on
# the MSE
fig = pl.figure()
pl.plot(x, f(x), 'r:', label=u'$f(x) = x\,\sin(x)$')
pl.plot(X, y, 'r.', markersize=10, label=u'Observations')
pl.plot(x, y_pred, 'b-', label=u'Prediction')
pl.fill(np.concatenate([x, x[::-1]]),
        np.concatenate([y_pred - 1.9600 * sigma,
                       (y_pred + 1.9600 * sigma)[::-1]]),
        alpha=.5, fc='b', ec='None', label='95% confidence interval')
pl.xlabel('$x$')
pl.ylabel('$f(x)$')
pl.ylim(-10, 20)
pl.legend(loc='upper left')

#----------------------------------------------------------------------
# now the noisy case
X = np.linspace(0.1, 9.9, 20)
X = np.atleast_2d(X).T

# Observations and noise
y = f(X).ravel()
dy = 0.5 + 1.0 * np.random.random(y.shape)
noise = np.random.normal(0, dy)
y += noise

# Mesh the input space for evaluations of the real function, the prediction and
# its MSE
x = np.atleast_2d(np.linspace(0, 10, 1000)).T

# Instanciate a Gaussian Process model
gp = GaussianProcess(corr='squared_exponential', theta0=1e-1,
                     thetaL=1e-3, thetaU=1,
                     nugget=(dy / y) ** 2,
                     random_start=100)

# Fit to data using Maximum Likelihood Estimation of the parameters
gp.fit(X, y)

# Make the prediction on the meshed x-axis (ask for MSE as well)
y_pred, MSE = gp.predict(x, eval_MSE=True)
sigma = np.sqrt(MSE)

# Plot the function, the prediction and the 95% confidence interval based on
# the MSE
fig = pl.figure()
pl.plot(x, f(x), 'r:', label=u'$f(x) = x\,\sin(x)$')
pl.errorbar(X.ravel(), y, dy, fmt='r.', markersize=10, label=u'Observations')
pl.plot(x, y_pred, 'b-', label=u'Prediction')
pl.fill(np.concatenate([x, x[::-1]]),
        np.concatenate([y_pred - 1.9600 * sigma,
                       (y_pred + 1.9600 * sigma)[::-1]]),
        alpha=.5, fc='b', ec='None', label='95% confidence interval')
pl.xlabel('$x$')
pl.ylabel('$f(x)$')
pl.ylim(-10, 20)
pl.legend(loc='upper left')

pl.show()


# coding: utf-8

# In[1]:

import numpy as np
import matplotlib.pyplot as plt
import gensim
import src.main

# In[2]:

def get_all_ranks(project):
    r_lda = [x for x,y,z in src.main.read_ranks(project, 'release_lda')]
    r_lsi = [x for x,y,z in src.main.read_ranks(project, 'release_lsi')]
    c_lda = [x for x,y,z in src.main.read_ranks(project, 'changeset_lda')]
    c_lsi = [x for x,y,z in src.main.read_ranks(project, 'changeset_lsi')]
    try:
        t_lda = [x for x,y,z in src.main.read_ranks(project, 'temporal_lda')]
        t_lsi = [x for x,y,z in src.main.read_ranks(project, 'temporal_lsi')]
    except:
        t_lda = []
        t_lsi = []

    return r_lda, c_lda, t_lda, r_lsi, c_lsi, t_lsi

# In[3]:

projects = src.main.load_projects()

# In[8]:

for project in projects:
    ranks = get_all_ranks(project)
    fig = plt.figure(dpi=300)
    fig.gca().boxplot(ranks,
                      labels=['S-LDA', 'C-LDA', 'T-LDA', 'S-LSI', 'C-LSI', 'T-LSI'])
    fig.gca().set_title(' '.join([project.name, project.version, project.level]))
    plt.savefig('paper/figures/' + project.name + project.version + project.level + '.png')
    plt.close()

# In[ ]:


"""
==================================================
Plot different SVM classifiers in the iris dataset
==================================================

Comparison of different linear SVM classifiers on a 2D projection of the iris
dataset. We only consider the first 2 features of this dataset:

- Sepal length
- Sepal width

This example shows how to plot the decision surface for four SVM classifiers
with different kernels.

The linear models ``LinearSVC()`` and ``SVC(kernel='linear')`` yield slightly
different decision boundaries. This can be a consequence of the following
differences:

- ``LinearSVC`` minimizes the squared hinge loss while ``SVC`` minimizes the
  regular hinge loss.

- ``LinearSVC`` uses the One-vs-All (also known as One-vs-Rest) multiclass
  reduction while ``SVC`` uses the One-vs-One multiclass reduction.

Both linear models have linear decision boundaries (intersecting hyperplanes)
while the non-linear kernel models (polynomial or Gaussian RBF) have more
flexible non-linear decision boundaries with shapes that depend on the kind of
kernel and its parameters.

.. NOTE:: while plotting the decision function of classifiers for toy 2D
   datasets can help get an intuitive understanding of their respective
   expressive power, be aware that those intuitions don't always generalize to
   more realistic high-dimensional problems.

"""
print(__doc__)

import numpy as np
import matplotlib.pyplot as plt
from sklearn import svm, datasets

# import some data to play with
iris = datasets.load_iris()
X = iris.data[:, :2]  # we only take the first two features. We could
                      # avoid this ugly slicing by using a two-dim dataset
y = iris.target

h = .02  # step size in the mesh

# we create an instance of SVM and fit out data. We do not scale our
# data since we want to plot the support vectors
C = 1.0  # SVM regularization parameter
svc = svm.SVC(kernel='linear', C=C).fit(X, y)
rbf_svc = svm.SVC(kernel='rbf', gamma=0.7, C=C).fit(X, y)
poly_svc = svm.SVC(kernel='poly', degree=3, C=C).fit(X, y)
lin_svc = svm.LinearSVC(C=C).fit(X, y)

# create a mesh to plot in
x_min, x_max = X[:, 0].min() - 1, X[:, 0].max() + 1
y_min, y_max = X[:, 1].min() - 1, X[:, 1].max() + 1
xx, yy = np.meshgrid(np.arange(x_min, x_max, h),
                     np.arange(y_min, y_max, h))

# title for the plots
titles = ['SVC with linear kernel',
          'LinearSVC (linear kernel)',
          'SVC with RBF kernel',
          'SVC with polynomial (degree 3) kernel']

for i, clf in enumerate((svc, lin_svc, rbf_svc, poly_svc)):
    # Plot the decision boundary. For that, we will assign a color to each
    # point in the mesh [x_min, m_max]x[y_min, y_max].
    plt.subplot(2, 2, i + 1)
    plt.subplots_adjust(wspace=0.4, hspace=0.4)

    Z = clf.predict(np.c_[xx.ravel(), yy.ravel()])

    # Put the result into a color plot
    Z = Z.reshape(xx.shape)
    plt.contourf(xx, yy, Z, cmap=plt.cm.Paired, alpha=0.8)

    # Plot also the training points
    plt.scatter(X[:, 0], X[:, 1], c=y, cmap=plt.cm.Paired)
    plt.xlabel('Sepal length')
    plt.ylabel('Sepal width')
    plt.xlim(xx.min(), xx.max())
    plt.ylim(yy.min(), yy.max())
    plt.xticks(())
    plt.yticks(())
    plt.title(titles[i])

plt.show()