from __future__ import print_function

__author__ = 'Thomas Rueckstiess, ruecksti@in.tum.de; Christian Osendorfer, osendorf@in.tum.de'

from scipy import r_, exp, zeros, eye, array, asarray, random, ravel, diag, sqrt, sin, cos, sort, mgrid, dot, floor
from scipy import c_ #@UnusedImport
from scipy.linalg import solve, inv
from pybrain.datasets import SupervisedDataSet
from scipy.linalg import norm

class GaussianProcess:
    """ This class represents a basic n-dimensional Gaussian Process. The implementation
        follows the book 'Gaussian Processes for Machine Learning' by Carl E. Rasmussen
        (an online version is available at: http://www.gaussianprocess.org/gpml/chapters/).
        The hyper parameters of the GP can be adjusted by setting the self.hyper varible,
        which must be a tuple of size 3.
    """

    def __init__(self, indim, start=0, stop=1, step=0.1):
        """ initializes the gaussian process object.

            :arg indim: input dimension
            :key start: start of interval for sampling the GP.
            :key stop: stop of interval for sampling the GP.
            :key step: stepsize for sampling interval.
            :note: start, stop, step can either be scalars or tuples of size 'indim'.
        """
        self.mean = 0
        self.start = start
        self.stop = stop
        self.step = step
        self.indim = indim
        self.trainx = zeros((0, indim), float)
        self.trainy = zeros((0), float)
        self.noise = zeros((0), float)
        self.testx = self._buildGrid()
        self.calculated = True
        self.pred_mean = zeros(len(self.testx))
        self.pred_cov = eye(len(self.testx))
        self.autonoise = False
        self.hyper = (0.5, 2.0, 0.1)

    def _kernel(self, a, b):
        """ kernel function, here RBF kernel """
        (l, sigma_f, _sigma_n) = self.hyper
        r = sigma_f ** 2 * exp(-1.0 / (2 * l ** 2) * norm(a - b, 2) ** 2)
        # if a == b:
        #   r += sigma_n**2
        return r

    def _buildGrid(self):
        (start, stop, step) = (self.start, self.stop, self.step)
        """ returns a mgrid type of array for 'dim' dimensions """
        if isinstance(start, (int, float, complex)):
            dimstr = 'start:stop:step, '*self.indim
        else:
            assert len(start) == len(stop) == len(step)
            dimstr = ["start[%i]:stop[%i]:step[%i], " % (i, i, i) for i in range(len(start))]
            dimstr = ''.join(dimstr)
        return eval('c_[map(ravel, mgrid[' + dimstr + '])]').T

    def _buildCov(self, a, b):
        K = zeros((len(a), len(b)), float)
        for i in range(len(a)):
            for j in range(len(b)):
                K[i, j] = self._kernel(a[i, :], b[j, :])
        return K

    def reset(self):
        self.trainx = zeros((0, self.indim), float)
        self.trainy = zeros((0), float)
        self.noise = zeros((0), float)
        self.pred_mean = zeros(len(self.testx))
        self.pred_cov = eye(len(self.testx))

    def trainOnDataset(self, dataset):
        """ takes a SequentialDataSet with indim input dimension and scalar target """
        assert (dataset.getDimension('input') == self.indim)
        assert (dataset.getDimension('target') == 1)

        self.trainx = dataset.getField('input')
        self.trainy = ravel(dataset.getField('target'))
        self.noise = array([0.001] * len(self.trainx))
        # print(self.trainx, self.trainy)
        self.calculated = False

    def addDataset(self, dataset):
        """ adds the points from the dataset to the training set """
        assert (dataset.getDimension('input') == self.indim)
        assert (dataset.getDimension('target') == 1)

        self.trainx = r_[self.trainx, dataset.getField('input')]
        self.trainy = r_[self.trainy, ravel(dataset.getField('target'))]
        self.noise = array([0.001] * len(self.trainx))
        self.calculated = False

    def addSample(self, train, target):
        self.trainx = r_[self.trainx, asarray([train])]
        self.trainy = r_[self.trainy, asarray(target)]
        self.noise = r_[self.noise, array([0.001])]
        self.calculated = False

    def testOnArray(self, arr):
        self.testx = arr
        self._calculate()
        return self.pred_mean

    def _calculate(self):
        # calculate only of necessary
        if len(self.trainx) == 0:
            return

        # build covariance matrices
        train_train = self._buildCov(self.trainx, self.trainx)
        train_test = self._buildCov(self.trainx, self.testx)
        test_train = train_test.T
        test_test = self._buildCov(self.testx, self.testx)

        # calculate predictive mean and covariance
        K = train_train + self.noise * eye(len(self.trainx))

        if self.autonoise:
            # calculate average neighboring distance for auto-noise
            avgdist = 0
            sort_trainx = sort(self.trainx)
            for i, d in enumerate(sort_trainx):
                if i == 0:
                    continue
                avgdist += d - sort_trainx[i - 1]
            avgdist /= len(sort_trainx) - 1
            # sort(self.trainx)

            # add auto-noise from neighbouring samples (not standard gp)
            for i in range(len(self.trainx)):
                for j in range(len(self.trainx)):
                    if norm(self.trainx[i] - self.trainx[j]) > avgdist:
                        continue

                    d = norm(self.trainy[i] - self.trainy[j]) / (exp(norm(self.trainx[i] - self.trainx[j])))
                    K[i, i] += d

        self.pred_mean = self.mean + dot(test_train, solve(K, self.trainy - self.mean, sym_pos=0))
        self.pred_cov = test_test - dot(test_train, dot(inv(K), train_test))
        self.calculated = True

    def draw(self):
        if not self.calculated:
            self._calculate()

        return self.pred_mean + random.multivariate_normal(zeros(len(self.testx)), self.pred_cov)

    def plotCurves(self, showSamples=False, force2D=True):
        from pylab import clf, hold, plot, fill, title, gcf, pcolor, gray

        if not self.calculated:
            self._calculate()

        if self.indim == 1:
            clf()
            hold(True)
            if showSamples:
                # plot samples (gray)
                for _ in range(5):
                    plot(self.testx, self.pred_mean + random.multivariate_normal(zeros(len(self.testx)), self.pred_cov), color='gray')

            # plot training set
            plot(self.trainx, self.trainy, 'bx')
            # plot mean (blue)
            plot(self.testx, self.pred_mean, 'b', linewidth=1)
            # plot variance (as "polygon" going from left to right for upper half and back for lower half)
            fillx = r_[ravel(self.testx), ravel(self.testx[::-1])]
            filly = r_[self.pred_mean + 2 * diag(self.pred_cov), self.pred_mean[::-1] - 2 * diag(self.pred_cov)[::-1]]
            fill(fillx, filly, facecolor='gray', edgecolor='white', alpha=0.3)
            title('1D Gaussian Process with mean and variance')

        elif self.indim == 2 and not force2D:
            from matplotlib import axes3d as a3

            fig = gcf()
            fig.clear()
            ax = a3.Axes3D(fig) #@UndefinedVariable

            # plot training set
            ax.plot3D(ravel(self.trainx[:, 0]), ravel(self.trainx[:, 1]), ravel(self.trainy), 'ro')

            # plot mean
            (x, y, z) = [m.reshape(sqrt(len(m)), sqrt(len(m))) for m in (self.testx[:, 0], self.testx[:, 1], self.pred_mean)]
            ax.plot_wireframe(x, y, z, colors='gray')
            return ax

        elif self.indim == 2 and force2D:
            # plot mean on pcolor map
            gray()
            # (x, y, z) = map(lambda m: m.reshape(sqrt(len(m)), sqrt(len(m))), (self.testx[:,0], self.testx[:,1], self.pred_mean))
            m = floor(sqrt(len(self.pred_mean)))
            pcolor(self.pred_mean.reshape(m, m)[::-1, :])

        else: print("plotting only supported for indim=1 or indim=2.")

if __name__ == '__main__':

    from pylab import figure, show

    # --- example on how to use the GP in 1 dimension
    ds = SupervisedDataSet(1, 1)
    gp = GaussianProcess(indim=1, start= -3, stop=3, step=0.05)
    figure()

    x = mgrid[-3:3:0.2]
    y = 0.1 * x ** 2 + x + 1
    z = sin(x) + 0.5 * cos(y)

    ds.addSample(-2.5, -1)
    ds.addSample(-1.0, 3)
    gp.mean = 0

    # new feature "autonoise" adds uncertainty to data depending on
    # it's distance to other points in the dataset. not tested much yet.
    # gp.autonoise = True

    gp.trainOnDataset(ds)
    gp.plotCurves(showSamples=True)

    # you can also test the gp on single points, but this deletes the
    # original testing grid. it can be restored with a call to _buildGrid()
    print((gp.testOnArray(array([[0.4]]))))

    # --- example on how to use the GP in 2 dimensions

    ds = SupervisedDataSet(2, 1)
    gp = GaussianProcess(indim=2, start=0, stop=5, step=0.2)
    figure()

    x, y = mgrid[0:5:4j, 0:5:4j]
    z = cos(x) * sin(y)
    (x, y, z) = list(map(ravel, [x, y, z]))

    for i, j, k in zip(x, y, z):
        ds.addSample([i, j], [k])

    gp.trainOnDataset(ds)
    gp.plotCurves()

    show()


"""
Code for plotting curves with tangent lines.
"""

__author__ = "Ian Goodfellow"

try:
    from matplotlib import pyplot
except Exception:
    pyplot = None
from theano.compat.six.moves import xrange

def tangent_plot(x, y, s):
    """
    Plots a curve with tangent lines.

    Parameters
    ----------
    x : list
        List of x coordinates.
        Assumed to be sorted into ascending order, so that the tangent
        lines occupy 80 percent of the horizontal space between each pair
        of points.
    y : list
        List of y coordinates
    s : list
        List of slopes
    """

    assert isinstance(x, list)
    assert isinstance(y, list)
    assert isinstance(s, list)
    n = len(x)
    assert len(y) == n
    assert len(s) == n

    if pyplot is None:
        raise RuntimeError("Could not import pyplot, can't run this code.")

    pyplot.plot(x, y, color='b')

    if n == 0:
        pyplot.show()
        return

    pyplot.hold(True)

    # Add dummy entries so that the for loop can use the same code on every
    # entry
    if n == 1:
        x = [x[0] - 1] + x[0] + [x[0] + 1.]
    else:
        x = [x[0] - (x[1] - x[0])] + x + [x[-2] + (x[-1] - x[-2])]

    y = [0.] + y + [0]
    s = [0.] + s + [0]

    for i in xrange(1, n + 1):
        ld = 0.4 * (x[i] - x[i - 1])
        lx = x[i] - ld
        ly = y[i] - ld * s[i]
        rd = 0.4 * (x[i + 1] - x[i])
        rx = x[i] + rd
        ry = y[i] + rd * s[i]
        pyplot.plot([lx, rx], [ly, ry], color='g')

    pyplot.show()

if __name__ == "__main__":
    # Demo by plotting a quadratic function
    import numpy as np
    x = np.arange(-5., 5., .1)
    y = 0.5 * (x ** 2)
    x = list(x)
    y = list(y)
    tangent_plot(x, y, x)


# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

from collections import OrderedDict
import argparse
import binascii
import glob
import itertools
import json
import os
import random
import six
import string
import subprocess
import tempfile
import uuid
import errno

import numpy as np

ARROW_HOME = os.path.abspath(__file__).rsplit("/", 2)[0]

# Control for flakiness
np.random.seed(12345)

def load_version_from_pom():
    import xml.etree.ElementTree as ET
    tree = ET.parse(os.path.join(ARROW_HOME, 'java', 'pom.xml'))
    tag_pattern = '{http://maven.apache.org/POM/4.0.0}version'
    version_tag = list(tree.getroot().findall(tag_pattern))[0]
    return version_tag.text

def guid():
    return uuid.uuid4().hex

# from pandas
RANDS_CHARS = np.array(list(string.ascii_letters + string.digits),
                       dtype=(np.str_, 1))

def rands(nchars):
    """
    Generate one random byte string.

    See `rands_array` if you want to create an array of random strings.

    """
    return ''.join(np.random.choice(RANDS_CHARS, nchars))

def tobytes(o):
    if isinstance(o, six.text_type):
        return o.encode('utf8')
    return o

def frombytes(o):
    if isinstance(o, six.binary_type):
        return o.decode('utf8')
    return o

# from the merge_arrow_pr.py script
def run_cmd(cmd):
    if isinstance(cmd, six.string_types):
        cmd = cmd.split(' ')

    try:
        output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
    except subprocess.CalledProcessError as e:
        # this avoids hiding the stdout / stderr of failed processes
        print('Command failed: %s' % ' '.join(cmd))
        print('With output:')
        print('--------------')
        print(frombytes(e.output))
        print('--------------')
        raise e

    return frombytes(output)

# ----------------------------------------------------------------------
# Data generation

class DataType(object):

    def __init__(self, name, nullable=True):
        self.name = name
        self.nullable = nullable

    def get_json(self):
        return OrderedDict([
            ('name', self.name),
            ('type', self._get_type()),
            ('nullable', self.nullable),
            ('children', self._get_children())
        ])

    def _make_is_valid(self, size):
        if self.nullable:
            return np.random.randint(0, 2, size=size)
        else:
            return np.ones(size)

class Column(object):

    def __init__(self, name, count):
        self.name = name
        self.count = count

    def __len__(self):
        return self.count

    def _get_children(self):
        return []

    def _get_buffers(self):
        return []

    def get_json(self):
        entries = [
            ('name', self.name),
            ('count', self.count)
        ]

        buffers = self._get_buffers()
        entries.extend(buffers)

        children = self._get_children()
        if len(children) > 0:
            entries.append(('children', children))

        return OrderedDict(entries)

class PrimitiveType(DataType):

    def _get_children(self):
        return []

class PrimitiveColumn(Column):

    def __init__(self, name, count, is_valid, values):
        super(PrimitiveColumn, self).__init__(name, count)
        self.is_valid = is_valid
        self.values = values

    def _encode_value(self, x):
        return x

    def _get_buffers(self):
        return [
            ('VALIDITY', [int(v) for v in self.is_valid]),
            ('DATA', list([self._encode_value(x) for x in self.values]))
        ]

TEST_INT_MAX = 2 ** 31 - 1
TEST_INT_MIN = ~TEST_INT_MAX

class IntegerType(PrimitiveType):

    def __init__(self, name, is_signed, bit_width, nullable=True,
                 min_value=TEST_INT_MIN,
                 max_value=TEST_INT_MAX):
        super(IntegerType, self).__init__(name, nullable=nullable)
        self.is_signed = is_signed
        self.bit_width = bit_width
        self.min_value = min_value
        self.max_value = max_value

    def _get_generated_data_bounds(self):
        signed_iinfo = np.iinfo('int' + str(self.bit_width))
        if self.is_signed:
            min_value, max_value = signed_iinfo.min, signed_iinfo.max
        else:
            # ARROW-1837 Remove this hack and restore full unsigned integer
            # range
            min_value, max_value = 0, signed_iinfo.max

        lower_bound = max(min_value, self.min_value)
        upper_bound = min(max_value, self.max_value)
        return lower_bound, upper_bound

    def _get_type(self):
        return OrderedDict([
            ('name', 'int'),
            ('isSigned', self.is_signed),
            ('bitWidth', self.bit_width)
        ])

    def generate_column(self, size, name=None):
        lower_bound, upper_bound = self._get_generated_data_bounds()
        return self.generate_range(size, lower_bound, upper_bound, name=name)

    def generate_range(self, size, lower, upper, name=None):
        values = [int(x) for x in
                  np.random.randint(lower, upper, size=size)]

        is_valid = self._make_is_valid(size)

        if name is None:
            name = self.name
        return PrimitiveColumn(name, size, is_valid, values)

class DateType(IntegerType):

    DAY = 0
    MILLISECOND = 1

    # 1/1/1 to 12/31/9999
    _ranges = {
        DAY: [-719162, 2932896],
        MILLISECOND: [-62135596800000, 253402214400000]
    }

    def __init__(self, name, unit, nullable=True):
        bit_width = 32 if unit == self.DAY else 64

        min_value, max_value = self._ranges[unit]
        super(DateType, self).__init__(
            name, True, bit_width, nullable=nullable,
            min_value=min_value, max_value=max_value
        )
        self.unit = unit

    def _get_type(self):
        return OrderedDict([
            ('name', 'date'),
            ('unit', 'DAY' if self.unit == self.DAY else 'MILLISECOND')
        ])

TIMEUNIT_NAMES = {
    's': 'SECOND',
    'ms': 'MILLISECOND',
    'us': 'MICROSECOND',
    'ns': 'NANOSECOND'
}

class TimeType(IntegerType):

    BIT_WIDTHS = {
        's': 32,
        'ms': 32,
        'us': 64,
        'ns': 64
    }

    _ranges = {
        's': [0, 86400],
        'ms': [0, 86400000],
        'us': [0, 86400000000],
        'ns': [0, 86400000000000]
    }

    def __init__(self, name, unit='s', nullable=True):
        min_val, max_val = self._ranges[unit]
        super(TimeType, self).__init__(name, True, self.BIT_WIDTHS[unit],
                                       nullable=nullable,
                                       min_value=min_val,
                                       max_value=max_val)
        self.unit = unit

    def _get_type(self):
        return OrderedDict([
            ('name', 'time'),
            ('unit', TIMEUNIT_NAMES[self.unit]),
            ('bitWidth', self.bit_width)
        ])

class TimestampType(IntegerType):

    # 1/1/1 to 12/31/9999
    _ranges = {
        's': [-62135596800, 253402214400],
        'ms': [-62135596800000, 253402214400000],
        'us': [-62135596800000000, 253402214400000000],

        # Physical range for int64, ~584 years and change
        'ns': [np.iinfo('int64').min, np.iinfo('int64').max]
    }

    def __init__(self, name, unit='s', tz=None, nullable=True):
        min_val, max_val = self._ranges[unit]
        super(TimestampType, self).__init__(name, True, 64, nullable=nullable,
                                            min_value=min_val,
                                            max_value=max_val)
        self.unit = unit
        self.tz = tz

    def _get_type(self):
        fields = [
            ('name', 'timestamp'),
            ('unit', TIMEUNIT_NAMES[self.unit])
        ]

        if self.tz is not None:
            fields.append(('timezone', self.tz))

        return OrderedDict(fields)

class FloatingPointType(PrimitiveType):

    def __init__(self, name, bit_width, nullable=True):
        super(FloatingPointType, self).__init__(name, nullable=nullable)

        self.bit_width = bit_width
        self.precision = {
            16: 'HALF',
            32: 'SINGLE',
            64: 'DOUBLE'
        }[self.bit_width]

    @property
    def numpy_type(self):
        return 'float' + str(self.bit_width)

    def _get_type(self):
        return OrderedDict([
            ('name', 'floatingpoint'),
            ('precision', self.precision)
        ])

    def generate_column(self, size, name=None):
        values = np.random.randn(size) * 1000
        values = np.round(values, 3)

        is_valid = self._make_is_valid(size)
        if name is None:
            name = self.name
        return PrimitiveColumn(name, size, is_valid, values)

DECIMAL_PRECISION_TO_VALUE = {
    key: (1 << (8 * i - 1)) - 1 for i, key in enumerate(
        [1, 3, 5, 7, 10, 12, 15, 17, 19, 22, 24, 27, 29, 32, 34, 36],
        start=1,
    )
}

def decimal_range_from_precision(precision):
    assert 1 <= precision <= 38
    try:
        max_value = DECIMAL_PRECISION_TO_VALUE[precision]
    except KeyError:
        return decimal_range_from_precision(precision - 1)
    else:
        return ~max_value, max_value

class DecimalType(PrimitiveType):
    def __init__(self, name, precision, scale, bit_width=128, nullable=True):
        super(DecimalType, self).__init__(name, nullable=True)
        self.precision = precision
        self.scale = scale
        self.bit_width = bit_width

    @property
    def numpy_type(self):
        return object

    def _get_type(self):
        return OrderedDict([
            ('name', 'decimal'),
            ('precision', self.precision),
            ('scale', self.scale),
        ])

    def generate_column(self, size, name=None):
        min_value, max_value = decimal_range_from_precision(self.precision)
        values = [random.randint(min_value, max_value) for _ in range(size)]

        is_valid = self._make_is_valid(size)
        if name is None:
            name = self.name
        return DecimalColumn(name, size, is_valid, values, self.bit_width)

class DecimalColumn(PrimitiveColumn):

    def __init__(self, name, count, is_valid, values, bit_width=128):
        super(DecimalColumn, self).__init__(name, count, is_valid, values)
        self.bit_width = bit_width

    def _encode_value(self, x):
        return str(x)

class BooleanType(PrimitiveType):
    bit_width = 1

    def _get_type(self):
        return OrderedDict([('name', 'bool')])

    @property
    def numpy_type(self):
        return 'bool'

    def generate_column(self, size, name=None):
        values = list(map(bool, np.random.randint(0, 2, size=size)))
        is_valid = self._make_is_valid(size)
        if name is None:
            name = self.name
        return PrimitiveColumn(name, size, is_valid, values)

class BinaryType(PrimitiveType):

    @property
    def numpy_type(self):
        return object

    @property
    def column_class(self):
        return BinaryColumn

    def _get_type(self):
        return OrderedDict([('name', 'binary')])

    def generate_column(self, size, name=None):
        K = 7
        is_valid = self._make_is_valid(size)
        values = []

        for i in range(size):
            if is_valid[i]:
                draw = (np.random.randint(0, 255, size=K)
                        .astype(np.uint8)
                        .tostring())
                values.append(draw)
            else:
                values.append(b"")

        if name is None:
            name = self.name
        return self.column_class(name, size, is_valid, values)

class FixedSizeBinaryType(PrimitiveType):

    def __init__(self, name, byte_width, nullable=True):
        super(FixedSizeBinaryType, self).__init__(name, nullable=nullable)
        self.byte_width = byte_width

    @property
    def numpy_type(self):
        return object

    @property
    def column_class(self):
        return FixedSizeBinaryColumn

    def _get_type(self):
        return OrderedDict([('name', 'fixedsizebinary'), ('byteWidth', self.byte_width)])

    def _get_type_layout(self):
        return OrderedDict([
            ('vectors',
             [OrderedDict([('type', 'VALIDITY'),
                           ('typeBitWidth', 1)]),
              OrderedDict([('type', 'DATA'),
                           ('typeBitWidth', self.byte_width)])])])

    def generate_column(self, size, name=None):
        is_valid = self._make_is_valid(size)
        values = []

        for i in range(size):
            draw = (np.random.randint(0, 255, size=self.byte_width)
                    .astype(np.uint8)
                    .tostring())
            values.append(draw)

        if name is None:
            name = self.name
        return self.column_class(name, size, is_valid, values)

class StringType(BinaryType):

    @property
    def column_class(self):
        return StringColumn

    def _get_type(self):
        return OrderedDict([('name', 'utf8')])

    def generate_column(self, size, name=None):
        K = 7
        is_valid = self._make_is_valid(size)
        values = []

        for i in range(size):
            if is_valid[i]:
                values.append(tobytes(rands(K)))
            else:
                values.append(b"")

        if name is None:
            name = self.name
        return self.column_class(name, size, is_valid, values)

class JsonSchema(object):

    def __init__(self, fields):
        self.fields = fields

    def get_json(self):
        return OrderedDict([
            ('fields', [field.get_json() for field in self.fields])
        ])

class BinaryColumn(PrimitiveColumn):

    def _encode_value(self, x):
        return frombytes(binascii.hexlify(x).upper())

    def _get_buffers(self):
        offset = 0
        offsets = [0]

        data = []
        for i, v in enumerate(self.values):
            if self.is_valid[i]:
                offset += len(v)
            else:
                v = b""

            offsets.append(offset)
            data.append(self._encode_value(v))

        return [
            ('VALIDITY', [int(x) for x in self.is_valid]),
            ('OFFSET', offsets),
            ('DATA', data)
        ]

class FixedSizeBinaryColumn(PrimitiveColumn):

    def _encode_value(self, x):
        return ''.join('{:02x}'.format(c).upper() for c in x)

    def _get_buffers(self):
        data = []
        for i, v in enumerate(self.values):
            data.append(self._encode_value(v))

        return [
            ('VALIDITY', [int(x) for x in self.is_valid]),
            ('DATA', data)
        ]

class StringColumn(BinaryColumn):

    def _encode_value(self, x):
        return frombytes(x)

class ListType(DataType):

    def __init__(self, name, value_type, nullable=True):
        super(ListType, self).__init__(name, nullable=nullable)
        self.value_type = value_type

    def _get_type(self):
        return OrderedDict([
            ('name', 'list')
        ])

    def _get_children(self):
        return [self.value_type.get_json()]

    def generate_column(self, size, name=None):
        MAX_LIST_SIZE = 4

        is_valid = self._make_is_valid(size)
        list_sizes = np.random.randint(0, MAX_LIST_SIZE + 1, size=size)
        offsets = [0]

        offset = 0
        for i in range(size):
            if is_valid[i]:
                offset += int(list_sizes[i])
            offsets.append(offset)

        # The offset now is the total number of elements in the child array
        values = self.value_type.generate_column(offset)

        if name is None:
            name = self.name
        return ListColumn(name, size, is_valid, offsets, values)

class ListColumn(Column):

    def __init__(self, name, count, is_valid, offsets, values):
        super(ListColumn, self).__init__(name, count)
        self.is_valid = is_valid
        self.offsets = offsets
        self.values = values

    def _get_buffers(self):
        return [
            ('VALIDITY', [int(v) for v in self.is_valid]),
            ('OFFSET', list(self.offsets))
        ]

    def _get_children(self):
        return [self.values.get_json()]

class StructType(DataType):

    def __init__(self, name, field_types, nullable=True):
        super(StructType, self).__init__(name, nullable=nullable)
        self.field_types = field_types

    def _get_type(self):
        return OrderedDict([
            ('name', 'struct')
        ])

    def _get_children(self):
        return [type_.get_json() for type_ in self.field_types]

    def generate_column(self, size, name=None):
        is_valid = self._make_is_valid(size)

        field_values = [type_.generate_column(size)
                        for type_ in self.field_types]
        if name is None:
            name = self.name
        return StructColumn(name, size, is_valid, field_values)

class Dictionary(object):

    def __init__(self, id_, field, values, ordered=False):
        self.id_ = id_
        self.field = field
        self.values = values
        self.ordered = ordered

    def __len__(self):
        return len(self.values)

    def get_json(self):
        dummy_batch = JsonRecordBatch(len(self.values), [self.values])
        return OrderedDict([
            ('id', self.id_),
            ('data', dummy_batch.get_json())
        ])

class DictionaryType(DataType):

    def __init__(self, name, index_type, dictionary, nullable=True):
        super(DictionaryType, self).__init__(name, nullable=nullable)
        assert isinstance(index_type, IntegerType)
        assert isinstance(dictionary, Dictionary)

        self.index_type = index_type
        self.dictionary = dictionary

    def get_json(self):
        dict_field = self.dictionary.field
        return OrderedDict([
            ('name', self.name),
            ('type', dict_field._get_type()),
            ('nullable', self.nullable),
            ('children', dict_field._get_children()),
            ('dictionary', OrderedDict([
                ('id', self.dictionary.id_),
                ('indexType', self.index_type._get_type()),
                ('isOrdered', self.dictionary.ordered)
            ]))
        ])

    def generate_column(self, size, name=None):
        if name is None:
            name = self.name
        return self.index_type.generate_range(size, 0, len(self.dictionary),
                                              name=name)

class StructColumn(Column):

    def __init__(self, name, count, is_valid, field_values):
        super(StructColumn, self).__init__(name, count)
        self.is_valid = is_valid
        self.field_values = field_values

    def _get_buffers(self):
        return [
            ('VALIDITY', [int(v) for v in self.is_valid])
        ]

    def _get_children(self):
        return [field.get_json() for field in self.field_values]

class JsonRecordBatch(object):

    def __init__(self, count, columns):
        self.count = count
        self.columns = columns

    def get_json(self):
        return OrderedDict([
            ('count', self.count),
            ('columns', [col.get_json() for col in self.columns])
        ])

class JsonFile(object):

    def __init__(self, name, schema, batches, dictionaries=None):
        self.name = name
        self.schema = schema
        self.dictionaries = dictionaries or []
        self.batches = batches

    def get_json(self):
        entries = [
            ('schema', self.schema.get_json())
        ]

        if len(self.dictionaries) > 0:
            entries.append(('dictionaries',
                            [dictionary.get_json()
                             for dictionary in self.dictionaries]))

        entries.append(('batches', [batch.get_json()
                                    for batch in self.batches]))
        return OrderedDict(entries)

    def write(self, path):
        with open(path, 'wb') as f:
            f.write(json.dumps(self.get_json(), indent=2).encode('utf-8'))

def get_field(name, type_, nullable=True):
    if type_ == 'binary':
        return BinaryType(name, nullable=nullable)
    elif type_ == 'utf8':
        return StringType(name, nullable=nullable)
    elif type_.startswith('fixedsizebinary_'):
        byte_width = int(type_.split('_')[1])
        return FixedSizeBinaryType(name, byte_width=byte_width, nullable=nullable)

    dtype = np.dtype(type_)

    if dtype.kind in ('i', 'u'):
        return IntegerType(name, dtype.kind == 'i', dtype.itemsize * 8,
                           nullable=nullable)
    elif dtype.kind == 'f':
        return FloatingPointType(name, dtype.itemsize * 8,
                                 nullable=nullable)
    elif dtype.kind == 'b':
        return BooleanType(name, nullable=nullable)
    else:
        raise TypeError(dtype)

def _generate_file(name, fields, batch_sizes, dictionaries=None):
    schema = JsonSchema(fields)
    batches = []
    for size in batch_sizes:
        columns = []
        for field in fields:
            col = field.generate_column(size)
            columns.append(col)

        batches.append(JsonRecordBatch(size, columns))

    return JsonFile(name, schema, batches, dictionaries)

def generate_primitive_case(batch_sizes, name='primitive'):
    types = ['bool', 'int8', 'int16', 'int32', 'int64',
             'uint8', 'uint16', 'uint32', 'uint64',
             'float32', 'float64', 'binary', 'utf8',
             'fixedsizebinary_19', 'fixedsizebinary_120']

    fields = []

    for type_ in types:
        fields.append(get_field(type_ + "_nullable", type_, True))
        fields.append(get_field(type_ + "_nonnullable", type_, False))

    return _generate_file(name, fields, batch_sizes)

def generate_decimal_case():
    fields = [
        DecimalType(name='f{}'.format(i), precision=precision, scale=2)
        for i, precision in enumerate(range(3, 39))
    ]

    possible_batch_sizes = 7, 10
    batch_sizes = [possible_batch_sizes[i % 2] for i in range(len(fields))]

    return _generate_file('decimal', fields, batch_sizes)

def generate_datetime_case():
    fields = [
        DateType('f0', DateType.DAY),
        DateType('f1', DateType.MILLISECOND),
        TimeType('f2', 's'),
        TimeType('f3', 'ms'),
        TimeType('f4', 'us'),
        TimeType('f5', 'ns'),
        TimestampType('f6', 's'),
        TimestampType('f7', 'ms'),
        TimestampType('f8', 'us'),
        TimestampType('f9', 'ns'),
        TimestampType('f10', 'ms', tz=None),
        TimestampType('f11', 's', tz='UTC'),
        TimestampType('f12', 'ms', tz='US/Eastern'),
        TimestampType('f13', 'us', tz='Europe/Paris'),
        TimestampType('f14', 'ns', tz='US/Pacific')
    ]

    batch_sizes = [7, 10]
    return _generate_file("datetime", fields, batch_sizes)

def generate_nested_case():
    fields = [
        ListType('list_nullable', get_field('item', 'int32')),
        StructType('struct_nullable', [get_field('f1', 'int32'),
                                       get_field('f2', 'utf8')]),

        # TODO(wesm): this causes segfault
        # ListType('list_nonnullable', get_field('item', 'int32'), False),
    ]

    batch_sizes = [7, 10]
    return _generate_file("nested", fields, batch_sizes)

def generate_dictionary_case():
    dict_type1 = StringType('dictionary1')
    dict_type2 = get_field('dictionary2', 'int64')

    dict1 = Dictionary(0, dict_type1,
                       dict_type1.generate_column(10, name='DICT0'))
    dict2 = Dictionary(1, dict_type2,
                       dict_type2.generate_column(50, name='DICT1'))

    fields = [
        DictionaryType('dict1_0', get_field('', 'int8'), dict1),
        DictionaryType('dict1_1', get_field('', 'int32'), dict1),
        DictionaryType('dict2_0', get_field('', 'int16'), dict2)
    ]
    batch_sizes = [7, 10]
    return _generate_file("dictionary", fields, batch_sizes,
                          dictionaries=[dict1, dict2])

def get_generated_json_files():
    temp_dir = tempfile.mkdtemp()

    def _temp_path():
        return

    file_objs = [
        generate_primitive_case([17, 20], name='primitive'),
        generate_primitive_case([0, 0, 0], name='primitive_zerolength'),
        generate_decimal_case(),
        generate_datetime_case(),
        generate_nested_case(),
        generate_dictionary_case()
    ]

    generated_paths = []
    for file_obj in file_objs:
        out_path = os.path.join(temp_dir, 'generated_' +
                                file_obj.name + '.json')
        file_obj.write(out_path)
        generated_paths.append(out_path)

    return generated_paths

# ----------------------------------------------------------------------
# Testing harness

class IntegrationRunner(object):

    def __init__(self, json_files, testers, debug=False):
        self.json_files = json_files
        self.testers = testers
        self.temp_dir = tempfile.mkdtemp()
        self.debug = debug

    def run(self):
        for producer, consumer in itertools.product(filter(lambda t: t.PRODUCER, self.testers),
                                                    filter(lambda t: t.CONSUMER, self.testers)):
            self._compare_implementations(producer, consumer)

    def _compare_implementations(self, producer, consumer):
        print('##########################################################')
        print(
            '{0} producing, {1} consuming'.format(producer.name, consumer.name)
        )
        print('##########################################################')

        for json_path in self.json_files:
            print('==========================================================')
            print('Testing file {0}'.format(json_path))
            print('==========================================================')

            name = os.path.splitext(os.path.basename(json_path))[0]

            # Make the random access file
            print('-- Creating binary inputs')
            producer_file_path = os.path.join(self.temp_dir, guid() + '_' +
                                              name + '.json_to_arrow')
            producer.json_to_file(json_path, producer_file_path)

            # Validate the file
            print('-- Validating file')
            consumer.validate(json_path, producer_file_path)

            print('-- Validating stream')
            producer_stream_path = os.path.join(self.temp_dir, guid() + '_' +
                                                name + '.arrow_to_stream')
            consumer_file_path = os.path.join(self.temp_dir, guid() + '_' +
                                              name + '.stream_to_arrow')
            producer.file_to_stream(producer_file_path,
                                    producer_stream_path)
            consumer.stream_to_file(producer_stream_path,
                                    consumer_file_path)
            consumer.validate(json_path, consumer_file_path)

class Tester(object):
    PRODUCER = False
    CONSUMER = False

    def __init__(self, debug=False):
        self.debug = debug

    def json_to_file(self, json_path, arrow_path):
        raise NotImplementedError

    def stream_to_file(self, stream_path, file_path):
        raise NotImplementedError

    def file_to_stream(self, file_path, stream_path):
        raise NotImplementedError

    def validate(self, json_path, arrow_path):
        raise NotImplementedError

class JavaTester(Tester):
    PRODUCER = True
    CONSUMER = True

    _arrow_version = load_version_from_pom()
    ARROW_TOOLS_JAR = os.environ.get(
        'ARROW_JAVA_INTEGRATION_JAR',
        os.path.join(ARROW_HOME,
                     'java/tools/target/arrow-tools-{}-'
                     'jar-with-dependencies.jar'.format(_arrow_version)))

    name = 'Java'

    def _run(self, arrow_path=None, json_path=None, command='VALIDATE'):
        cmd = ['java', '-cp', self.ARROW_TOOLS_JAR,
               'org.apache.arrow.tools.Integration']

        if arrow_path is not None:
            cmd.extend(['-a', arrow_path])

        if json_path is not None:
            cmd.extend(['-j', json_path])

        cmd.extend(['-c', command])

        if self.debug:
            print(' '.join(cmd))

        run_cmd(cmd)

    def validate(self, json_path, arrow_path):
        return self._run(arrow_path, json_path, 'VALIDATE')

    def json_to_file(self, json_path, arrow_path):
        return self._run(arrow_path, json_path, 'JSON_TO_ARROW')

    def stream_to_file(self, stream_path, file_path):
        cmd = ['java', '-cp', self.ARROW_TOOLS_JAR,
               'org.apache.arrow.tools.StreamToFile',
               stream_path, file_path]
        if self.debug:
            print(' '.join(cmd))
        run_cmd(cmd)

    def file_to_stream(self, file_path, stream_path):
        cmd = ['java', '-cp', self.ARROW_TOOLS_JAR,
               'org.apache.arrow.tools.FileToStream',
               file_path, stream_path]
        if self.debug:
            print(' '.join(cmd))
        run_cmd(cmd)

class CPPTester(Tester):
    PRODUCER = True
    CONSUMER = True

    EXE_PATH = os.environ.get(
        'ARROW_CPP_EXE_PATH',
        os.path.join(ARROW_HOME, 'cpp/build/debug'))

    CPP_INTEGRATION_EXE = os.path.join(EXE_PATH, 'json-integration-test')
    STREAM_TO_FILE = os.path.join(EXE_PATH, 'stream-to-file')
    FILE_TO_STREAM = os.path.join(EXE_PATH, 'file-to-stream')

    name = 'C++'

    def _run(self, arrow_path=None, json_path=None, command='VALIDATE'):
        cmd = [self.CPP_INTEGRATION_EXE, '--integration']

        if arrow_path is not None:
            cmd.append('--arrow=' + arrow_path)

        if json_path is not None:
            cmd.append('--json=' + json_path)

        cmd.append('--mode=' + command)

        if self.debug:
            print(' '.join(cmd))

        run_cmd(cmd)

    def validate(self, json_path, arrow_path):
        return self._run(arrow_path, json_path, 'VALIDATE')

    def json_to_file(self, json_path, arrow_path):
        return self._run(arrow_path, json_path, 'JSON_TO_ARROW')

    def stream_to_file(self, stream_path, file_path):
        cmd = ['cat', stream_path, '|', self.STREAM_TO_FILE, '>', file_path]
        cmd = ' '.join(cmd)
        if self.debug:
            print(cmd)
        os.system(cmd)

    def file_to_stream(self, file_path, stream_path):
        cmd = [self.FILE_TO_STREAM, file_path, '>', stream_path]
        cmd = ' '.join(cmd)
        if self.debug:
            print(cmd)
        os.system(cmd)

class JSTester(Tester):
    PRODUCER = False
    CONSUMER = True

    INTEGRATION_EXE = os.path.join(ARROW_HOME, 'js/bin/integration.js')

    name = 'JS'

    def _run(self, arrow_path=None, json_path=None, command='VALIDATE'):
        cmd = [self.INTEGRATION_EXE]

        if arrow_path is not None:
            cmd.extend(['-a', arrow_path])

        if json_path is not None:
            cmd.extend(['-j', json_path])

        cmd.extend(['--mode', command])

        if self.debug:
            print(' '.join(cmd))

        run_cmd(cmd)

    def validate(self, json_path, arrow_path):
        return self._run(arrow_path, json_path, 'VALIDATE')

    def stream_to_file(self, stream_path, file_path):
        # Just copy stream to file, we can read the stream directly
        cmd = ['cp', stream_path, file_path]
        cmd = ' '.join(cmd)
        if self.debug:
            print(cmd)
        os.system(cmd)

def get_static_json_files():
    glob_pattern = os.path.join(ARROW_HOME, 'integration', 'data', '*.json')
    return glob.glob(glob_pattern)

def run_all_tests(debug=False):
    testers = [CPPTester(debug=debug), JavaTester(debug=debug), JSTester(debug=debug)]
    static_json_files = get_static_json_files()
    generated_json_files = get_generated_json_files()
    json_files = static_json_files + generated_json_files

    runner = IntegrationRunner(json_files, testers, debug=debug)
    runner.run()
    print('-- All tests passed!')

def write_js_test_json(directory):
    generate_nested_case().write(os.path.join(directory, 'nested.json'))
    generate_decimal_case().write(os.path.join(directory, 'decimal.json'))
    generate_datetime_case().write(os.path.join(directory, 'datetime.json'))
    (generate_dictionary_case()
     .write(os.path.join(directory, 'dictionary.json')))
    (generate_primitive_case([7, 10])
     .write(os.path.join(directory, 'primitive.json')))
    (generate_primitive_case([0, 0, 0])
     .write(os.path.join(directory, 'primitive-empty.json')))

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Arrow integration test CLI')
    parser.add_argument('--write_generated_json', dest='generated_json_path',
                        action='store', default=False,
                        help='Generate test JSON')
    parser.add_argument('--debug', dest='debug', action='store_true',
                        default=False,
                        help='Run executables in debug mode as relevant')
    args = parser.parse_args()
    if args.generated_json_path:
        try:
            os.makedirs(args.generated_json_path)
        except OSError as e:
            if e.errno != errno.EEXIST:
                raise
        write_js_test_json(args.generated_json_path)
    else:
        run_all_tests(debug=args.debug)


# coding=utf-8
# pylint: disable-msg=E1101,W0612

from datetime import datetime, timedelta
from distutils.version import LooseVersion

import numpy as np
from numpy import nan
import pytest
import pytz

from pandas._libs.tslib import iNaT
from pandas.compat import range
from pandas.errors import PerformanceWarning
import pandas.util._test_decorators as td

import pandas as pd
from pandas import (
    Categorical, DataFrame, Index, IntervalIndex, MultiIndex, NaT, Series,
    Timestamp, date_range, isna)
from pandas.core.series import remove_na
import pandas.util.testing as tm
from pandas.util.testing import assert_frame_equal, assert_series_equal

try:
    import scipy
    _is_scipy_ge_0190 = (LooseVersion(scipy.__version__) >=
                         LooseVersion('0.19.0'))
except ImportError:
    _is_scipy_ge_0190 = False

def _skip_if_no_pchip():
    try:
        from scipy.interpolate import pchip_interpolate  # noqa
    except ImportError:
        import pytest
        pytest.skip('scipy.interpolate.pchip missing')

def _skip_if_no_akima():
    try:
        from scipy.interpolate import Akima1DInterpolator  # noqa
    except ImportError:
        import pytest
        pytest.skip('scipy.interpolate.Akima1DInterpolator missing')

def _simple_ts(start, end, freq='D'):
    rng = date_range(start, end, freq=freq)
    return Series(np.random.randn(len(rng)), index=rng)

class TestSeriesMissingData():

    def test_remove_na_deprecation(self):
        # see gh-16971
        with tm.assert_produces_warning(FutureWarning):
            remove_na(Series([]))

    def test_timedelta_fillna(self):
        # GH 3371
        s = Series([Timestamp('20130101'), Timestamp('20130101'),
                    Timestamp('20130102'), Timestamp('20130103 9:01:01')])
        td = s.diff()

        # reg fillna
        result = td.fillna(0)
        expected = Series([timedelta(0), timedelta(0), timedelta(1),
                           timedelta(days=1, seconds=9 * 3600 + 60 + 1)])
        assert_series_equal(result, expected)

        # interprested as seconds
        result = td.fillna(1)
        expected = Series([timedelta(seconds=1), timedelta(0), timedelta(1),
                           timedelta(days=1, seconds=9 * 3600 + 60 + 1)])
        assert_series_equal(result, expected)

        result = td.fillna(timedelta(days=1, seconds=1))
        expected = Series([timedelta(days=1, seconds=1), timedelta(0),
                           timedelta(1),
                           timedelta(days=1, seconds=9 * 3600 + 60 + 1)])
        assert_series_equal(result, expected)

        result = td.fillna(np.timedelta64(int(1e9)))
        expected = Series([timedelta(seconds=1), timedelta(0), timedelta(1),
                           timedelta(days=1, seconds=9 * 3600 + 60 + 1)])
        assert_series_equal(result, expected)

        result = td.fillna(NaT)
        expected = Series([NaT, timedelta(0), timedelta(1),
                           timedelta(days=1, seconds=9 * 3600 + 60 + 1)],
                          dtype='m8[ns]')
        assert_series_equal(result, expected)

        # ffill
        td[2] = np.nan
        result = td.ffill()
        expected = td.fillna(0)
        expected[0] = np.nan
        assert_series_equal(result, expected)

        # bfill
        td[2] = np.nan
        result = td.bfill()
        expected = td.fillna(0)
        expected[2] = timedelta(days=1, seconds=9 * 3600 + 60 + 1)
        assert_series_equal(result, expected)

    def test_datetime64_fillna(self):

        s = Series([Timestamp('20130101'), Timestamp('20130101'), Timestamp(
            '20130102'), Timestamp('20130103 9:01:01')])
        s[2] = np.nan

        # reg fillna
        result = s.fillna(Timestamp('20130104'))
        expected = Series([Timestamp('20130101'), Timestamp(
            '20130101'), Timestamp('20130104'), Timestamp('20130103 9:01:01')])
        assert_series_equal(result, expected)

        result = s.fillna(NaT)
        expected = s
        assert_series_equal(result, expected)

        # ffill
        result = s.ffill()
        expected = Series([Timestamp('20130101'), Timestamp(
            '20130101'), Timestamp('20130101'), Timestamp('20130103 9:01:01')])
        assert_series_equal(result, expected)

        # bfill
        result = s.bfill()
        expected = Series([Timestamp('20130101'), Timestamp('20130101'),
                           Timestamp('20130103 9:01:01'), Timestamp(
                               '20130103 9:01:01')])
        assert_series_equal(result, expected)

        # GH 6587
        # make sure that we are treating as integer when filling
        # this also tests inference of a datetime-like with NaT's
        s = Series([pd.NaT, pd.NaT, '2013-08-05 15:30:00.000001'])
        expected = Series(
            ['2013-08-05 15:30:00.000001', '2013-08-05 15:30:00.000001',
             '2013-08-05 15:30:00.000001'], dtype='M8[ns]')
        result = s.fillna(method='backfill')
        assert_series_equal(result, expected)

    def test_datetime64_tz_fillna(self):

        for tz in ['US/Eastern', 'Asia/Tokyo']:
            # DatetimeBlock
            s = Series([Timestamp('2011-01-01 10:00'), pd.NaT,
                        Timestamp('2011-01-03 10:00'), pd.NaT])
            null_loc = pd.Series([False, True, False, True])

            result = s.fillna(pd.Timestamp('2011-01-02 10:00'))
            expected = Series([Timestamp('2011-01-01 10:00'),
                               Timestamp('2011-01-02 10:00'),
                               Timestamp('2011-01-03 10:00'),
                               Timestamp('2011-01-02 10:00')])
            tm.assert_series_equal(expected, result)
            # check s is not changed
            tm.assert_series_equal(pd.isna(s), null_loc)

            result = s.fillna(pd.Timestamp('2011-01-02 10:00', tz=tz))
            expected = Series([Timestamp('2011-01-01 10:00'),
                               Timestamp('2011-01-02 10:00', tz=tz),
                               Timestamp('2011-01-03 10:00'),
                               Timestamp('2011-01-02 10:00', tz=tz)])
            tm.assert_series_equal(expected, result)
            tm.assert_series_equal(pd.isna(s), null_loc)

            result = s.fillna('AAA')
            expected = Series([Timestamp('2011-01-01 10:00'), 'AAA',
                               Timestamp('2011-01-03 10:00'), 'AAA'],
                              dtype=object)
            tm.assert_series_equal(expected, result)
            tm.assert_series_equal(pd.isna(s), null_loc)

            result = s.fillna({1: pd.Timestamp('2011-01-02 10:00', tz=tz),
                               3: pd.Timestamp('2011-01-04 10:00')})
            expected = Series([Timestamp('2011-01-01 10:00'),
                               Timestamp('2011-01-02 10:00', tz=tz),
                               Timestamp('2011-01-03 10:00'),
                               Timestamp('2011-01-04 10:00')])
            tm.assert_series_equal(expected, result)
            tm.assert_series_equal(pd.isna(s), null_loc)

            result = s.fillna({1: pd.Timestamp('2011-01-02 10:00'),
                               3: pd.Timestamp('2011-01-04 10:00')})
            expected = Series([Timestamp('2011-01-01 10:00'),
                               Timestamp('2011-01-02 10:00'),
                               Timestamp('2011-01-03 10:00'),
                               Timestamp('2011-01-04 10:00')])
            tm.assert_series_equal(expected, result)
            tm.assert_series_equal(pd.isna(s), null_loc)

            # DatetimeBlockTZ
            idx = pd.DatetimeIndex(['2011-01-01 10:00', pd.NaT,
                                    '2011-01-03 10:00', pd.NaT], tz=tz)
            s = pd.Series(idx)
            assert s.dtype == 'datetime64[ns, {0}]'.format(tz)
            tm.assert_series_equal(pd.isna(s), null_loc)

            result = s.fillna(pd.Timestamp('2011-01-02 10:00'))
            expected = Series([Timestamp('2011-01-01 10:00', tz=tz),
                               Timestamp('2011-01-02 10:00'),
                               Timestamp('2011-01-03 10:00', tz=tz),
                               Timestamp('2011-01-02 10:00')])
            tm.assert_series_equal(expected, result)
            tm.assert_series_equal(pd.isna(s), null_loc)

            result = s.fillna(pd.Timestamp('2011-01-02 10:00', tz=tz))
            idx = pd.DatetimeIndex(['2011-01-01 10:00', '2011-01-02 10:00',
                                    '2011-01-03 10:00', '2011-01-02 10:00'],
                                   tz=tz)
            expected = Series(idx)
            tm.assert_series_equal(expected, result)
            tm.assert_series_equal(pd.isna(s), null_loc)

            result = s.fillna(pd.Timestamp('2011-01-02 10:00',
                                           tz=tz).to_pydatetime())
            idx = pd.DatetimeIndex(['2011-01-01 10:00', '2011-01-02 10:00',
                                    '2011-01-03 10:00', '2011-01-02 10:00'],
                                   tz=tz)
            expected = Series(idx)
            tm.assert_series_equal(expected, result)
            tm.assert_series_equal(pd.isna(s), null_loc)

            result = s.fillna('AAA')
            expected = Series([Timestamp('2011-01-01 10:00', tz=tz), 'AAA',
                               Timestamp('2011-01-03 10:00', tz=tz), 'AAA'],
                              dtype=object)
            tm.assert_series_equal(expected, result)
            tm.assert_series_equal(pd.isna(s), null_loc)

            result = s.fillna({1: pd.Timestamp('2011-01-02 10:00', tz=tz),
                               3: pd.Timestamp('2011-01-04 10:00')})
            expected = Series([Timestamp('2011-01-01 10:00', tz=tz),
                               Timestamp('2011-01-02 10:00', tz=tz),
                               Timestamp('2011-01-03 10:00', tz=tz),
                               Timestamp('2011-01-04 10:00')])
            tm.assert_series_equal(expected, result)
            tm.assert_series_equal(pd.isna(s), null_loc)

            result = s.fillna({1: pd.Timestamp('2011-01-02 10:00', tz=tz),
                               3: pd.Timestamp('2011-01-04 10:00', tz=tz)})
            expected = Series([Timestamp('2011-01-01 10:00', tz=tz),
                               Timestamp('2011-01-02 10:00', tz=tz),
                               Timestamp('2011-01-03 10:00', tz=tz),
                               Timestamp('2011-01-04 10:00', tz=tz)])
            tm.assert_series_equal(expected, result)
            tm.assert_series_equal(pd.isna(s), null_loc)

            # filling with a naive/other zone, coerce to object
            result = s.fillna(Timestamp('20130101'))
            expected = Series([Timestamp('2011-01-01 10:00', tz=tz),
                               Timestamp('2013-01-01'),
                               Timestamp('2011-01-03 10:00', tz=tz),
                               Timestamp('2013-01-01')])
            tm.assert_series_equal(expected, result)
            tm.assert_series_equal(pd.isna(s), null_loc)

            result = s.fillna(Timestamp('20130101', tz='US/Pacific'))
            expected = Series([Timestamp('2011-01-01 10:00', tz=tz),
                               Timestamp('2013-01-01', tz='US/Pacific'),
                               Timestamp('2011-01-03 10:00', tz=tz),
                               Timestamp('2013-01-01', tz='US/Pacific')])
            tm.assert_series_equal(expected, result)
            tm.assert_series_equal(pd.isna(s), null_loc)

        # with timezone
        # GH 15855
        df = pd.Series([pd.Timestamp('2012-11-11 00:00:00+01:00'), pd.NaT])
        exp = pd.Series([pd.Timestamp('2012-11-11 00:00:00+01:00'),
                         pd.Timestamp('2012-11-11 00:00:00+01:00')])
        assert_series_equal(df.fillna(method='pad'), exp)

        df = pd.Series([pd.NaT, pd.Timestamp('2012-11-11 00:00:00+01:00')])
        exp = pd.Series([pd.Timestamp('2012-11-11 00:00:00+01:00'),
                         pd.Timestamp('2012-11-11 00:00:00+01:00')])
        assert_series_equal(df.fillna(method='bfill'), exp)

    def test_fillna_consistency(self):
        # GH 16402
        # fillna with a tz aware to a tz-naive, should result in object

        s = Series([Timestamp('20130101'), pd.NaT])

        result = s.fillna(Timestamp('20130101', tz='US/Eastern'))
        expected = Series([Timestamp('20130101'),
                           Timestamp('2013-01-01', tz='US/Eastern')],
                          dtype='object')
        assert_series_equal(result, expected)

        # where (we ignore the errors=)
        result = s.where([True, False],
                         Timestamp('20130101', tz='US/Eastern'),
                         errors='ignore')
        assert_series_equal(result, expected)

        result = s.where([True, False],
                         Timestamp('20130101', tz='US/Eastern'),
                         errors='ignore')
        assert_series_equal(result, expected)

        # with a non-datetime
        result = s.fillna('foo')
        expected = Series([Timestamp('20130101'),
                           'foo'])
        assert_series_equal(result, expected)

        # assignment
        s2 = s.copy()
        s2[1] = 'foo'
        assert_series_equal(s2, expected)

    def test_datetime64tz_fillna_round_issue(self):
        # GH 14872

        data = pd.Series([pd.NaT, pd.NaT,
                          datetime(2016, 12, 12, 22, 24, 6, 100001,
                                   tzinfo=pytz.utc)])

        filled = data.fillna(method='bfill')

        expected = pd.Series([datetime(2016, 12, 12, 22, 24, 6,
                                       100001, tzinfo=pytz.utc),
                              datetime(2016, 12, 12, 22, 24, 6,
                                       100001, tzinfo=pytz.utc),
                              datetime(2016, 12, 12, 22, 24, 6,
                                       100001, tzinfo=pytz.utc)])

        assert_series_equal(filled, expected)

    def test_fillna_downcast(self):
        # GH 15277
        # infer int64 from float64
        s = pd.Series([1., np.nan])
        result = s.fillna(0, downcast='infer')
        expected = pd.Series([1, 0])
        assert_series_equal(result, expected)

        # infer int64 from float64 when fillna value is a dict
        s = pd.Series([1., np.nan])
        result = s.fillna({1: 0}, downcast='infer')
        expected = pd.Series([1, 0])
        assert_series_equal(result, expected)

    def test_fillna_int(self):
        s = Series(np.random.randint(-100, 100, 50))
        s.fillna(method='ffill', inplace=True)
        assert_series_equal(s.fillna(method='ffill', inplace=False), s)

    def test_fillna_raise(self):
        s = Series(np.random.randint(-100, 100, 50))
        pytest.raises(TypeError, s.fillna, [1, 2])
        pytest.raises(TypeError, s.fillna, (1, 2))

        # related GH 9217, make sure limit is an int and greater than 0
        s = Series([1, 2, 3, None])
        for limit in [-1, 0, 1., 2.]:
            for method in ['backfill', 'bfill', 'pad', 'ffill', None]:
                with pytest.raises(ValueError):
                    s.fillna(1, limit=limit, method=method)

    def test_categorical_nan_equality(self):
        cat = Series(Categorical(["a", "b", "c", np.nan]))
        exp = Series([True, True, True, False])
        res = (cat == cat)
        tm.assert_series_equal(res, exp)

    def test_categorical_nan_handling(self):

        # NaNs are represented as -1 in labels
        s = Series(Categorical(["a", "b", np.nan, "a"]))
        tm.assert_index_equal(s.cat.categories, Index(["a", "b"]))
        tm.assert_numpy_array_equal(s.values.codes,
                                    np.array([0, 1, -1, 0], dtype=np.int8))

    @pytest.mark.parametrize('fill_value, expected_output', [
        ('a', ['a', 'a', 'b', 'a', 'a']),
        ({1: 'a', 3: 'b', 4: 'b'}, ['a', 'a', 'b', 'b', 'b']),
        ({1: 'a'}, ['a', 'a', 'b', np.nan, np.nan]),
        ({1: 'a', 3: 'b'}, ['a', 'a', 'b', 'b', np.nan]),
        (Series('a'), ['a', np.nan, 'b', np.nan, np.nan]),
        (Series('a', index=[1]), ['a', 'a', 'b', np.nan, np.nan]),
        (Series({1: 'a', 3: 'b'}), ['a', 'a', 'b', 'b', np.nan]),
        (Series(['a', 'b'], index=[3, 4]), ['a', np.nan, 'b', 'a', 'b'])
    ])
    def test_fillna_categorical(self, fill_value, expected_output):
        # GH 17033
        # Test fillna for a Categorical series
        data = ['a', np.nan, 'b', np.nan, np.nan]
        s = Series(Categorical(data, categories=['a', 'b']))
        exp = Series(Categorical(expected_output, categories=['a', 'b']))
        tm.assert_series_equal(s.fillna(fill_value), exp)

    def test_fillna_categorical_raise(self):
        data = ['a', np.nan, 'b', np.nan, np.nan]
        s = Series(Categorical(data, categories=['a', 'b']))

        with tm.assert_raises_regex(ValueError,
                                    "fill value must be in categories"):
            s.fillna('d')

        with tm.assert_raises_regex(ValueError,
                                    "fill value must be in categories"):
            s.fillna(Series('d'))

        with tm.assert_raises_regex(ValueError,
                                    "fill value must be in categories"):
            s.fillna({1: 'd', 3: 'a'})

        with tm.assert_raises_regex(TypeError,
                                    '"value" parameter must be a scalar or '
                                    'dict, but you passed a "list"'):
            s.fillna(['a', 'b'])

        with tm.assert_raises_regex(TypeError,
                                    '"value" parameter must be a scalar or '
                                    'dict, but you passed a "tuple"'):
            s.fillna(('a', 'b'))

        with tm.assert_raises_regex(TypeError,
                                    '"value" parameter must be a scalar, dict '
                                    'or Series, but you passed a "DataFrame"'):
            s.fillna(DataFrame({1: ['a'], 3: ['b']}))

    def test_fillna_nat(self):
        series = Series([0, 1, 2, iNaT], dtype='M8[ns]')

        filled = series.fillna(method='pad')
        filled2 = series.fillna(value=series.values[2])

        expected = series.copy()
        expected.values[3] = expected.values[2]

        assert_series_equal(filled, expected)
        assert_series_equal(filled2, expected)

        df = DataFrame({'A': series})
        filled = df.fillna(method='pad')
        filled2 = df.fillna(value=series.values[2])
        expected = DataFrame({'A': expected})
        assert_frame_equal(filled, expected)
        assert_frame_equal(filled2, expected)

        series = Series([iNaT, 0, 1, 2], dtype='M8[ns]')

        filled = series.fillna(method='bfill')
        filled2 = series.fillna(value=series[1])

        expected = series.copy()
        expected[0] = expected[1]

        assert_series_equal(filled, expected)
        assert_series_equal(filled2, expected)

        df = DataFrame({'A': series})
        filled = df.fillna(method='bfill')
        filled2 = df.fillna(value=series[1])
        expected = DataFrame({'A': expected})
        assert_frame_equal(filled, expected)
        assert_frame_equal(filled2, expected)

    def test_isna_for_inf(self):
        s = Series(['a', np.inf, np.nan, 1.0])
        with pd.option_context('mode.use_inf_as_na', True):
            r = s.isna()
            dr = s.dropna()
        e = Series([False, True, True, False])
        de = Series(['a', 1.0], index=[0, 3])
        tm.assert_series_equal(r, e)
        tm.assert_series_equal(dr, de)

    @tm.capture_stdout
    def test_isnull_for_inf_deprecated(self):
        # gh-17115
        s = Series(['a', np.inf, np.nan, 1.0])
        with pd.option_context('mode.use_inf_as_null', True):
            r = s.isna()
            dr = s.dropna()

        e = Series([False, True, True, False])
        de = Series(['a', 1.0], index=[0, 3])
        tm.assert_series_equal(r, e)
        tm.assert_series_equal(dr, de)

    def test_fillna(self, datetime_series):
        ts = Series([0., 1., 2., 3., 4.], index=tm.makeDateIndex(5))

        tm.assert_series_equal(ts, ts.fillna(method='ffill'))

        ts[2] = np.NaN

        exp = Series([0., 1., 1., 3., 4.], index=ts.index)
        tm.assert_series_equal(ts.fillna(method='ffill'), exp)

        exp = Series([0., 1., 3., 3., 4.], index=ts.index)
        tm.assert_series_equal(ts.fillna(method='backfill'), exp)

        exp = Series([0., 1., 5., 3., 4.], index=ts.index)
        tm.assert_series_equal(ts.fillna(value=5), exp)

        pytest.raises(ValueError, ts.fillna)
        pytest.raises(ValueError, datetime_series.fillna, value=0,
                      method='ffill')

        # GH 5703
        s1 = Series([np.nan])
        s2 = Series([1])
        result = s1.fillna(s2)
        expected = Series([1.])
        assert_series_equal(result, expected)
        result = s1.fillna({})
        assert_series_equal(result, s1)
        result = s1.fillna(Series(()))
        assert_series_equal(result, s1)
        result = s2.fillna(s1)
        assert_series_equal(result, s2)
        result = s1.fillna({0: 1})
        assert_series_equal(result, expected)
        result = s1.fillna({1: 1})
        assert_series_equal(result, Series([np.nan]))
        result = s1.fillna({0: 1, 1: 1})
        assert_series_equal(result, expected)
        result = s1.fillna(Series({0: 1, 1: 1}))
        assert_series_equal(result, expected)
        result = s1.fillna(Series({0: 1, 1: 1}, index=[4, 5]))
        assert_series_equal(result, s1)

        s1 = Series([0, 1, 2], list('abc'))
        s2 = Series([0, np.nan, 2], list('bac'))
        result = s2.fillna(s1)
        expected = Series([0, 0, 2.], list('bac'))
        assert_series_equal(result, expected)

        # limit
        s = Series(np.nan, index=[0, 1, 2])
        result = s.fillna(999, limit=1)
        expected = Series([999, np.nan, np.nan], index=[0, 1, 2])
        assert_series_equal(result, expected)

        result = s.fillna(999, limit=2)
        expected = Series([999, 999, np.nan], index=[0, 1, 2])
        assert_series_equal(result, expected)

        # GH 9043
        # make sure a string representation of int/float values can be filled
        # correctly without raising errors or being converted
        vals = ['0', '1.5', '-0.3']
        for val in vals:
            s = Series([0, 1, np.nan, np.nan, 4], dtype='float64')
            result = s.fillna(val)
            expected = Series([0, 1, val, val, 4], dtype='object')
            assert_series_equal(result, expected)

    def test_fillna_bug(self):
        x = Series([nan, 1., nan, 3., nan], ['z', 'a', 'b', 'c', 'd'])
        filled = x.fillna(method='ffill')
        expected = Series([nan, 1., 1., 3., 3.], x.index)
        assert_series_equal(filled, expected)

        filled = x.fillna(method='bfill')
        expected = Series([1., 1., 3., 3., nan], x.index)
        assert_series_equal(filled, expected)

    def test_fillna_inplace(self):
        x = Series([nan, 1., nan, 3., nan], ['z', 'a', 'b', 'c', 'd'])
        y = x.copy()

        y.fillna(value=0, inplace=True)

        expected = x.fillna(value=0)
        assert_series_equal(y, expected)

    def test_fillna_invalid_method(self, datetime_series):
        try:
            datetime_series.fillna(method='ffil')
        except ValueError as inst:
            assert 'ffil' in str(inst)

    def test_ffill(self):
        ts = Series([0., 1., 2., 3., 4.], index=tm.makeDateIndex(5))
        ts[2] = np.NaN
        assert_series_equal(ts.ffill(), ts.fillna(method='ffill'))

    def test_ffill_mixed_dtypes_without_missing_data(self):
        # GH14956
        series = pd.Series([datetime(2015, 1, 1, tzinfo=pytz.utc), 1])
        result = series.ffill()
        assert_series_equal(series, result)

    def test_bfill(self):
        ts = Series([0., 1., 2., 3., 4.], index=tm.makeDateIndex(5))
        ts[2] = np.NaN
        assert_series_equal(ts.bfill(), ts.fillna(method='bfill'))

    def test_timedelta64_nan(self):

        td = Series([timedelta(days=i) for i in range(10)])

        # nan ops on timedeltas
        td1 = td.copy()
        td1[0] = np.nan
        assert isna(td1[0])
        assert td1[0].value == iNaT
        td1[0] = td[0]
        assert not isna(td1[0])

        td1[1] = iNaT
        assert isna(td1[1])
        assert td1[1].value == iNaT
        td1[1] = td[1]
        assert not isna(td1[1])

        td1[2] = NaT
        assert isna(td1[2])
        assert td1[2].value == iNaT
        td1[2] = td[2]
        assert not isna(td1[2])

        # boolean setting
        # this doesn't work, not sure numpy even supports it
        # result = td[(td>np.timedelta64(timedelta(days=3))) &
        # td<np.timedelta64(timedelta(days=7)))] = np.nan
        # assert isna(result).sum() == 7

        # NumPy limitiation =(

        # def test_logical_range_select(self):
        #     np.random.seed(12345)
        #     selector = -0.5 <= datetime_series <= 0.5
        #     expected = (datetime_series >= -0.5) & (datetime_series <= 0.5)
        #     assert_series_equal(selector, expected)

    def test_dropna_empty(self):
        s = Series([])
        assert len(s.dropna()) == 0
        s.dropna(inplace=True)
        assert len(s) == 0

        # invalid axis
        pytest.raises(ValueError, s.dropna, axis=1)

    def test_datetime64_tz_dropna(self):
        # DatetimeBlock
        s = Series([Timestamp('2011-01-01 10:00'), pd.NaT, Timestamp(
            '2011-01-03 10:00'), pd.NaT])
        result = s.dropna()
        expected = Series([Timestamp('2011-01-01 10:00'),
                           Timestamp('2011-01-03 10:00')], index=[0, 2])
        tm.assert_series_equal(result, expected)

        # DatetimeBlockTZ
        idx = pd.DatetimeIndex(['2011-01-01 10:00', pd.NaT,
                                '2011-01-03 10:00', pd.NaT],
                               tz='Asia/Tokyo')
        s = pd.Series(idx)
        assert s.dtype == 'datetime64[ns, Asia/Tokyo]'
        result = s.dropna()
        expected = Series([Timestamp('2011-01-01 10:00', tz='Asia/Tokyo'),
                           Timestamp('2011-01-03 10:00', tz='Asia/Tokyo')],
                          index=[0, 2])
        assert result.dtype == 'datetime64[ns, Asia/Tokyo]'
        tm.assert_series_equal(result, expected)

    def test_dropna_no_nan(self):
        for s in [Series([1, 2, 3], name='x'), Series(
                [False, True, False], name='x')]:

            result = s.dropna()
            tm.assert_series_equal(result, s)
            assert result is not s

            s2 = s.copy()
            s2.dropna(inplace=True)
            tm.assert_series_equal(s2, s)

    def test_dropna_intervals(self):
        s = Series([np.nan, 1, 2, 3], IntervalIndex.from_arrays(
            [np.nan, 0, 1, 2],
            [np.nan, 1, 2, 3]))

        result = s.dropna()
        expected = s.iloc[1:]
        assert_series_equal(result, expected)

    def test_valid(self, datetime_series):
        ts = datetime_series.copy()
        ts[::2] = np.NaN

        result = ts.dropna()
        assert len(result) == ts.count()
        tm.assert_series_equal(result, ts[1::2])
        tm.assert_series_equal(result, ts[pd.notna(ts)])

    def test_isna(self):
        ser = Series([0, 5.4, 3, nan, -0.001])
        expected = Series([False, False, False, True, False])
        tm.assert_series_equal(ser.isna(), expected)

        ser = Series(["hi", "", nan])
        expected = Series([False, False, True])
        tm.assert_series_equal(ser.isna(), expected)

    def test_notna(self):
        ser = Series([0, 5.4, 3, nan, -0.001])
        expected = Series([True, True, True, False, True])
        tm.assert_series_equal(ser.notna(), expected)

        ser = Series(["hi", "", nan])
        expected = Series([True, True, False])
        tm.assert_series_equal(ser.notna(), expected)

    def test_pad_nan(self):
        x = Series([np.nan, 1., np.nan, 3., np.nan], ['z', 'a', 'b', 'c', 'd'],
                   dtype=float)

        x.fillna(method='pad', inplace=True)

        expected = Series([np.nan, 1.0, 1.0, 3.0, 3.0],
                          ['z', 'a', 'b', 'c', 'd'], dtype=float)
        assert_series_equal(x[1:], expected[1:])
        assert np.isnan(x[0]), np.isnan(expected[0])

    def test_pad_require_monotonicity(self):
        rng = date_range('1/1/2000', '3/1/2000', freq='B')

        # neither monotonic increasing or decreasing
        rng2 = rng[[1, 0, 2]]

        pytest.raises(ValueError, rng2.get_indexer, rng, method='pad')

    def test_dropna_preserve_name(self, datetime_series):
        datetime_series[:5] = np.nan
        result = datetime_series.dropna()
        assert result.name == datetime_series.name
        name = datetime_series.name
        ts = datetime_series.copy()
        ts.dropna(inplace=True)
        assert ts.name == name

    def test_fill_value_when_combine_const(self):
        # GH12723
        s = Series([0, 1, np.nan, 3, 4, 5])

        exp = s.fillna(0).add(2)
        res = s.add(2, fill_value=0)
        assert_series_equal(res, exp)

    def test_series_fillna_limit(self):
        index = np.arange(10)
        s = Series(np.random.randn(10), index=index)

        result = s[:2].reindex(index)
        result = result.fillna(method='pad', limit=5)

        expected = s[:2].reindex(index).fillna(method='pad')
        expected[-3:] = np.nan
        assert_series_equal(result, expected)

        result = s[-2:].reindex(index)
        result = result.fillna(method='bfill', limit=5)

        expected = s[-2:].reindex(index).fillna(method='backfill')
        expected[:3] = np.nan
        assert_series_equal(result, expected)

    def test_sparse_series_fillna_limit(self):
        index = np.arange(10)
        s = Series(np.random.randn(10), index=index)

        ss = s[:2].reindex(index).to_sparse()
        # TODO: what is this test doing? why are result an expected
        # the same call to fillna?
        with tm.assert_produces_warning(PerformanceWarning):
            # TODO: release-note fillna performance warning
            result = ss.fillna(method='pad', limit=5)
            expected = ss.fillna(method='pad', limit=5)
        expected = expected.to_dense()
        expected[-3:] = np.nan
        expected = expected.to_sparse()
        assert_series_equal(result, expected)

        ss = s[-2:].reindex(index).to_sparse()
        with tm.assert_produces_warning(PerformanceWarning):
            result = ss.fillna(method='backfill', limit=5)
            expected = ss.fillna(method='backfill')
        expected = expected.to_dense()
        expected[:3] = np.nan
        expected = expected.to_sparse()
        assert_series_equal(result, expected)

    def test_sparse_series_pad_backfill_limit(self):
        index = np.arange(10)
        s = Series(np.random.randn(10), index=index)
        s = s.to_sparse()

        result = s[:2].reindex(index, method='pad', limit=5)
        with tm.assert_produces_warning(PerformanceWarning):
            expected = s[:2].reindex(index).fillna(method='pad')
        expected = expected.to_dense()
        expected[-3:] = np.nan
        expected = expected.to_sparse()
        assert_series_equal(result, expected)

        result = s[-2:].reindex(index, method='backfill', limit=5)
        with tm.assert_produces_warning(PerformanceWarning):
            expected = s[-2:].reindex(index).fillna(method='backfill')
        expected = expected.to_dense()
        expected[:3] = np.nan
        expected = expected.to_sparse()
        assert_series_equal(result, expected)

    def test_series_pad_backfill_limit(self):
        index = np.arange(10)
        s = Series(np.random.randn(10), index=index)

        result = s[:2].reindex(index, method='pad', limit=5)

        expected = s[:2].reindex(index).fillna(method='pad')
        expected[-3:] = np.nan
        assert_series_equal(result, expected)

        result = s[-2:].reindex(index, method='backfill', limit=5)

        expected = s[-2:].reindex(index).fillna(method='backfill')
        expected[:3] = np.nan
        assert_series_equal(result, expected)

class TestSeriesInterpolateData():

    def test_interpolate(self, datetime_series, string_series):
        ts = Series(np.arange(len(datetime_series), dtype=float),
                    datetime_series.index)

        ts_copy = ts.copy()
        ts_copy[5:10] = np.NaN

        linear_interp = ts_copy.interpolate(method='linear')
        tm.assert_series_equal(linear_interp, ts)

        ord_ts = Series([d.toordinal() for d in datetime_series.index],
                        index=datetime_series.index).astype(float)

        ord_ts_copy = ord_ts.copy()
        ord_ts_copy[5:10] = np.NaN

        time_interp = ord_ts_copy.interpolate(method='time')
        tm.assert_series_equal(time_interp, ord_ts)

        # try time interpolation on a non-TimeSeries
        # Only raises ValueError if there are NaNs.
        non_ts = string_series.copy()
        non_ts[0] = np.NaN
        pytest.raises(ValueError, non_ts.interpolate, method='time')

    @td.skip_if_no_scipy
    def test_interpolate_pchip(self):
        _skip_if_no_pchip()

        ser = Series(np.sort(np.random.uniform(size=100)))

        # interpolate at new_index
        new_index = ser.index.union(Index([49.25, 49.5, 49.75, 50.25, 50.5,
                                           50.75]))
        interp_s = ser.reindex(new_index).interpolate(method='pchip')
        # does not blow up, GH5977
        interp_s[49:51]

    @td.skip_if_no_scipy
    def test_interpolate_akima(self):
        _skip_if_no_akima()

        ser = Series([10, 11, 12, 13])

        expected = Series([11.00, 11.25, 11.50, 11.75,
                           12.00, 12.25, 12.50, 12.75, 13.00],
                          index=Index([1.0, 1.25, 1.5, 1.75,
                                       2.0, 2.25, 2.5, 2.75, 3.0]))
        # interpolate at new_index
        new_index = ser.index.union(Index([1.25, 1.5, 1.75, 2.25, 2.5, 2.75]))
        interp_s = ser.reindex(new_index).interpolate(method='akima')
        assert_series_equal(interp_s[1:3], expected)

    @td.skip_if_no_scipy
    def test_interpolate_piecewise_polynomial(self):
        ser = Series([10, 11, 12, 13])

        expected = Series([11.00, 11.25, 11.50, 11.75,
                           12.00, 12.25, 12.50, 12.75, 13.00],
                          index=Index([1.0, 1.25, 1.5, 1.75,
                                       2.0, 2.25, 2.5, 2.75, 3.0]))
        # interpolate at new_index
        new_index = ser.index.union(Index([1.25, 1.5, 1.75, 2.25, 2.5, 2.75]))
        interp_s = ser.reindex(new_index).interpolate(
            method='piecewise_polynomial')
        assert_series_equal(interp_s[1:3], expected)

    @td.skip_if_no_scipy
    def test_interpolate_from_derivatives(self):
        ser = Series([10, 11, 12, 13])

        expected = Series([11.00, 11.25, 11.50, 11.75,
                           12.00, 12.25, 12.50, 12.75, 13.00],
                          index=Index([1.0, 1.25, 1.5, 1.75,
                                       2.0, 2.25, 2.5, 2.75, 3.0]))
        # interpolate at new_index
        new_index = ser.index.union(Index([1.25, 1.5, 1.75, 2.25, 2.5, 2.75]))
        interp_s = ser.reindex(new_index).interpolate(
            method='from_derivatives')
        assert_series_equal(interp_s[1:3], expected)

    @pytest.mark.parametrize("kwargs", [
        {},
        pytest.param({'method': 'polynomial', 'order': 1},
                     marks=td.skip_if_no_scipy)
    ])
    def test_interpolate_corners(self, kwargs):
        s = Series([np.nan, np.nan])
        assert_series_equal(s.interpolate(**kwargs), s)

        s = Series([]).interpolate()
        assert_series_equal(s.interpolate(**kwargs), s)

    def test_interpolate_index_values(self):
        s = Series(np.nan, index=np.sort(np.random.rand(30)))
        s[::3] = np.random.randn(10)

        vals = s.index.values.astype(float)

        result = s.interpolate(method='index')

        expected = s.copy()
        bad = isna(expected.values)
        good = ~bad
        expected = Series(np.interp(vals[bad], vals[good],
                                    s.values[good]),
                          index=s.index[bad])

        assert_series_equal(result[bad], expected)

        # 'values' is synonymous with 'index' for the method kwarg
        other_result = s.interpolate(method='values')

        assert_series_equal(other_result, result)
        assert_series_equal(other_result[bad], expected)

    def test_interpolate_non_ts(self):
        s = Series([1, 3, np.nan, np.nan, np.nan, 11])
        with pytest.raises(ValueError):
            s.interpolate(method='time')

    @pytest.mark.parametrize("kwargs", [
        {},
        pytest.param({'method': 'polynomial', 'order': 1},
                     marks=td.skip_if_no_scipy)
    ])
    def test_nan_interpolate(self, kwargs):
        s = Series([0, 1, np.nan, 3])
        result = s.interpolate(**kwargs)
        expected = Series([0., 1., 2., 3.])
        assert_series_equal(result, expected)

    def test_nan_irregular_index(self):
        s = Series([1, 2, np.nan, 4], index=[1, 3, 5, 9])
        result = s.interpolate()
        expected = Series([1., 2., 3., 4.], index=[1, 3, 5, 9])
        assert_series_equal(result, expected)

    def test_nan_str_index(self):
        s = Series([0, 1, 2, np.nan], index=list('abcd'))
        result = s.interpolate()
        expected = Series([0., 1., 2., 2.], index=list('abcd'))
        assert_series_equal(result, expected)

    @td.skip_if_no_scipy
    def test_interp_quad(self):
        sq = Series([1, 4, np.nan, 16], index=[1, 2, 3, 4])
        result = sq.interpolate(method='quadratic')
        expected = Series([1., 4., 9., 16.], index=[1, 2, 3, 4])
        assert_series_equal(result, expected)

    @td.skip_if_no_scipy
    def test_interp_scipy_basic(self):
        s = Series([1, 3, np.nan, 12, np.nan, 25])
        # slinear
        expected = Series([1., 3., 7.5, 12., 18.5, 25.])
        result = s.interpolate(method='slinear')
        assert_series_equal(result, expected)

        result = s.interpolate(method='slinear', downcast='infer')
        assert_series_equal(result, expected)
        # nearest
        expected = Series([1, 3, 3, 12, 12, 25])
        result = s.interpolate(method='nearest')
        assert_series_equal(result, expected.astype('float'))

        result = s.interpolate(method='nearest', downcast='infer')
        assert_series_equal(result, expected)
        # zero
        expected = Series([1, 3, 3, 12, 12, 25])
        result = s.interpolate(method='zero')
        assert_series_equal(result, expected.astype('float'))

        result = s.interpolate(method='zero', downcast='infer')
        assert_series_equal(result, expected)
        # quadratic
        # GH #15662.
        # new cubic and quadratic interpolation algorithms from scipy 0.19.0.
        # previously `splmake` was used. See scipy/scipy#6710
        if _is_scipy_ge_0190:
            expected = Series([1, 3., 6.823529, 12., 18.058824, 25.])
        else:
            expected = Series([1, 3., 6.769231, 12., 18.230769, 25.])
        result = s.interpolate(method='quadratic')
        assert_series_equal(result, expected)

        result = s.interpolate(method='quadratic', downcast='infer')
        assert_series_equal(result, expected)
        # cubic
        expected = Series([1., 3., 6.8, 12., 18.2, 25.])
        result = s.interpolate(method='cubic')
        assert_series_equal(result, expected)

    def test_interp_limit(self):
        s = Series([1, 3, np.nan, np.nan, np.nan, 11])

        expected = Series([1., 3., 5., 7., np.nan, 11.])
        result = s.interpolate(method='linear', limit=2)
        assert_series_equal(result, expected)

        # GH 9217, make sure limit is an int and greater than 0
        methods = ['linear', 'time', 'index', 'values', 'nearest', 'zero',
                   'slinear', 'quadratic', 'cubic', 'barycentric', 'krogh',
                   'polynomial', 'spline', 'piecewise_polynomial', None,
                   'from_derivatives', 'pchip', 'akima']
        s = pd.Series([1, 2, np.nan, np.nan, 5])
        for limit in [-1, 0, 1., 2.]:
            for method in methods:
                with pytest.raises(ValueError):
                    s.interpolate(limit=limit, method=method)

    def test_interp_limit_forward(self):
        s = Series([1, 3, np.nan, np.nan, np.nan, 11])

        # Provide 'forward' (the default) explicitly here.
        expected = Series([1., 3., 5., 7., np.nan, 11.])

        result = s.interpolate(method='linear', limit=2,
                               limit_direction='forward')
        assert_series_equal(result, expected)

        result = s.interpolate(method='linear', limit=2,
                               limit_direction='FORWARD')
        assert_series_equal(result, expected)

    def test_interp_unlimited(self):
        # these test are for issue #16282 default Limit=None is unlimited
        s = Series([np.nan, 1., 3., np.nan, np.nan, np.nan, 11., np.nan])
        expected = Series([1., 1., 3., 5., 7., 9., 11., 11.])
        result = s.interpolate(method='linear',
                               limit_direction='both')
        assert_series_equal(result, expected)

        expected = Series([np.nan, 1., 3., 5., 7., 9., 11., 11.])
        result = s.interpolate(method='linear',
                               limit_direction='forward')
        assert_series_equal(result, expected)

        expected = Series([1., 1., 3., 5., 7., 9., 11., np.nan])
        result = s.interpolate(method='linear',
                               limit_direction='backward')
        assert_series_equal(result, expected)

    def test_interp_limit_bad_direction(self):
        s = Series([1, 3, np.nan, np.nan, np.nan, 11])

        pytest.raises(ValueError, s.interpolate, method='linear', limit=2,
                      limit_direction='abc')

        # raises an error even if no limit is specified.
        pytest.raises(ValueError, s.interpolate, method='linear',
                      limit_direction='abc')

    # limit_area introduced GH #16284
    def test_interp_limit_area(self):
        # These tests are for issue #9218 -- fill NaNs in both directions.
        s = Series([nan, nan, 3, nan, nan, nan, 7, nan, nan])

        expected = Series([nan, nan, 3., 4., 5., 6., 7., nan, nan])
        result = s.interpolate(method='linear', limit_area='inside')
        assert_series_equal(result, expected)

        expected = Series([nan, nan, 3., 4., nan, nan, 7., nan, nan])
        result = s.interpolate(method='linear', limit_area='inside',
                               limit=1)

        expected = Series([nan, nan, 3., 4., nan, 6., 7., nan, nan])
        result = s.interpolate(method='linear', limit_area='inside',
                               limit_direction='both', limit=1)
        assert_series_equal(result, expected)

        expected = Series([nan, nan, 3., nan, nan, nan, 7., 7., 7.])
        result = s.interpolate(method='linear', limit_area='outside')
        assert_series_equal(result, expected)

        expected = Series([nan, nan, 3., nan, nan, nan, 7., 7., nan])
        result = s.interpolate(method='linear', limit_area='outside',
                               limit=1)

        expected = Series([nan, 3., 3., nan, nan, nan, 7., 7., nan])
        result = s.interpolate(method='linear', limit_area='outside',
                               limit_direction='both', limit=1)
        assert_series_equal(result, expected)

        expected = Series([3., 3., 3., nan, nan, nan, 7., nan, nan])
        result = s.interpolate(method='linear', limit_area='outside',
                               direction='backward')

        # raises an error even if limit type is wrong.
        pytest.raises(ValueError, s.interpolate, method='linear',
                      limit_area='abc')

    def test_interp_limit_direction(self):
        # These tests are for issue #9218 -- fill NaNs in both directions.
        s = Series([1, 3, np.nan, np.nan, np.nan, 11])

        expected = Series([1., 3., np.nan, 7., 9., 11.])
        result = s.interpolate(method='linear', limit=2,
                               limit_direction='backward')
        assert_series_equal(result, expected)

        expected = Series([1., 3., 5., np.nan, 9., 11.])
        result = s.interpolate(method='linear', limit=1,
                               limit_direction='both')
        assert_series_equal(result, expected)

        # Check that this works on a longer series of nans.
        s = Series([1, 3, np.nan, np.nan, np.nan, 7, 9, np.nan, np.nan, 12,
                    np.nan])

        expected = Series([1., 3., 4., 5., 6., 7., 9., 10., 11., 12., 12.])
        result = s.interpolate(method='linear', limit=2,
                               limit_direction='both')
        assert_series_equal(result, expected)

        expected = Series([1., 3., 4., np.nan, 6., 7., 9., 10., 11., 12., 12.])
        result = s.interpolate(method='linear', limit=1,
                               limit_direction='both')
        assert_series_equal(result, expected)

    def test_interp_limit_to_ends(self):
        # These test are for issue #10420 -- flow back to beginning.
        s = Series([np.nan, np.nan, 5, 7, 9, np.nan])

        expected = Series([5., 5., 5., 7., 9., np.nan])
        result = s.interpolate(method='linear', limit=2,
                               limit_direction='backward')
        assert_series_equal(result, expected)

        expected = Series([5., 5., 5., 7., 9., 9.])
        result = s.interpolate(method='linear', limit=2,
                               limit_direction='both')
        assert_series_equal(result, expected)

    def test_interp_limit_before_ends(self):
        # These test are for issue #11115 -- limit ends properly.
        s = Series([np.nan, np.nan, 5, 7, np.nan, np.nan])

        expected = Series([np.nan, np.nan, 5., 7., 7., np.nan])
        result = s.interpolate(method='linear', limit=1,
                               limit_direction='forward')
        assert_series_equal(result, expected)

        expected = Series([np.nan, 5., 5., 7., np.nan, np.nan])
        result = s.interpolate(method='linear', limit=1,
                               limit_direction='backward')
        assert_series_equal(result, expected)

        expected = Series([np.nan, 5., 5., 7., 7., np.nan])
        result = s.interpolate(method='linear', limit=1,
                               limit_direction='both')
        assert_series_equal(result, expected)

    @td.skip_if_no_scipy
    def test_interp_all_good(self):
        s = Series([1, 2, 3])
        result = s.interpolate(method='polynomial', order=1)
        assert_series_equal(result, s)

        # non-scipy
        result = s.interpolate()
        assert_series_equal(result, s)

    @pytest.mark.parametrize("check_scipy", [
        False,
        pytest.param(True, marks=td.skip_if_no_scipy)
    ])
    def test_interp_multiIndex(self, check_scipy):
        idx = MultiIndex.from_tuples([(0, 'a'), (1, 'b'), (2, 'c')])
        s = Series([1, 2, np.nan], index=idx)

        expected = s.copy()
        expected.loc[2] = 2
        result = s.interpolate()
        assert_series_equal(result, expected)

        if check_scipy:
            with pytest.raises(ValueError):
                s.interpolate(method='polynomial', order=1)

    @td.skip_if_no_scipy
    def test_interp_nonmono_raise(self):
        s = Series([1, np.nan, 3], index=[0, 2, 1])
        with pytest.raises(ValueError):
            s.interpolate(method='krogh')

    @td.skip_if_no_scipy
    def test_interp_datetime64(self):
        df = Series([1, np.nan, 3], index=date_range('1/1/2000', periods=3))
        result = df.interpolate(method='nearest')
        expected = Series([1., 1., 3.],
                          index=date_range('1/1/2000', periods=3))
        assert_series_equal(result, expected)

    def test_interp_limit_no_nans(self):
        # GH 7173
        s = pd.Series([1., 2., 3.])
        result = s.interpolate(limit=1)
        expected = s
        assert_series_equal(result, expected)

    @td.skip_if_no_scipy
    @pytest.mark.parametrize("method", ['polynomial', 'spline'])
    def test_no_order(self, method):
        s = Series([0, 1, np.nan, 3])
        with pytest.raises(ValueError):
            s.interpolate(method=method)

    @td.skip_if_no_scipy
    def test_spline(self):
        s = Series([1, 2, np.nan, 4, 5, np.nan, 7])
        result = s.interpolate(method='spline', order=1)
        expected = Series([1., 2., 3., 4., 5., 6., 7.])
        assert_series_equal(result, expected)

    @td.skip_if_no('scipy', min_version='0.15')
    def test_spline_extrapolate(self):
        s = Series([1, 2, 3, 4, np.nan, 6, np.nan])
        result3 = s.interpolate(method='spline', order=1, ext=3)
        expected3 = Series([1., 2., 3., 4., 5., 6., 6.])
        assert_series_equal(result3, expected3)

        result1 = s.interpolate(method='spline', order=1, ext=0)
        expected1 = Series([1., 2., 3., 4., 5., 6., 7.])
        assert_series_equal(result1, expected1)

    @td.skip_if_no_scipy
    def test_spline_smooth(self):
        s = Series([1, 2, np.nan, 4, 5.1, np.nan, 7])
        assert (s.interpolate(method='spline', order=3, s=0)[5] !=
                s.interpolate(method='spline', order=3)[5])

    @td.skip_if_no_scipy
    def test_spline_interpolation(self):
        s = Series(np.arange(10) ** 2)
        s[np.random.randint(0, 9, 3)] = np.nan
        result1 = s.interpolate(method='spline', order=1)
        expected1 = s.interpolate(method='spline', order=1)
        assert_series_equal(result1, expected1)

    @td.skip_if_no_scipy
    def test_spline_error(self):
        # see gh-10633
        s = pd.Series(np.arange(10) ** 2)
        s[np.random.randint(0, 9, 3)] = np.nan
        with pytest.raises(ValueError):
            s.interpolate(method='spline')

        with pytest.raises(ValueError):
            s.interpolate(method='spline', order=0)

    def test_interp_timedelta64(self):
        # GH 6424
        df = Series([1, np.nan, 3],
                    index=pd.to_timedelta([1, 2, 3]))
        result = df.interpolate(method='time')
        expected = Series([1., 2., 3.],
                          index=pd.to_timedelta([1, 2, 3]))
        assert_series_equal(result, expected)

        # test for non uniform spacing
        df = Series([1, np.nan, 3],
                    index=pd.to_timedelta([1, 2, 4]))
        result = df.interpolate(method='time')
        expected = Series([1., 1.666667, 3.],
                          index=pd.to_timedelta([1, 2, 4]))
        assert_series_equal(result, expected)

    def test_series_interpolate_method_values(self):
        # #1646
        ts = _simple_ts('1/1/2000', '1/20/2000')
        ts[::2] = np.nan

        result = ts.interpolate(method='values')
        exp = ts.interpolate()
        assert_series_equal(result, exp)

    def test_series_interpolate_intraday(self):
        # #1698
        index = pd.date_range('1/1/2012', periods=4, freq='12D')
        ts = pd.Series([0, 12, 24, 36], index)
        new_index = index.append(index + pd.DateOffset(days=1)).sort_values()

        exp = ts.reindex(new_index).interpolate(method='time')

        index = pd.date_range('1/1/2012', periods=4, freq='12H')
        ts = pd.Series([0, 12, 24, 36], index)
        new_index = index.append(index + pd.DateOffset(hours=1)).sort_values()
        result = ts.reindex(new_index).interpolate(method='time')

        tm.assert_numpy_array_equal(result.values, exp.values)