"""This file contains code for use with "Think Stats",
by Allen B. Downey, available from greenteapress.com

Copyright 2014 Allen B. Downey
License: GNU GPLv3 http://www.gnu.org/licenses/gpl.html
"""

from __future__ import print_function

import pandas
import numpy as np
import statsmodels.formula.api as smf
import statsmodels.tsa.stattools as smtsa

import matplotlib.pyplot as pyplot

import thinkplot
import thinkstats2

FORMATS = ['png']

def ReadData():
    """Reads data about cannabis transactions.

    http://zmjones.com/static/data/mj-clean.csv

    returns: DataFrame
    """
    transactions = pandas.read_csv('mj-clean.csv', parse_dates=[5])
    return transactions

def tmean(series):
    """Computes a trimmed mean.

    series: Series

    returns: float
    """
    t = series.values
    n = len(t)
    if n <= 3:
        return t.mean()
    trim = max(1, n/10)
    return np.mean(sorted(t)[trim:n-trim])

def GroupByDay(transactions, func=np.mean):
    """Groups transactions by day and compute the daily mean ppg.

    transactions: DataFrame of transactions

    returns: DataFrame of daily prices
    """
    groups = transactions[['date', 'ppg']].groupby('date')
    daily = groups.aggregate(func)

    daily['date'] = daily.index
    start = daily.date[0]
    one_year = np.timedelta64(1, 'Y')
    daily['years'] = (daily.date - start) / one_year

    return daily

def GroupByQualityAndDay(transactions):
    """Divides transactions by quality and computes mean daily price.

    transaction: DataFrame of transactions

    returns: map from quality to time series of ppg
    """
    groups = transactions.groupby('quality')
    dailies = {}
    for name, group in groups:
        dailies[name] = GroupByDay(group)

    return dailies

def PlotDailies(dailies):
    """Makes a plot with daily prices for different qualities.

    dailies: map from name to DataFrame
    """
    thinkplot.PrePlot(rows=3)
    for i, (name, daily) in enumerate(dailies.items()):
        thinkplot.SubPlot(i+1)
        title = 'price per gram ($)' if i == 0 else ''
        thinkplot.Config(ylim=[0, 20], title=title)
        thinkplot.Scatter(daily.ppg, s=10, label=name)
        if i == 2:
            pyplot.xticks(rotation=30)
        else:
            thinkplot.Config(xticks=[])

    thinkplot.Save(root='timeseries1',
                   formats=FORMATS)

def RunLinearModel(daily):
    """Runs a linear model of prices versus years.

    daily: DataFrame of daily prices

    returns: model, results
    """
    model = smf.ols('ppg ~ years', data=daily)
    results = model.fit()
    return model, results

def PlotFittedValues(model, results, label=''):
    """Plots original data and fitted values.

    model: StatsModel model object
    results: StatsModel results object
    """
    years = model.exog[:, 1]
    values = model.endog
    thinkplot.Scatter(years, values, s=15, label=label)
    thinkplot.Plot(years, results.fittedvalues, label='model')

def PlotResiduals(model, results):
    """Plots the residuals of a model.

    model: StatsModel model object
    results: StatsModel results object
    """
    years = model.exog[:, 1]
    thinkplot.Plot(years, results.resid, linewidth=0.5, alpha=0.5)

def PlotResidualPercentiles(model, results, index=1, num_bins=20):
    """Plots percentiles of the residuals.

    model: StatsModel model object
    results: StatsModel results object
    index: which exogenous variable to use
    num_bins: how many bins to divide the x-axis into
    """
    exog = model.exog[:, index]
    resid = results.resid.values
    df = pandas.DataFrame(dict(exog=exog, resid=resid))

    bins = np.linspace(np.min(exog), np.max(exog), num_bins)
    indices = np.digitize(exog, bins)
    groups = df.groupby(indices)

    means = [group.exog.mean() for _, group in groups][1:-1]
    cdfs = [thinkstats2.Cdf(group.resid) for _, group in groups][1:-1]

    thinkplot.PrePlot(3)
    for percent in [75, 50, 25]:
        percentiles = [cdf.Percentile(percent) for cdf in cdfs]
        label = '%dth' % percent
        thinkplot.Plot(means, percentiles, label=label)

def SimulateResults(daily, iters=101, func=RunLinearModel):
    """Run simulations based on resampling residuals.

    daily: DataFrame of daily prices
    iters: number of simulations
    func: function that fits a model to the data

    returns: list of result objects
    """
    _, results = func(daily)
    fake = daily.copy()

    result_seq = []
    for _ in range(iters):
        fake.ppg = results.fittedvalues + thinkstats2.Resample(results.resid)
        _, fake_results = func(fake)
        result_seq.append(fake_results)

    return result_seq

def SimulateIntervals(daily, iters=101, func=RunLinearModel):
    """Run simulations based on different subsets of the data.

    daily: DataFrame of daily prices
    iters: number of simulations
    func: function that fits a model to the data

    returns: list of result objects
    """
    result_seq = []
    starts = np.linspace(0, len(daily), iters).astype(int)

    for start in starts[:-2]:
        subset = daily[start:]
        _, results = func(subset)
        fake = subset.copy()

        for _ in range(iters):
            fake.ppg = (results.fittedvalues +
                        thinkstats2.Resample(results.resid))
            _, fake_results = func(fake)
            result_seq.append(fake_results)

    return result_seq

def GeneratePredictions(result_seq, years, add_resid=False):
    """Generates an array of predicted values from a list of model results.

    When add_resid is False, predictions represent sampling error only.

    When add_resid is True, they also include residual error (which is
    more relevant to prediction).

    result_seq: list of model results
    years: sequence of times (in years) to make predictions for
    add_resid: boolean, whether to add in resampled residuals

    returns: sequence of predictions
    """
    n = len(years)
    d = dict(Intercept=np.ones(n), years=years, years2=years**2)
    predict_df = pandas.DataFrame(d)

    predict_seq = []
    for fake_results in result_seq:
        predict = fake_results.predict(predict_df)
        if add_resid:
            predict += thinkstats2.Resample(fake_results.resid, n)
        predict_seq.append(predict)

    return predict_seq

def GenerateSimplePrediction(results, years):
    """Generates a simple prediction.

    results: results object
    years: sequence of times (in years) to make predictions for

    returns: sequence of predicted values
    """
    n = len(years)
    inter = np.ones(n)
    d = dict(Intercept=inter, years=years, years2=years**2)
    predict_df = pandas.DataFrame(d)
    predict = results.predict(predict_df)
    return predict

def PlotPredictions(daily, years, iters=101, percent=90, func=RunLinearModel):
    """Plots predictions.

    daily: DataFrame of daily prices
    years: sequence of times (in years) to make predictions for
    iters: number of simulations
    percent: what percentile range to show
    func: function that fits a model to the data
    """
    result_seq = SimulateResults(daily, iters=iters, func=func)
    p = (100 - percent) / 2
    percents = p, 100-p

    predict_seq = GeneratePredictions(result_seq, years, add_resid=True)
    low, high = thinkstats2.PercentileRows(predict_seq, percents)
    thinkplot.FillBetween(years, low, high, alpha=0.3, color='gray')

    predict_seq = GeneratePredictions(result_seq, years, add_resid=False)
    low, high = thinkstats2.PercentileRows(predict_seq, percents)
    thinkplot.FillBetween(years, low, high, alpha=0.5, color='gray')

def PlotIntervals(daily, years, iters=101, percent=90, func=RunLinearModel):
    """Plots predictions based on different intervals.

    daily: DataFrame of daily prices
    years: sequence of times (in years) to make predictions for
    iters: number of simulations
    percent: what percentile range to show
    func: function that fits a model to the data
    """
    result_seq = SimulateIntervals(daily, iters=iters, func=func)
    p = (100 - percent) / 2
    percents = p, 100-p

    predict_seq = GeneratePredictions(result_seq, years, add_resid=True)
    low, high = thinkstats2.PercentileRows(predict_seq, percents)
    thinkplot.FillBetween(years, low, high, alpha=0.2, color='gray')

def Correlate(dailies):
    """Compute the correlation matrix between prices for difference qualities.

    dailies: map from quality to time series of ppg

    returns: correlation matrix
    """
    df = pandas.DataFrame()
    for name, daily in dailies.items():
        df[name] = daily.ppg

    return df.corr()

def CorrelateResid(dailies):
    """Compute the correlation matrix between residuals.

    dailies: map from quality to time series of ppg

    returns: correlation matrix
    """
    df = pandas.DataFrame()
    for name, daily in dailies.items():
        _, results = RunLinearModel(daily)
        df[name] = results.resid

    return df.corr()

def TestCorrelateResid(dailies, iters=101):
    """Tests observed correlations.

    dailies: map from quality to time series of ppg
    iters: number of simulations
    """

    t = []
    names = ['high', 'medium', 'low']
    for name in names:
        daily = dailies[name]
        t.append(SimulateResults(daily, iters=iters))

    corr = CorrelateResid(dailies)

    arrays = []
    for result_seq in zip(*t):
        df = pandas.DataFrame()
        for name, results in zip(names, result_seq):
            df[name] = results.resid

        opp_sign = corr * df.corr() < 0
        arrays.append((opp_sign.astype(int)))

    print(np.sum(arrays))

def RunModels(dailies):
    """Runs linear regression for each group in dailies.

    dailies: map from group name to DataFrame
    """
    rows = []
    for daily in dailies.values():
        _, results = RunLinearModel(daily)
        intercept, slope = results.params
        p1, p2 = results.pvalues
        r2 = results.rsquared
        s = r'%0.3f (%0.2g) & %0.3f (%0.2g) & %0.3f \\'
        row = s % (intercept, p1, slope, p2, r2)
        rows.append(row)

    # print results in a LaTeX table
    print(r'\begin{tabular}{|c|c|c|}')
    print(r'\hline')
    print(r'intercept & slope & $R^2$ \\ \hline')
    for row in rows:
        print(row)
    print(r'\hline')
    print(r'\end{tabular}')

def FillMissing(daily, span=30):
    """Fills missing values with an exponentially weighted moving average.

    Resulting DataFrame has new columns 'ewma' and 'resid'.

    daily: DataFrame of daily prices
    span: window size (sort of) passed to ewma

    returns: new DataFrame of daily prices
    """
    dates = pandas.date_range(daily.index.min(), daily.index.max())
    reindexed = daily.reindex(dates)

    ewma = pandas.ewma(reindexed.ppg, span=span)

    resid = (reindexed.ppg - ewma).dropna()
    fake_data = ewma + thinkstats2.Resample(resid, len(reindexed))
    reindexed.ppg.fillna(fake_data, inplace=True)

    reindexed['ewma'] = ewma
    reindexed['resid'] = reindexed.ppg - ewma
    return reindexed

def AddWeeklySeasonality(daily):
    """Adds a weekly pattern.

    daily: DataFrame of daily prices

    returns: new DataFrame of daily prices
    """
    frisat = (daily.index.dayofweek==4) | (daily.index.dayofweek==5)
    fake = daily.copy()
    fake.ppg[frisat] += np.random.uniform(0, 2, frisat.sum())
    return fake

def PrintSerialCorrelations(dailies):
    """Prints a table of correlations with different lags.

    dailies: map from category name to DataFrame of daily prices
    """
    filled_dailies = {}
    for name, daily in dailies.items():
        filled_dailies[name] = FillMissing(daily, span=30)

    # print serial correlations for raw price data
    for name, filled in filled_dailies.items():
        corr = thinkstats2.SerialCorr(filled.ppg, lag=1)
        print(name, corr)

    rows = []
    for lag in [1, 7, 30, 365]:
        row = [str(lag)]
        for name, filled in filled_dailies.items():
            corr = thinkstats2.SerialCorr(filled.resid, lag)
            row.append('%.2g' % corr)
        rows.append(row)

    print(r'\begin{tabular}{|c|c|c|c|}')
    print(r'\hline')
    print(r'lag & high & medium & low \\ \hline')
    for row in rows:
        print(' & '.join(row) + r' \\')
    print(r'\hline')
    print(r'\end{tabular}')

    filled = filled_dailies['high']
    acf = smtsa.acf(filled.resid, nlags=365, unbiased=True)
    print('%0.3f, %0.3f, %0.3f, %0.3f, %0.3f' %
          (acf[0], acf[1], acf[7], acf[30], acf[365]))

def SimulateAutocorrelation(daily, iters=1001, nlags=40):
    """Resample residuals, compute autocorrelation, and plot percentiles.

    daily: DataFrame
    iters: number of simulations to run
    nlags: maximum lags to compute autocorrelation
    """
    # run simulations
    t = []
    for _ in range(iters):
        filled = FillMissing(daily, span=30)
        resid = thinkstats2.Resample(filled.resid)
        acf = smtsa.acf(resid, nlags=nlags, unbiased=True)[1:]
        t.append(np.abs(acf))

    high = thinkstats2.PercentileRows(t, [97.5])[0]
    low = -high
    lags = range(1, nlags+1)
    thinkplot.FillBetween(lags, low, high, alpha=0.2, color='gray')

def PlotAutoCorrelation(dailies, nlags=40, add_weekly=False):
    """Plots autocorrelation functions.

    dailies: map from category name to DataFrame of daily prices
    nlags: number of lags to compute
    add_weekly: boolean, whether to add a simulated weekly pattern
    """
    thinkplot.PrePlot(3)
    daily = dailies['high']
    SimulateAutocorrelation(daily)

    for name, daily in dailies.items():

        if add_weekly:
            daily = AddWeeklySeasonality(daily)

        filled = FillMissing(daily, span=30)

        acf = smtsa.acf(filled.resid, nlags=nlags, unbiased=True)
        lags = np.arange(len(acf))
        thinkplot.Plot(lags[1:], acf[1:], label=name)

def MakeAcfPlot(dailies):
    """Makes a figure showing autocorrelation functions.

    dailies: map from category name to DataFrame of daily prices
    """
    axis = [0, 41, -0.2, 0.2]

    thinkplot.PrePlot(cols=2)
    PlotAutoCorrelation(dailies, add_weekly=False)
    thinkplot.Config(axis=axis,
                     loc='lower right',
                     ylabel='correlation',
                     xlabel='lag (day)')

    thinkplot.SubPlot(2)
    PlotAutoCorrelation(dailies, add_weekly=True)
    thinkplot.Save(root='timeseries9',
                   axis=axis,
                   loc='lower right',
                   xlabel='lag (days)',
                   formats=FORMATS)

def PlotRollingMean(daily, name):
    """Plots rolling mean and EWMA.

    daily: DataFrame of daily prices
    """
    dates = pandas.date_range(daily.index.min(), daily.index.max())
    reindexed = daily.reindex(dates)

    thinkplot.PrePlot(cols=2)
    thinkplot.Scatter(reindexed.ppg, s=15, alpha=0.1, label=name)
    roll_mean = pandas.rolling_mean(reindexed.ppg, 30)
    thinkplot.Plot(roll_mean, label='rolling mean')
    pyplot.xticks(rotation=30)
    thinkplot.Config(ylabel='price per gram ($)')

    thinkplot.SubPlot(2)
    thinkplot.Scatter(reindexed.ppg, s=15, alpha=0.1, label=name)
    ewma = pandas.ewma(reindexed.ppg, span=30)
    thinkplot.Plot(ewma, label='EWMA')
    pyplot.xticks(rotation=30)
    thinkplot.Save(root='timeseries10',
                   formats=FORMATS)

def PlotFilled(daily, name):
    """Plots the EWMA and filled data.

    daily: DataFrame of daily prices
    """
    filled = FillMissing(daily, span=30)
    thinkplot.Scatter(filled.ppg, s=15, alpha=0.3, label=name)
    thinkplot.Plot(filled.ewma, label='EWMA', alpha=0.4)
    pyplot.xticks(rotation=30)
    thinkplot.Save(root='timeseries8',
                   ylabel='price per gram ($)',
                   formats=FORMATS)

def PlotLinearModel(daily, name):
    """Plots a linear fit to a sequence of prices, and the residuals.

    daily: DataFrame of daily prices
    name: string
    """
    model, results = RunLinearModel(daily)
    PlotFittedValues(model, results, label=name)
    thinkplot.Save(root='timeseries2',
                   title='fitted values',
                   xlabel='years',
                   xlim=[-0.1, 3.8],
                   ylabel='price per gram ($)',
                   formats=FORMATS)

    PlotResidualPercentiles(model, results)
    thinkplot.Save(root='timeseries3',
                   title='residuals',
                   xlabel='years',
                   ylabel='price per gram ($)',
                   formats=FORMATS)

    #years = np.linspace(0, 5, 101)
    #predict = GenerateSimplePrediction(results, years)

def main(name):
    thinkstats2.RandomSeed(18)
    transactions = ReadData()

    dailies = GroupByQualityAndDay(transactions)
    PlotDailies(dailies)
    RunModels(dailies)
    PrintSerialCorrelations(dailies)
    MakeAcfPlot(dailies)

    name = 'high'
    daily = dailies[name]

    PlotLinearModel(daily, name)
    PlotRollingMean(daily, name)
    PlotFilled(daily, name)

    years = np.linspace(0, 5, 101)
    thinkplot.Scatter(daily.years, daily.ppg, alpha=0.1, label=name)
    PlotPredictions(daily, years)
    xlim = years[0]-0.1, years[-1]+0.1
    thinkplot.Save(root='timeseries4',
                   title='predictions',
                   xlabel='years',
                   xlim=xlim,
                   ylabel='price per gram ($)',
                   formats=FORMATS)

    name = 'medium'
    daily = dailies[name]

    thinkplot.Scatter(daily.years, daily.ppg, alpha=0.1, label=name)
    PlotIntervals(daily, years)
    PlotPredictions(daily, years)
    xlim = years[0]-0.1, years[-1]+0.1
    thinkplot.Save(root='timeseries5',
                   title='predictions',
                   xlabel='years',
                   xlim=xlim,
                   ylabel='price per gram ($)',
                   formats=FORMATS)

if __name__ == '__main__':
    import sys
    main(*sys.argv)


"""
==========================
Model Complexity Influence
==========================

Demonstrate how model complexity influences both prediction accuracy and
computational performance.

The dataset is the Boston Housing dataset (resp. 20 Newsgroups) for
regression (resp. classification).

For each class of models we make the model complexity vary through the choice
of relevant model parameters and measure the influence on both computational
performance (latency) and predictive power (MSE or Hamming Loss).
"""

print(__doc__)

# Author: Eustache Diemert <eustache@diemert.fr>
# License: BSD 3 clause

import time
import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits.axes_grid1.parasite_axes import host_subplot
from mpl_toolkits.axisartist.axislines import Axes
from scipy.sparse.csr import csr_matrix

from sklearn import datasets
from sklearn.utils import shuffle
from sklearn.metrics import mean_squared_error
from sklearn.svm.classes import NuSVR
from sklearn.ensemble.gradient_boosting import GradientBoostingRegressor
from sklearn.linear_model.stochastic_gradient import SGDClassifier
from sklearn.metrics import hamming_loss

###############################################################################
# Routines

# initialize random generator
np.random.seed(0)

def generate_data(case, sparse=False):
    """Generate regression/classification data."""
    bunch = None
    if case == 'regression':
        bunch = datasets.load_boston()
    elif case == 'classification':
        bunch = datasets.fetch_20newsgroups_vectorized(subset='all')
    X, y = shuffle(bunch.data, bunch.target)
    offset = int(X.shape[0] * 0.8)
    X_train, y_train = X[:offset], y[:offset]
    X_test, y_test = X[offset:], y[offset:]
    if sparse:
        X_train = csr_matrix(X_train)
        X_test = csr_matrix(X_test)
    else:
        X_train = np.array(X_train)
        X_test = np.array(X_test)
    y_test = np.array(y_test)
    y_train = np.array(y_train)
    data = {'X_train': X_train, 'X_test': X_test, 'y_train': y_train,
            'y_test': y_test}
    return data

def benchmark_influence(conf):
    """
    Benchmark influence of :changing_param: on both MSE and latency.
    """
    prediction_times = []
    prediction_powers = []
    complexities = []
    for param_value in conf['changing_param_values']:
        conf['tuned_params'][conf['changing_param']] = param_value
        estimator = conf['estimator'](**conf['tuned_params'])
        print("Benchmarking %s" % estimator)
        estimator.fit(conf['data']['X_train'], conf['data']['y_train'])
        conf['postfit_hook'](estimator)
        complexity = conf['complexity_computer'](estimator)
        complexities.append(complexity)
        start_time = time.time()
        for _ in range(conf['n_samples']):
            y_pred = estimator.predict(conf['data']['X_test'])
        elapsed_time = (time.time() - start_time) / float(conf['n_samples'])
        prediction_times.append(elapsed_time)
        pred_score = conf['prediction_performance_computer'](
            conf['data']['y_test'], y_pred)
        prediction_powers.append(pred_score)
        print("Complexity: %d | %s: %.4f | Pred. Time: %fs\n" % (
            complexity, conf['prediction_performance_label'], pred_score,
            elapsed_time))
    return prediction_powers, prediction_times, complexities

def plot_influence(conf, mse_values, prediction_times, complexities):
    """
    Plot influence of model complexity on both accuracy and latency.
    """
    plt.figure(figsize=(12, 6))
    host = host_subplot(111, axes_class=Axes)
    plt.subplots_adjust(right=0.75)
    par1 = host.twinx()
    host.set_xlabel('Model Complexity (%s)' % conf['complexity_label'])
    y1_label = conf['prediction_performance_label']
    y2_label = "Time (s)"
    host.set_ylabel(y1_label)
    par1.set_ylabel(y2_label)
    p1, = host.plot(complexities, mse_values, 'b-', label="prediction error")
    p2, = par1.plot(complexities, prediction_times, 'r-',
                    label="latency")
    host.legend(loc='upper right')
    host.axis["left"].label.set_color(p1.get_color())
    par1.axis["right"].label.set_color(p2.get_color())
    plt.title('Influence of Model Complexity - %s' % conf['estimator'].__name__)
    plt.show()

def _count_nonzero_coefficients(estimator):
    a = estimator.coef_.toarray()
    return np.count_nonzero(a)

###############################################################################
# main code
regression_data = generate_data('regression')
classification_data = generate_data('classification', sparse=True)
configurations = [
    {'estimator': SGDClassifier,
     'tuned_params': {'penalty': 'elasticnet', 'alpha': 0.001, 'loss':
                      'modified_huber', 'fit_intercept': True},
     'changing_param': 'l1_ratio',
     'changing_param_values': [0.25, 0.5, 0.75, 0.9],
     'complexity_label': 'non_zero coefficients',
     'complexity_computer': _count_nonzero_coefficients,
     'prediction_performance_computer': hamming_loss,
     'prediction_performance_label': 'Hamming Loss (Misclassification Ratio)',
     'postfit_hook': lambda x: x.sparsify(),
     'data': classification_data,
     'n_samples': 30},
    {'estimator': NuSVR,
     'tuned_params': {'C': 1e3, 'gamma': 2 ** -15},
     'changing_param': 'nu',
     'changing_param_values': [0.1, 0.25, 0.5, 0.75, 0.9],
     'complexity_label': 'n_support_vectors',
     'complexity_computer': lambda x: len(x.support_vectors_),
     'data': regression_data,
     'postfit_hook': lambda x: x,
     'prediction_performance_computer': mean_squared_error,
     'prediction_performance_label': 'MSE',
     'n_samples': 30},
    {'estimator': GradientBoostingRegressor,
     'tuned_params': {'loss': 'ls'},
     'changing_param': 'n_estimators',
     'changing_param_values': [10, 50, 100, 200, 500],
     'complexity_label': 'n_trees',
     'complexity_computer': lambda x: x.n_estimators,
     'data': regression_data,
     'postfit_hook': lambda x: x,
     'prediction_performance_computer': mean_squared_error,
     'prediction_performance_label': 'MSE',
     'n_samples': 30},
]
for conf in configurations:
    prediction_performances, prediction_times, complexities = \
        benchmark_influence(conf)
    plot_influence(conf, prediction_performances, prediction_times,
                   complexities)


# -*- coding: utf-8 -*-
# Licensed under a 3-clause BSD style license - see LICENSE.rst

import gc
import sys
import copy
from io import StringIO
from collections import OrderedDict

import pytest
import numpy as np
from numpy.testing import assert_allclose

from ...io import fits
from ...tests.helper import (assert_follows_unicode_guidelines,
                             ignore_warnings, catch_warnings)
from ...utils.data import get_pkg_data_filename
from ... import table
from ... import units as u
from .conftest import MaskedTable

try:
    with ignore_warnings(DeprecationWarning):
        # Ignore DeprecationWarning on pandas import in Python 3.5--see
        # https://github.com/astropy/astropy/issues/4380
        import pandas  # pylint: disable=W0611
except ImportError:
    HAS_PANDAS = False
else:
    HAS_PANDAS = True

class SetupData:
    def _setup(self, table_types):
        self._table_type = table_types.Table
        self._column_type = table_types.Column

    @property
    def a(self):
        if self._column_type is not None:
            if not hasattr(self, '_a'):
                self._a = self._column_type(
                    [1, 2, 3], name='a', format='%d',
                    meta={'aa': [0, 1, 2, 3, 4]})
            return self._a

    @property
    def b(self):
        if self._column_type is not None:
            if not hasattr(self, '_b'):
                self._b = self._column_type(
                    [4, 5, 6], name='b', format='%d', meta={'aa': 1})
            return self._b

    @property
    def c(self):
        if self._column_type is not None:
            if not hasattr(self, '_c'):
                self._c = self._column_type([7, 8, 9], 'c')
            return self._c

    @property
    def d(self):
        if self._column_type is not None:
            if not hasattr(self, '_d'):
                self._d = self._column_type([7, 8, 7], 'd')
            return self._d

    @property
    def obj(self):
        if self._column_type is not None:
            if not hasattr(self, '_obj'):
                self._obj = self._column_type([1, 'string', 3], 'obj', dtype='O')
            return self._obj

    @property
    def t(self):
        if self._table_type is not None:
            if not hasattr(self, '_t'):
                self._t = self._table_type([self.a, self.b])
            return self._t

@pytest.mark.usefixtures('table_types')
class TestSetTableColumn(SetupData):

    def test_set_row(self, table_types):
        """Set a row from a tuple of values"""
        self._setup(table_types)
        t = table_types.Table([self.a, self.b])
        t[1] = (20, 21)
        assert t['a'][0] == 1
        assert t['a'][1] == 20
        assert t['a'][2] == 3
        assert t['b'][0] == 4
        assert t['b'][1] == 21
        assert t['b'][2] == 6

    def test_set_row_existing(self, table_types):
        """Set a row from another existing row"""
        self._setup(table_types)
        t = table_types.Table([self.a, self.b])
        t[0] = t[1]
        assert t[0][0] == 2
        assert t[0][1] == 5

    def test_set_row_fail_1(self, table_types):
        """Set a row from an incorrectly-sized or typed set of values"""
        self._setup(table_types)
        t = table_types.Table([self.a, self.b])
        with pytest.raises(ValueError):
            t[1] = (20, 21, 22)
        with pytest.raises(ValueError):
            t[1] = 0

    def test_set_row_fail_2(self, table_types):
        """Set a row from an incorrectly-typed tuple of values"""
        self._setup(table_types)
        t = table_types.Table([self.a, self.b])
        with pytest.raises(ValueError):
            t[1] = ('abc', 'def')

    def test_set_new_col_new_table(self, table_types):
        """Create a new column in empty table using the item access syntax"""
        self._setup(table_types)
        t = table_types.Table()
        t['aa'] = self.a
        # Test that the new column name is 'aa' and that the values match
        assert np.all(t['aa'] == self.a)
        assert t.colnames == ['aa']

    def test_set_new_col_new_table_quantity(self, table_types):
        """Create a new column (from a quantity) in empty table using the item access syntax"""
        self._setup(table_types)
        t = table_types.Table()

        t['aa'] = np.array([1, 2, 3]) * u.m
        assert np.all(t['aa'] == np.array([1, 2, 3]))
        assert t['aa'].unit == u.m

        t['bb'] = 3 * u.m
        assert np.all(t['bb'] == 3)
        assert t['bb'].unit == u.m

    def test_set_new_col_existing_table(self, table_types):
        """Create a new column in an existing table using the item access syntax"""
        self._setup(table_types)
        t = table_types.Table([self.a])

        # Add a column
        t['bb'] = self.b
        assert np.all(t['bb'] == self.b)
        assert t.colnames == ['a', 'bb']
        assert t['bb'].meta == self.b.meta
        assert t['bb'].format == self.b.format

        # Add another column
        t['c'] = t['a']
        assert np.all(t['c'] == t['a'])
        assert t.colnames == ['a', 'bb', 'c']
        assert t['c'].meta == t['a'].meta
        assert t['c'].format == t['a'].format

        # Add a multi-dimensional column
        t['d'] = table_types.Column(np.arange(12).reshape(3, 2, 2))
        assert t['d'].shape == (3, 2, 2)
        assert t['d'][0, 0, 1] == 1

        # Add column from a list
        t['e'] = ['hello', 'the', 'world']
        assert np.all(t['e'] == np.array(['hello', 'the', 'world']))

        # Make sure setting existing column still works
        t['e'] = ['world', 'hello', 'the']
        assert np.all(t['e'] == np.array(['world', 'hello', 'the']))

        # Add a column via broadcasting
        t['f'] = 10
        assert np.all(t['f'] == 10)

        # Add a column from a Quantity
        t['g'] = np.array([1, 2, 3]) * u.m
        assert np.all(t['g'].data == np.array([1, 2, 3]))
        assert t['g'].unit == u.m

        # Add a column from a (scalar) Quantity
        t['g'] = 3 * u.m
        assert np.all(t['g'].data == 3)
        assert t['g'].unit == u.m

    def test_set_new_unmasked_col_existing_table(self, table_types):
        """Create a new column in an existing table using the item access syntax"""
        self._setup(table_types)
        t = table_types.Table([self.a])  # masked or unmasked
        b = table.Column(name='b', data=[1, 2, 3])  # unmasked
        t['b'] = b
        assert np.all(t['b'] == b)

    def test_set_new_masked_col_existing_table(self, table_types):
        """Create a new column in an existing table using the item access syntax"""
        self._setup(table_types)
        t = table_types.Table([self.a])  # masked or unmasked
        b = table.MaskedColumn(name='b', data=[1, 2, 3])  # masked
        t['b'] = b
        assert np.all(t['b'] == b)

    def test_set_new_col_existing_table_fail(self, table_types):
        """Generate failure when creating a new column using the item access syntax"""
        self._setup(table_types)
        t = table_types.Table([self.a])
        # Wrong size
        with pytest.raises(ValueError):
            t['b'] = [1, 2]

@pytest.mark.usefixtures('table_types')
class TestEmptyData():

    def test_1(self, table_types):
        t = table_types.Table()
        t.add_column(table_types.Column(name='a', dtype=int, length=100))
        assert len(t['a']) == 100

    def test_2(self, table_types):
        t = table_types.Table()
        t.add_column(table_types.Column(name='a', dtype=int, shape=(3, ), length=100))
        assert len(t['a']) == 100

    def test_3(self, table_types):
        t = table_types.Table()  # length is not given
        t.add_column(table_types.Column(name='a', dtype=int))
        assert len(t['a']) == 0

    def test_4(self, table_types):
        t = table_types.Table()  # length is not given
        t.add_column(table_types.Column(name='a', dtype=int, shape=(3, 4)))
        assert len(t['a']) == 0

    def test_5(self, table_types):
        t = table_types.Table()
        t.add_column(table_types.Column(name='a'))  # dtype is not specified
        assert len(t['a']) == 0

    def test_add_via_setitem_and_slice(self, table_types):
        """Test related to #3023 where a MaskedColumn is created with name=None
        and then gets changed to name='a'.  After PR #2790 this test fails
        without the #3023 fix."""
        t = table_types.Table()
        t['a'] = table_types.Column([1, 2, 3])
        t2 = t[:]
        assert t2.colnames == t.colnames

@pytest.mark.usefixtures('table_types')
class TestNewFromColumns():

    def test_simple(self, table_types):
        cols = [table_types.Column(name='a', data=[1, 2, 3]),
                table_types.Column(name='b', data=[4, 5, 6], dtype=np.float32)]
        t = table_types.Table(cols)
        assert np.all(t['a'].data == np.array([1, 2, 3]))
        assert np.all(t['b'].data == np.array([4, 5, 6], dtype=np.float32))
        assert type(t['b'][1]) is np.float32

    def test_from_np_array(self, table_types):
        cols = [table_types.Column(name='a', data=np.array([1, 2, 3], dtype=np.int64),
                       dtype=np.float64),
                table_types.Column(name='b', data=np.array([4, 5, 6], dtype=np.float32))]
        t = table_types.Table(cols)
        assert np.all(t['a'] == np.array([1, 2, 3], dtype=np.float64))
        assert np.all(t['b'] == np.array([4, 5, 6], dtype=np.float32))
        assert type(t['a'][1]) is np.float64
        assert type(t['b'][1]) is np.float32

    def test_size_mismatch(self, table_types):
        cols = [table_types.Column(name='a', data=[1, 2, 3]),
                table_types.Column(name='b', data=[4, 5, 6, 7])]
        with pytest.raises(ValueError):
            table_types.Table(cols)

    def test_name_none(self, table_types):
        """Column with name=None can init a table whether or not names are supplied"""
        c = table_types.Column(data=[1, 2], name='c')
        d = table_types.Column(data=[3, 4])
        t = table_types.Table([c, d], names=(None, 'd'))
        assert t.colnames == ['c', 'd']
        t = table_types.Table([c, d])
        assert t.colnames == ['c', 'col1']

@pytest.mark.usefixtures('table_types')
class TestReverse():

    def test_reverse(self, table_types):
        t = table_types.Table([[1, 2, 3],
                   ['a', 'b', 'cc']])
        t.reverse()
        assert np.all(t['col0'] == np.array([3, 2, 1]))
        assert np.all(t['col1'] == np.array(['cc', 'b', 'a']))

        t2 = table_types.Table(t, copy=False)
        assert np.all(t2['col0'] == np.array([3, 2, 1]))
        assert np.all(t2['col1'] == np.array(['cc', 'b', 'a']))

        t2 = table_types.Table(t, copy=True)
        assert np.all(t2['col0'] == np.array([3, 2, 1]))
        assert np.all(t2['col1'] == np.array(['cc', 'b', 'a']))

        t2.sort('col0')
        assert np.all(t2['col0'] == np.array([1, 2, 3]))
        assert np.all(t2['col1'] == np.array(['a', 'b', 'cc']))

    def test_reverse_big(self, table_types):
        x = np.arange(10000)
        y = x + 1
        t = table_types.Table([x, y], names=('x', 'y'))
        t.reverse()
        assert np.all(t['x'] == x[::-1])
        assert np.all(t['y'] == y[::-1])

@pytest.mark.usefixtures('table_types')
class TestColumnAccess():

    def test_1(self, table_types):
        t = table_types.Table()
        with pytest.raises(KeyError):
            t['a']

    def test_2(self, table_types):
        t = table_types.Table()
        t.add_column(table_types.Column(name='a', data=[1, 2, 3]))
        assert np.all(t['a'] == np.array([1, 2, 3]))
        with pytest.raises(KeyError):
            t['b']  # column does not exist

    def test_itercols(self, table_types):
        names = ['a', 'b', 'c']
        t = table_types.Table([[1], [2], [3]], names=names)
        for name, col in zip(names, t.itercols()):
            assert name == col.name
            assert isinstance(col, table_types.Column)

@pytest.mark.usefixtures('table_types')
class TestAddLength(SetupData):

    def test_right_length(self, table_types):
        self._setup(table_types)
        t = table_types.Table([self.a])
        t.add_column(self.b)

    def test_too_long(self, table_types):
        self._setup(table_types)
        t = table_types.Table([self.a])
        with pytest.raises(ValueError):
            t.add_column(table_types.Column(name='b', data=[4, 5, 6, 7]))  # data too long

    def test_too_short(self, table_types):
        self._setup(table_types)
        t = table_types.Table([self.a])
        with pytest.raises(ValueError):
            t.add_column(table_types.Column(name='b', data=[4, 5]))  # data too short

@pytest.mark.usefixtures('table_types')
class TestAddPosition(SetupData):

    def test_1(self, table_types):
        self._setup(table_types)
        t = table_types.Table()
        t.add_column(self.a, 0)

    def test_2(self, table_types):
        self._setup(table_types)
        t = table_types.Table()
        t.add_column(self.a, 1)

    def test_3(self, table_types):
        self._setup(table_types)
        t = table_types.Table()
        t.add_column(self.a, -1)

    def test_5(self, table_types):
        self._setup(table_types)
        t = table_types.Table()
        with pytest.raises(ValueError):
            t.index_column('b')

    def test_6(self, table_types):
        self._setup(table_types)
        t = table_types.Table()
        t.add_column(self.a)
        t.add_column(self.b)
        assert t.columns.keys() == ['a', 'b']

    def test_7(self, table_types):
        self._setup(table_types)
        t = table_types.Table([self.a])
        t.add_column(self.b, t.index_column('a'))
        assert t.columns.keys() == ['b', 'a']

    def test_8(self, table_types):
        self._setup(table_types)
        t = table_types.Table([self.a])
        t.add_column(self.b, t.index_column('a') + 1)
        assert t.columns.keys() == ['a', 'b']

    def test_9(self, table_types):
        self._setup(table_types)
        t = table_types.Table()
        t.add_column(self.a)
        t.add_column(self.b, t.index_column('a') + 1)
        t.add_column(self.c, t.index_column('b'))
        assert t.columns.keys() == ['a', 'c', 'b']

    def test_10(self, table_types):
        self._setup(table_types)
        t = table_types.Table()
        t.add_column(self.a)
        ia = t.index_column('a')
        t.add_column(self.b, ia + 1)
        t.add_column(self.c, ia)
        assert t.columns.keys() == ['c', 'a', 'b']

@pytest.mark.usefixtures('table_types')
class TestAddName(SetupData):

    def test_override_name(self, table_types):
        self._setup(table_types)
        t = table_types.Table()

        # Check that we can override the name of the input column in the Table
        t.add_column(self.a, name='b')
        t.add_column(self.b, name='a')
        assert t.columns.keys() == ['b', 'a']
        # Check that we did not change the name of the input column
        assert self.a.info.name == 'a'
        assert self.b.info.name == 'b'

        # Now test with an input column from another table
        t2 = table_types.Table()
        t2.add_column(t['a'], name='c')
        assert t2.columns.keys() == ['c']
        # Check that we did not change the name of the input column
        assert t.columns.keys() == ['b', 'a']

        # Check that we can give a name if none was present
        col = table_types.Column([1, 2, 3])
        t.add_column(col, name='c')
        assert t.columns.keys() == ['b', 'a', 'c']

    def test_default_name(self, table_types):
        t = table_types.Table()
        col = table_types.Column([1, 2, 3])
        t.add_column(col)
        assert t.columns.keys() == ['col0']

@pytest.mark.usefixtures('table_types')
class TestInitFromTable(SetupData):

    def test_from_table_cols(self, table_types):
        """Ensure that using cols from an existing table gives
        a clean copy.
        """
        self._setup(table_types)
        t = self.t
        cols = t.columns
        # Construct Table with cols via Table._new_from_cols
        t2a = table_types.Table([cols['a'], cols['b'], self.c])

        # Construct with add_column
        t2b = table_types.Table()
        t2b.add_column(cols['a'])
        t2b.add_column(cols['b'])
        t2b.add_column(self.c)

        t['a'][1] = 20
        t['b'][1] = 21
        for t2 in [t2a, t2b]:
            t2['a'][2] = 10
            t2['b'][2] = 11
            t2['c'][2] = 12
            t2.columns['a'].meta['aa'][3] = 10
            assert np.all(t['a'] == np.array([1, 20, 3]))
            assert np.all(t['b'] == np.array([4, 21, 6]))
            assert np.all(t2['a'] == np.array([1, 2, 10]))
            assert np.all(t2['b'] == np.array([4, 5, 11]))
            assert np.all(t2['c'] == np.array([7, 8, 12]))
            assert t2['a'].name == 'a'
            assert t2.columns['a'].meta['aa'][3] == 10
            assert t.columns['a'].meta['aa'][3] == 3

@pytest.mark.usefixtures('table_types')
class TestAddColumns(SetupData):

    def test_add_columns1(self, table_types):
        self._setup(table_types)
        t = table_types.Table()
        t.add_columns([self.a, self.b, self.c])
        assert t.colnames == ['a', 'b', 'c']

    def test_add_columns2(self, table_types):
        self._setup(table_types)
        t = table_types.Table([self.a, self.b])
        t.add_columns([self.c, self.d])
        assert t.colnames == ['a', 'b', 'c', 'd']
        assert np.all(t['c'] == np.array([7, 8, 9]))

    def test_add_columns3(self, table_types):
        self._setup(table_types)
        t = table_types.Table([self.a, self.b])
        t.add_columns([self.c, self.d], indexes=[1, 0])
        assert t.colnames == ['d', 'a', 'c', 'b']

    def test_add_columns4(self, table_types):
        self._setup(table_types)
        t = table_types.Table([self.a, self.b])
        t.add_columns([self.c, self.d], indexes=[0, 0])
        assert t.colnames == ['c', 'd', 'a', 'b']

    def test_add_columns5(self, table_types):
        self._setup(table_types)
        t = table_types.Table([self.a, self.b])
        t.add_columns([self.c, self.d], indexes=[2, 2])
        assert t.colnames == ['a', 'b', 'c', 'd']

    def test_add_columns6(self, table_types):
        """Check that we can override column names."""
        self._setup(table_types)
        t = table_types.Table()
        t.add_columns([self.a, self.b, self.c], names=['b', 'c', 'a'])
        assert t.colnames == ['b', 'c', 'a']

    def test_add_columns7(self, table_types):
        """Check that default names are used when appropriate."""
        t = table_types.Table()
        col0 = table_types.Column([1, 2, 3])
        col1 = table_types.Column([4, 5, 3])
        t.add_columns([col0, col1])
        assert t.colnames == ['col0', 'col1']

    def test_add_duplicate_column(self, table_types):
        self._setup(table_types)
        t = table_types.Table()
        t.add_column(self.a)
        with pytest.raises(ValueError):
            t.add_column(table_types.Column(name='a', data=[0, 1, 2]))
        t.add_column(table_types.Column(name='a', data=[0, 1, 2]),
                     rename_duplicate=True)
        t.add_column(self.b)
        t.add_column(self.c)
        assert t.colnames == ['a', 'a_1', 'b', 'c']
        t.add_column(table_types.Column(name='a', data=[0, 1, 2]),
                     rename_duplicate=True)
        assert t.colnames == ['a', 'a_1', 'b', 'c', 'a_2']

        # test adding column from a separate Table
        t1 = table_types.Table()
        t1.add_column(self.a)
        with pytest.raises(ValueError):
            t.add_column(t1['a'])
        t.add_column(t1['a'], rename_duplicate=True)

        t1['a'][0] = 100  # Change original column
        assert t.colnames == ['a', 'a_1', 'b', 'c', 'a_2', 'a_3']
        assert t1.colnames == ['a']

        # Check new column didn't change (since name conflict forced a copy)
        assert t['a_3'][0] == self.a[0]

    def test_add_duplicate_columns(self, table_types):
        self._setup(table_types)
        t = table_types.Table([self.a, self.b, self.c])
        with pytest.raises(ValueError):
            t.add_columns([table_types.Column(name='a', data=[0, 1, 2]), table_types.Column(name='b', data=[0, 1, 2])])
        t.add_columns([table_types.Column(name='a', data=[0, 1, 2]),
                       table_types.Column(name='b', data=[0, 1, 2])],
                      rename_duplicate=True)
        t.add_column(self.d)
        assert t.colnames == ['a', 'b', 'c', 'a_1', 'b_1', 'd']

@pytest.mark.usefixtures('table_types')
class TestAddRow(SetupData):

    @property
    def b(self):
        if self._column_type is not None:
            if not hasattr(self, '_b'):
                self._b = self._column_type(name='b', data=[4.0, 5.1, 6.2])
            return self._b

    @property
    def c(self):
        if self._column_type is not None:
            if not hasattr(self, '_c'):
                self._c = self._column_type(name='c', data=['7', '8', '9'])
            return self._c

    @property
    def d(self):
        if self._column_type is not None:
            if not hasattr(self, '_d'):
                self._d = self._column_type(name='d', data=[[1, 2], [3, 4], [5, 6]])
            return self._d

    @property
    def t(self):
        if self._table_type is not None:
            if not hasattr(self, '_t'):
                self._t = self._table_type([self.a, self.b, self.c])
            return self._t

    def test_add_none_to_empty_table(self, table_types):
        self._setup(table_types)
        t = table_types.Table(names=('a', 'b', 'c'), dtype=('(2,)i', 'S4', 'O'))
        t.add_row()
        assert np.all(t['a'][0] == [0, 0])
        assert t['b'][0] == ''
        assert t['c'][0] == 0
        t.add_row()
        assert np.all(t['a'][1] == [0, 0])
        assert t['b'][1] == ''
        assert t['c'][1] == 0

    def test_add_stuff_to_empty_table(self, table_types):
        self._setup(table_types)
        t = table_types.Table(names=('a', 'b', 'obj'), dtype=('(2,)i', 'S8', 'O'))
        t.add_row([[1, 2], 'hello', 'world'])
        assert np.all(t['a'][0] == [1, 2])
        assert t['b'][0] == 'hello'
        assert t['obj'][0] == 'world'
        # Make sure it is not repeating last row but instead
        # adding zeros (as documented)
        t.add_row()
        assert np.all(t['a'][1] == [0, 0])
        assert t['b'][1] == ''
        assert t['obj'][1] == 0

    def test_add_table_row(self, table_types):
        self._setup(table_types)
        t = self.t
        t['d'] = self.d
        t2 = table_types.Table([self.a, self.b, self.c, self.d])
        t.add_row(t2[0])
        assert len(t) == 4
        assert np.all(t['a'] == np.array([1, 2, 3, 1]))
        assert np.allclose(t['b'], np.array([4.0, 5.1, 6.2, 4.0]))
        assert np.all(t['c'] == np.array(['7', '8', '9', '7']))
        assert np.all(t['d'] == np.array([[1, 2], [3, 4], [5, 6], [1, 2]]))

    def test_add_table_row_obj(self, table_types):
        self._setup(table_types)
        t = table_types.Table([self.a, self.b, self.obj])
        t.add_row([1, 4.0, [10]])
        assert len(t) == 4
        assert np.all(t['a'] == np.array([1, 2, 3, 1]))
        assert np.allclose(t['b'], np.array([4.0, 5.1, 6.2, 4.0]))
        assert np.all(t['obj'] == np.array([1, 'string', 3, [10]], dtype='O'))

    def test_add_qtable_row_multidimensional(self):
        q = [[1, 2], [3, 4]] * u.m
        qt = table.QTable([q])
        qt.add_row(([5, 6] * u.km,))
        assert np.all(qt['col0'] == [[1, 2], [3, 4], [5000, 6000]] * u.m)

    def test_add_with_tuple(self, table_types):
        self._setup(table_types)
        t = self.t
        t.add_row((4, 7.2, '1'))
        assert len(t) == 4
        assert np.all(t['a'] == np.array([1, 2, 3, 4]))
        assert np.allclose(t['b'], np.array([4.0, 5.1, 6.2, 7.2]))
        assert np.all(t['c'] == np.array(['7', '8', '9', '1']))

    def test_add_with_list(self, table_types):
        self._setup(table_types)
        t = self.t
        t.add_row([4, 7.2, '10'])
        assert len(t) == 4
        assert np.all(t['a'] == np.array([1, 2, 3, 4]))
        assert np.allclose(t['b'], np.array([4.0, 5.1, 6.2, 7.2]))
        assert np.all(t['c'] == np.array(['7', '8', '9', '1']))

    def test_add_with_dict(self, table_types):
        self._setup(table_types)
        t = self.t
        t.add_row({'a': 4, 'b': 7.2})
        assert len(t) == 4
        assert np.all(t['a'] == np.array([1, 2, 3, 4]))
        assert np.allclose(t['b'], np.array([4.0, 5.1, 6.2, 7.2]))
        if t.masked:
            assert np.all(t['c'] == np.array(['7', '8', '9', '7']))
        else:
            assert np.all(t['c'] == np.array(['7', '8', '9', '']))

    def test_add_with_none(self, table_types):
        self._setup(table_types)
        t = self.t
        t.add_row()
        assert len(t) == 4
        assert np.all(t['a'].data == np.array([1, 2, 3, 0]))
        assert np.allclose(t['b'], np.array([4.0, 5.1, 6.2, 0.0]))
        assert np.all(t['c'].data == np.array(['7', '8', '9', '']))

    def test_add_missing_column(self, table_types):
        self._setup(table_types)
        t = self.t
        with pytest.raises(ValueError):
            t.add_row({'bad_column': 1})

    def test_wrong_size_tuple(self, table_types):
        self._setup(table_types)
        t = self.t
        with pytest.raises(ValueError):
            t.add_row((1, 2))

    def test_wrong_vals_type(self, table_types):
        self._setup(table_types)
        t = self.t
        with pytest.raises(TypeError):
            t.add_row(1)

    def test_add_row_failures(self, table_types):
        self._setup(table_types)
        t = self.t
        t_copy = table_types.Table(t, copy=True)
        # Wrong number of columns
        try:
            t.add_row([1, 2, 3, 4])
        except ValueError:
            pass
        assert len(t) == 3
        assert np.all(t.as_array() == t_copy.as_array())
        # Wrong data type
        try:
            t.add_row(['one', 2, 3])
        except ValueError:
            pass
        assert len(t) == 3
        assert np.all(t.as_array() == t_copy.as_array())

    def test_insert_table_row(self, table_types):
        """
        Light testing of Table.insert_row() method.  The deep testing is done via
        the add_row() tests which calls insert_row(index=len(self), ...), so
        here just test that the added index parameter is handled correctly.
        """
        self._setup(table_types)
        row = (10, 40.0, 'x', [10, 20])
        for index in range(-3, 4):
            indices = np.insert(np.arange(3), index, 3)
            t = table_types.Table([self.a, self.b, self.c, self.d])
            t2 = t.copy()
            t.add_row(row)  # By now we know this works
            t2.insert_row(index, row)
            for name in t.colnames:
                if t[name].dtype.kind == 'f':
                    assert np.allclose(t[name][indices], t2[name])
                else:
                    assert np.all(t[name][indices] == t2[name])

        for index in (-4, 4):
            t = table_types.Table([self.a, self.b, self.c, self.d])
            with pytest.raises(IndexError):
                t.insert_row(index, row)

@pytest.mark.usefixtures('table_types')
class TestTableColumn(SetupData):

    def test_column_view(self, table_types):
        self._setup(table_types)
        t = self.t
        a = t.columns['a']
        a[2] = 10
        assert t['a'][2] == 10

@pytest.mark.usefixtures('table_types')
class TestArrayColumns(SetupData):

    def test_1d(self, table_types):
        self._setup(table_types)
        b = table_types.Column(name='b', dtype=int, shape=(2, ), length=3)
        t = table_types.Table([self.a])
        t.add_column(b)
        assert t['b'].shape == (3, 2)
        assert t['b'][0].shape == (2, )

    def test_2d(self, table_types):
        self._setup(table_types)
        b = table_types.Column(name='b', dtype=int, shape=(2, 4), length=3)
        t = table_types.Table([self.a])
        t.add_column(b)
        assert t['b'].shape == (3, 2, 4)
        assert t['b'][0].shape == (2, 4)

    def test_3d(self, table_types):
        self._setup(table_types)
        t = table_types.Table([self.a])
        b = table_types.Column(name='b', dtype=int, shape=(2, 4, 6), length=3)
        t.add_column(b)
        assert t['b'].shape == (3, 2, 4, 6)
        assert t['b'][0].shape == (2, 4, 6)

@pytest.mark.usefixtures('table_types')
class TestRemove(SetupData):

    @property
    def t(self):
        if self._table_type is not None:
            if not hasattr(self, '_t'):
                self._t = self._table_type([self.a])
            return self._t

    @property
    def t2(self):
        if self._table_type is not None:
            if not hasattr(self, '_t2'):
                self._t2 = self._table_type([self.a, self.b, self.c])
            return self._t2

    def test_1(self, table_types):
        self._setup(table_types)
        self.t.remove_columns('a')
        assert self.t.columns.keys() == []
        assert self.t.as_array() is None

    def test_2(self, table_types):
        self._setup(table_types)
        self.t.add_column(self.b)
        self.t.remove_columns('a')
        assert self.t.columns.keys() == ['b']
        assert self.t.dtype.names == ('b',)
        assert np.all(self.t['b'] == np.array([4, 5, 6]))

    def test_3(self, table_types):
        """Check remove_columns works for a single column with a name of
        more than one character.  Regression test against #2699"""
        self._setup(table_types)
        self.t['new_column'] = self.t['a']
        assert 'new_column' in self.t.columns.keys()
        self.t.remove_columns('new_column')
        assert 'new_column' not in self.t.columns.keys()

    def test_remove_nonexistent_row(self, table_types):
        self._setup(table_types)
        with pytest.raises(IndexError):
            self.t.remove_row(4)

    def test_remove_row_0(self, table_types):
        self._setup(table_types)
        self.t.add_column(self.b)
        self.t.add_column(self.c)
        self.t.remove_row(0)
        assert self.t.colnames == ['a', 'b', 'c']
        assert np.all(self.t['b'] == np.array([5, 6]))

    def test_remove_row_1(self, table_types):
        self._setup(table_types)
        self.t.add_column(self.b)
        self.t.add_column(self.c)
        self.t.remove_row(1)
        assert self.t.colnames == ['a', 'b', 'c']
        assert np.all(self.t['a'] == np.array([1, 3]))

    def test_remove_row_2(self, table_types):
        self._setup(table_types)
        self.t.add_column(self.b)
        self.t.add_column(self.c)
        self.t.remove_row(2)
        assert self.t.colnames == ['a', 'b', 'c']
        assert np.all(self.t['c'] == np.array([7, 8]))

    def test_remove_row_slice(self, table_types):
        self._setup(table_types)
        self.t.add_column(self.b)
        self.t.add_column(self.c)
        self.t.remove_rows(slice(0, 2, 1))
        assert self.t.colnames == ['a', 'b', 'c']
        assert np.all(self.t['c'] == np.array([9]))

    def test_remove_row_list(self, table_types):
        self._setup(table_types)
        self.t.add_column(self.b)
        self.t.add_column(self.c)
        self.t.remove_rows([0, 2])
        assert self.t.colnames == ['a', 'b', 'c']
        assert np.all(self.t['c'] == np.array([8]))

    def test_remove_row_preserves_meta(self, table_types):
        self._setup(table_types)
        self.t.add_column(self.b)
        self.t.remove_rows([0, 2])
        assert self.t['a'].meta == {'aa': [0, 1, 2, 3, 4]}
        assert self.t.dtype == np.dtype([(str('a'), 'int'),
                                         (str('b'), 'int')])

    def test_delitem_row(self, table_types):
        self._setup(table_types)
        self.t.add_column(self.b)
        self.t.add_column(self.c)
        del self.t[1]
        assert self.t.colnames == ['a', 'b', 'c']
        assert np.all(self.t['a'] == np.array([1, 3]))

    @pytest.mark.parametrize("idx", [[0, 2], np.array([0, 2])])
    def test_delitem_row_list(self, table_types, idx):
        self._setup(table_types)
        self.t.add_column(self.b)
        self.t.add_column(self.c)
        del self.t[idx]
        assert self.t.colnames == ['a', 'b', 'c']
        assert np.all(self.t['c'] == np.array([8]))

    def test_delitem_row_slice(self, table_types):
        self._setup(table_types)
        self.t.add_column(self.b)
        self.t.add_column(self.c)
        del self.t[0:2]
        assert self.t.colnames == ['a', 'b', 'c']
        assert np.all(self.t['c'] == np.array([9]))

    def test_delitem_row_fail(self, table_types):
        self._setup(table_types)
        with pytest.raises(IndexError):
            del self.t[4]

    def test_delitem_row_float(self, table_types):
        self._setup(table_types)
        with pytest.raises(IndexError):
            del self.t[1.]

    def test_delitem1(self, table_types):
        self._setup(table_types)
        del self.t['a']
        assert self.t.columns.keys() == []
        assert self.t.as_array() is None

    def test_delitem2(self, table_types):
        self._setup(table_types)
        del self.t2['b']
        assert self.t2.colnames == ['a', 'c']

    def test_delitems(self, table_types):
        self._setup(table_types)
        del self.t2['a', 'b']
        assert self.t2.colnames == ['c']

    def test_delitem_fail(self, table_types):
        self._setup(table_types)
        with pytest.raises(KeyError):
            del self.t['d']

@pytest.mark.usefixtures('table_types')
class TestKeep(SetupData):

    def test_1(self, table_types):
        self._setup(table_types)
        t = table_types.Table([self.a, self.b])
        t.keep_columns([])
        assert t.columns.keys() == []
        assert t.as_array() is None

    def test_2(self, table_types):
        self._setup(table_types)
        t = table_types.Table([self.a, self.b])
        t.keep_columns('b')
        assert t.columns.keys() == ['b']
        assert t.dtype.names == ('b',)
        assert np.all(t['b'] == np.array([4, 5, 6]))

@pytest.mark.usefixtures('table_types')
class TestRename(SetupData):

    def test_1(self, table_types):
        self._setup(table_types)
        t = table_types.Table([self.a])
        t.rename_column('a', 'b')
        assert t.columns.keys() == ['b']
        assert t.dtype.names == ('b',)
        assert np.all(t['b'] == np.array([1, 2, 3]))

    def test_2(self, table_types):
        self._setup(table_types)
        t = table_types.Table([self.a, self.b])
        t.rename_column('a', 'c')
        t.rename_column('b', 'a')
        assert t.columns.keys() == ['c', 'a']
        assert t.dtype.names == ('c', 'a')
        if t.masked:
            assert t.mask.dtype.names == ('c', 'a')
        assert np.all(t['c'] == np.array([1, 2, 3]))
        assert np.all(t['a'] == np.array([4, 5, 6]))

    def test_rename_by_attr(self, table_types):
        self._setup(table_types)
        t = table_types.Table([self.a, self.b])
        t['a'].name = 'c'
        t['b'].name = 'a'
        assert t.columns.keys() == ['c', 'a']
        assert t.dtype.names == ('c', 'a')
        assert np.all(t['c'] == np.array([1, 2, 3]))
        assert np.all(t['a'] == np.array([4, 5, 6]))

@pytest.mark.usefixtures('table_types')
class TestSort():

    def test_single(self, table_types):
        t = table_types.Table()
        t.add_column(table_types.Column(name='a', data=[2, 1, 3]))
        t.add_column(table_types.Column(name='b', data=[6, 5, 4]))
        t.add_column(table_types.Column(name='c', data=[(1, 2), (3, 4), (4, 5)]))
        assert np.all(t['a'] == np.array([2, 1, 3]))
        assert np.all(t['b'] == np.array([6, 5, 4]))
        t.sort('a')
        assert np.all(t['a'] == np.array([1, 2, 3]))
        assert np.all(t['b'] == np.array([5, 6, 4]))
        assert np.all(t['c'] == np.array([[3, 4],
                                          [1, 2],
                                          [4, 5]]))
        t.sort('b')
        assert np.all(t['a'] == np.array([3, 1, 2]))
        assert np.all(t['b'] == np.array([4, 5, 6]))
        assert np.all(t['c'] == np.array([[4, 5],
                                          [3, 4],
                                          [1, 2]]))

    def test_single_big(self, table_types):
        """Sort a big-ish table with a non-trivial sort order"""
        x = np.arange(10000)
        y = np.sin(x)
        t = table_types.Table([x, y], names=('x', 'y'))
        t.sort('y')
        idx = np.argsort(y)
        assert np.all(t['x'] == x[idx])
        assert np.all(t['y'] == y[idx])

    def test_empty(self, table_types):
        t = table_types.Table([[], []], dtype=['f4', 'U1'])
        t.sort('col1')

    def test_multiple(self, table_types):
        t = table_types.Table()
        t.add_column(table_types.Column(name='a', data=[2, 1, 3, 2, 3, 1]))
        t.add_column(table_types.Column(name='b', data=[6, 5, 4, 3, 5, 4]))
        assert np.all(t['a'] == np.array([2, 1, 3, 2, 3, 1]))
        assert np.all(t['b'] == np.array([6, 5, 4, 3, 5, 4]))
        t.sort(['a', 'b'])
        assert np.all(t['a'] == np.array([1, 1, 2, 2, 3, 3]))
        assert np.all(t['b'] == np.array([4, 5, 3, 6, 4, 5]))
        t.sort(['b', 'a'])
        assert np.all(t['a'] == np.array([2, 1, 3, 1, 3, 2]))
        assert np.all(t['b'] == np.array([3, 4, 4, 5, 5, 6]))
        t.sort(('a', 'b'))
        assert np.all(t['a'] == np.array([1, 1, 2, 2, 3, 3]))
        assert np.all(t['b'] == np.array([4, 5, 3, 6, 4, 5]))

    def test_multiple_with_bytes(self, table_types):
        t = table_types.Table()
        t.add_column(table_types.Column(name='firstname', data=[b"Max", b"Jo", b"John"]))
        t.add_column(table_types.Column(name='name', data=[b"Miller", b"Miller", b"Jackson"]))
        t.add_column(table_types.Column(name='tel', data=[12, 15, 19]))
        t.sort(['name', 'firstname'])
        assert np.all([t['firstname'] == np.array([b"John", b"Jo", b"Max"])])
        assert np.all([t['name'] == np.array([b"Jackson", b"Miller", b"Miller"])])
        assert np.all([t['tel'] == np.array([19, 15, 12])])

    def test_multiple_with_unicode(self, table_types):
        # Before Numpy 1.6.2, sorting with multiple column names
        # failed when a unicode column was present.
        t = table_types.Table()
        t.add_column(table_types.Column(
            name='firstname',
            data=[str(x) for x in ["Max", "Jo", "John"]]))
        t.add_column(table_types.Column(
            name='name',
            data=[str(x) for x in ["Miller", "Miller", "Jackson"]]))
        t.add_column(table_types.Column(name='tel', data=[12, 15, 19]))
        t.sort(['name', 'firstname'])
        assert np.all([t['firstname'] == np.array(
            [str(x) for x in ["John", "Jo", "Max"]])])
        assert np.all([t['name'] == np.array(
            [str(x) for x in ["Jackson", "Miller", "Miller"]])])
        assert np.all([t['tel'] == np.array([19, 15, 12])])

    def test_argsort(self, table_types):
        t = table_types.Table()
        t.add_column(table_types.Column(name='a', data=[2, 1, 3, 2, 3, 1]))
        t.add_column(table_types.Column(name='b', data=[6, 5, 4, 3, 5, 4]))
        assert np.all(t.argsort() == t.as_array().argsort())
        i0 = t.argsort('a')
        i1 = t.as_array().argsort(order=['a'])
        assert np.all(t['a'][i0] == t['a'][i1])
        i0 = t.argsort(['a', 'b'])
        i1 = t.as_array().argsort(order=['a', 'b'])
        assert np.all(t['a'][i0] == t['a'][i1])
        assert np.all(t['b'][i0] == t['b'][i1])

    def test_argsort_bytes(self, table_types):
        t = table_types.Table()
        t.add_column(table_types.Column(name='firstname', data=[b"Max", b"Jo", b"John"]))
        t.add_column(table_types.Column(name='name', data=[b"Miller", b"Miller", b"Jackson"]))
        t.add_column(table_types.Column(name='tel', data=[12, 15, 19]))
        assert np.all(t.argsort(['name', 'firstname']) == np.array([2, 1, 0]))

    def test_argsort_unicode(self, table_types):
        # Before Numpy 1.6.2, sorting with multiple column names
        # failed when a unicode column was present.
        t = table_types.Table()
        t.add_column(table_types.Column(
            name='firstname',
            data=[str(x) for x in ["Max", "Jo", "John"]]))
        t.add_column(table_types.Column(
            name='name',
            data=[str(x) for x in ["Miller", "Miller", "Jackson"]]))
        t.add_column(table_types.Column(name='tel', data=[12, 15, 19]))
        assert np.all(t.argsort(['name', 'firstname']) == np.array([2, 1, 0]))

    def test_rebuild_column_view_then_rename(self, table_types):
        """
        Issue #2039 where renaming fails after any method that calls
        _rebuild_table_column_view (this includes sort and add_row).
        """
        t = table_types.Table([[1]], names=('a',))
        assert t.colnames == ['a']
        assert t.dtype.names == ('a',)

        t.add_row((2,))
        assert t.colnames == ['a']
        assert t.dtype.names == ('a',)

        t.rename_column('a', 'b')
        assert t.colnames == ['b']
        assert t.dtype.names == ('b',)

        t.sort('b')
        assert t.colnames == ['b']
        assert t.dtype.names == ('b',)

        t.rename_column('b', 'c')
        assert t.colnames == ['c']
        assert t.dtype.names == ('c',)

@pytest.mark.usefixtures('table_types')
class TestIterator():

    def test_iterator(self, table_types):
        d = np.array([(2, 1),
                      (3, 6),
                      (4, 5)], dtype=[(str('a'), 'i4'), (str('b'), 'i4')])
        t = table_types.Table(d)
        if t.masked:
            with pytest.raises(ValueError):
                t[0] == d[0]
        else:
            for row, np_row in zip(t, d):
                assert np.all(row == np_row)

@pytest.mark.usefixtures('table_types')
class TestSetMeta():

    def test_set_meta(self, table_types):
        d = table_types.Table(names=('a', 'b'))
        d.meta['a'] = 1
        d.meta['b'] = 1
        d.meta['c'] = 1
        d.meta['d'] = 1
        assert list(d.meta.keys()) == ['a', 'b', 'c', 'd']

@pytest.mark.usefixtures('table_types')
class TestConvertNumpyArray():

    def test_convert_numpy_array(self, table_types):
        d = table_types.Table([[1, 2], [3, 4]], names=('a', 'b'))

        np_data = np.array(d)
        if table_types.Table is not MaskedTable:
            assert np.all(np_data == d.as_array())
        assert np_data is not d.as_array()
        assert d.colnames == list(np_data.dtype.names)

        np_data = np.array(d, copy=False)
        if table_types.Table is not MaskedTable:
            assert np.all(np_data == d.as_array())
        assert d.colnames == list(np_data.dtype.names)

        with pytest.raises(ValueError):
            np_data = np.array(d, dtype=[(str('c'), 'i8'), (str('d'), 'i8')])

    def test_as_array_byteswap(self, table_types):
        """Test for https://github.com/astropy/astropy/pull/4080"""

        byte_orders = ('>', '<')
        native_order = byte_orders[sys.byteorder == 'little']

        for order in byte_orders:
            col = table_types.Column([1.0, 2.0], name='a', dtype=order + 'f8')
            t = table_types.Table([col])
            arr = t.as_array()
            assert arr['a'].dtype.byteorder in (native_order, '=')
            arr = t.as_array(keep_byteorder=True)
            if order == native_order:
                assert arr['a'].dtype.byteorder in (order, '=')
            else:
                assert arr['a'].dtype.byteorder == order

    def test_byteswap_fits_array(self, table_types):
        """
        Test for https://github.com/astropy/astropy/pull/4080, demonstrating
        that FITS tables are converted to native byte order.
        """

        non_native_order = ('>', '<')[sys.byteorder != 'little']

        filename = get_pkg_data_filename('data/tb.fits',
                                         'astropy.io.fits.tests')
        t = table_types.Table.read(filename)
        arr = t.as_array()

        for idx in range(len(arr.dtype)):
            assert arr.dtype[idx].byteorder != non_native_order

        with fits.open(filename, character_as_bytes=True) as hdul:
            data = hdul[1].data
            for colname in data.columns.names:
                assert np.all(data[colname] == arr[colname])

            arr2 = t.as_array(keep_byteorder=True)
            for colname in data.columns.names:
                assert (data[colname].dtype.byteorder ==
                        arr2[colname].dtype.byteorder)

def _assert_copies(t, t2, deep=True):
    assert t.colnames == t2.colnames
    np.testing.assert_array_equal(t.as_array(), t2.as_array())
    assert t.meta == t2.meta

    for col, col2 in zip(t.columns.values(), t2.columns.values()):
        if deep:
            assert not np.may_share_memory(col, col2)
        else:
            assert np.may_share_memory(col, col2)

def test_copy():
    t = table.Table([[1, 2, 3], [2, 3, 4]], names=['x', 'y'])
    t2 = t.copy()
    _assert_copies(t, t2)

def test_copy_masked():
    t = table.Table([[1, 2, 3], [2, 3, 4]], names=['x', 'y'], masked=True,
                    meta={'name': 'test'})
    t['x'].mask == [True, False, True]
    t2 = t.copy()
    _assert_copies(t, t2)

def test_copy_protocol():
    t = table.Table([[1, 2, 3], [2, 3, 4]], names=['x', 'y'])

    t2 = copy.copy(t)
    t3 = copy.deepcopy(t)

    _assert_copies(t, t2, deep=False)
    _assert_copies(t, t3)

def test_disallow_inequality_comparisons():
    """
    Regression test for #828 - disallow comparison operators on whole Table
    """

    t = table.Table()

    with pytest.raises(TypeError):
        t > 2

    with pytest.raises(TypeError):
        t < 1.1

    with pytest.raises(TypeError):
        t >= 5.5

    with pytest.raises(TypeError):
        t <= -1.1

def test_equality():

    t = table.Table.read([' a b  c  d',
                          ' 2 c 7.0 0',
                          ' 2 b 5.0 1',
                          ' 2 b 6.0 2',
                          ' 2 a 4.0 3',
                          ' 0 a 0.0 4',
                          ' 1 b 3.0 5',
                          ' 1 a 2.0 6',
                          ' 1 a 1.0 7',
                         ], format='ascii')

    # All rows are equal
    assert np.all(t == t)

    # Assert no rows are different
    assert not np.any(t != t)

    # Check equality result for a given row
    assert np.all((t == t[3]) == np.array([0, 0, 0, 1, 0, 0, 0, 0], dtype=bool))

    # Check inequality result for a given row
    assert np.all((t != t[3]) == np.array([1, 1, 1, 0, 1, 1, 1, 1], dtype=bool))

    t2 = table.Table.read([' a b  c  d',
                           ' 2 c 7.0 0',
                           ' 2 b 5.0 1',
                           ' 3 b 6.0 2',
                           ' 2 a 4.0 3',
                           ' 0 a 1.0 4',
                           ' 1 b 3.0 5',
                           ' 1 c 2.0 6',
                           ' 1 a 1.0 7',
                          ], format='ascii')

    # In the above cases, Row.__eq__ gets called, but now need to make sure
    # Table.__eq__ also gets called.
    assert np.all((t == t2) == np.array([1, 1, 0, 1, 0, 1, 0, 1], dtype=bool))
    assert np.all((t != t2) == np.array([0, 0, 1, 0, 1, 0, 1, 0], dtype=bool))

    # Check that comparing to a structured array works
    assert np.all((t == t2.as_array()) == np.array([1, 1, 0, 1, 0, 1, 0, 1], dtype=bool))
    assert np.all((t.as_array() == t2) == np.array([1, 1, 0, 1, 0, 1, 0, 1], dtype=bool))

def test_equality_masked():

    t = table.Table.read([' a b  c  d',
                          ' 2 c 7.0 0',
                          ' 2 b 5.0 1',
                          ' 2 b 6.0 2',
                          ' 2 a 4.0 3',
                          ' 0 a 0.0 4',
                          ' 1 b 3.0 5',
                          ' 1 a 2.0 6',
                          ' 1 a 1.0 7',
                         ], format='ascii')

    # Make into masked table
    t = table.Table(t, masked=True)

    # All rows are equal
    assert np.all(t == t)

    # Assert no rows are different
    assert not np.any(t != t)

    # Check equality result for a given row
    assert np.all((t == t[3]) == np.array([0, 0, 0, 1, 0, 0, 0, 0], dtype=bool))

    # Check inequality result for a given row
    assert np.all((t != t[3]) == np.array([1, 1, 1, 0, 1, 1, 1, 1], dtype=bool))

    t2 = table.Table.read([' a b  c  d',
                           ' 2 c 7.0 0',
                           ' 2 b 5.0 1',
                           ' 3 b 6.0 2',
                           ' 2 a 4.0 3',
                           ' 0 a 1.0 4',
                           ' 1 b 3.0 5',
                           ' 1 c 2.0 6',
                           ' 1 a 1.0 7',
                          ], format='ascii')

    # In the above cases, Row.__eq__ gets called, but now need to make sure
    # Table.__eq__ also gets called.
    assert np.all((t == t2) == np.array([1, 1, 0, 1, 0, 1, 0, 1], dtype=bool))
    assert np.all((t != t2) == np.array([0, 0, 1, 0, 1, 0, 1, 0], dtype=bool))

    # Check that masking a value causes the row to differ
    t.mask['a'][0] = True
    assert np.all((t == t2) == np.array([0, 1, 0, 1, 0, 1, 0, 1], dtype=bool))
    assert np.all((t != t2) == np.array([1, 0, 1, 0, 1, 0, 1, 0], dtype=bool))

    # Check that comparing to a structured array works
    assert np.all((t == t2.as_array()) == np.array([0, 1, 0, 1, 0, 1, 0, 1], dtype=bool))

@pytest.mark.xfail
def test_equality_masked_bug():
    """
    This highlights a Numpy bug. Once it works, it can be moved into the
    test_equality_masked test. Related Numpy bug report:

      https://github.com/numpy/numpy/issues/3840
    """

    t = table.Table.read([' a b  c  d',
                          ' 2 c 7.0 0',
                          ' 2 b 5.0 1',
                          ' 2 b 6.0 2',
                          ' 2 a 4.0 3',
                          ' 0 a 0.0 4',
                          ' 1 b 3.0 5',
                          ' 1 a 2.0 6',
                          ' 1 a 1.0 7',
                         ], format='ascii')

    t = table.Table(t, masked=True)

    t2 = table.Table.read([' a b  c  d',
                           ' 2 c 7.0 0',
                           ' 2 b 5.0 1',
                           ' 3 b 6.0 2',
                           ' 2 a 4.0 3',
                           ' 0 a 1.0 4',
                           ' 1 b 3.0 5',
                           ' 1 c 2.0 6',
                           ' 1 a 1.0 7',
                          ], format='ascii')

    assert np.all((t.as_array() == t2) == np.array([0, 1, 0, 1, 0, 1, 0, 1], dtype=bool))

# Check that the meta descriptor is working as expected. The MetaBaseTest class
# takes care of defining all the tests, and we simply have to define the class
# and any minimal set of args to pass.

from ...utils.tests.test_metadata import MetaBaseTest

class TestMetaTable(MetaBaseTest):
    test_class = table.Table
    args = ()

def test_unicode_content():
    # If we don't have unicode literals then return
    if isinstance('', bytes):
        return

    # Define unicode literals
    string_a = 'астрономическая питона'
    string_b = 'миллиарды световых лет'

    a = table.Table(
        [[string_a, 2],
         [string_b, 3]],
        names=('a', 'b'))

    assert string_a in str(a)
    # This only works because the coding of this file is utf-8, which
    # matches the default encoding of Table.__str__
    assert string_a.encode('utf-8') in bytes(a)

def test_unicode_policy():
    t = table.Table.read([' a b  c  d',
                          ' 2 c 7.0 0',
                          ' 2 b 5.0 1',
                          ' 2 b 6.0 2',
                          ' 2 a 4.0 3',
                          ' 0 a 0.0 4',
                          ' 1 b 3.0 5',
                          ' 1 a 2.0 6',
                          ' 1 a 1.0 7',
                         ], format='ascii')
    assert_follows_unicode_guidelines(t)

def test_unicode_bytestring_conversion(table_types):
    t = table_types.Table([['abc'], ['def'], [1]], dtype=('S', 'U', 'i'))
    assert t['col0'].dtype.kind == 'S'
    assert t['col1'].dtype.kind == 'U'
    assert t['col2'].dtype.kind == 'i'

    t1 = t.copy()
    t1.convert_unicode_to_bytestring()
    assert t1['col0'].dtype.kind == 'S'
    assert t1['col1'].dtype.kind == 'S'
    assert t1['col2'].dtype.kind == 'i'
    assert t1['col0'][0] == 'abc'
    assert t1['col1'][0] == 'def'
    assert t1['col2'][0] == 1

    t1 = t.copy()
    t1.convert_bytestring_to_unicode()
    assert t1['col0'].dtype.kind == 'U'
    assert t1['col1'].dtype.kind == 'U'
    assert t1['col2'].dtype.kind == 'i'
    assert t1['col0'][0] == str('abc')
    assert t1['col1'][0] == str('def')
    assert t1['col2'][0] == 1

def test_table_deletion():
    """
    Regression test for the reference cycle discussed in
    https://github.com/astropy/astropy/issues/2877
    """

    deleted = set()

    # A special table subclass which leaves a record when it is finalized
    class TestTable(table.Table):
        def __del__(self):
            deleted.add(id(self))

    t = TestTable({'a': [1, 2, 3]})
    the_id = id(t)
    assert t['a'].parent_table is t

    del t

    # Cleanup
    gc.collect()

    assert the_id in deleted

def test_nested_iteration():
    """
    Regression test for issue 3358 where nested iteration over a single table fails.
    """
    t = table.Table([[0, 1]], names=['a'])
    out = []
    for r1 in t:
        for r2 in t:
            out.append((r1['a'], r2['a']))
    assert out == [(0, 0), (0, 1), (1, 0), (1, 1)]

def test_table_init_from_degenerate_arrays(table_types):
    t = table_types.Table(np.array([]))
    assert len(t.columns) == 0

    with pytest.raises(ValueError):
        t = table_types.Table(np.array(0))

    t = table_types.Table(np.array([1, 2, 3]))
    assert len(t.columns) == 3

@pytest.mark.skipif('not HAS_PANDAS')
class TestPandas:

    def test_simple(self):

        t = table.Table()

        for endian in ['<', '>']:
            for kind in ['f', 'i']:
                for byte in ['2', '4', '8']:
                    dtype = np.dtype(endian + kind + byte)
                    x = np.array([1, 2, 3], dtype=dtype)
                    t[endian + kind + byte] = x

        t['u'] = ['a', 'b', 'c']
        t['s'] = ['a', 'b', 'c']

        d = t.to_pandas()

        for column in t.columns:
            if column == 'u':
                assert np.all(t['u'] == np.array(['a', 'b', 'c']))
                assert d[column].dtype == np.dtype("O")  # upstream feature of pandas
            elif column == 's':
                assert np.all(t['s'] == np.array(['a', 'b', 'c']))
                assert d[column].dtype == np.dtype("O")  # upstream feature of pandas
            else:
                # We should be able to compare exact values here
                assert np.all(t[column] == d[column])
                if t[column].dtype.byteorder in ('=', '|'):
                    assert d[column].dtype == t[column].dtype
                else:
                    assert d[column].dtype == t[column].byteswap().newbyteorder().dtype

        # Regression test for astropy/astropy#1156 - the following code gave a
        # ValueError: Big-endian buffer not supported on little-endian
        # compiler. We now automatically swap the endian-ness to native order
        # upon adding the arrays to the data frame.
        d[['<i4', '>i4']]
        d[['<f4', '>f4']]

        t2 = table.Table.from_pandas(d)

        for column in t.columns:
            if column in ('u', 's'):
                assert np.all(t[column] == t2[column])
            else:
                assert_allclose(t[column], t2[column])
            if t[column].dtype.byteorder in ('=', '|'):
                assert t[column].dtype == t2[column].dtype
            else:
                assert t[column].byteswap().newbyteorder().dtype == t2[column].dtype

    def test_2d(self):

        t = table.Table()
        t['a'] = [1, 2, 3]
        t['b'] = np.ones((3, 2))

        with pytest.raises(ValueError) as exc:
            t.to_pandas()
        assert exc.value.args[0] == "Cannot convert a table with multi-dimensional columns to a pandas DataFrame"

    def test_mixin(self):

        from ...coordinates import SkyCoord

        t = table.Table()
        t['c'] = SkyCoord([1, 2, 3], [4, 5, 6], unit='deg')

        with pytest.raises(ValueError) as exc:
            t.to_pandas()
        assert exc.value.args[0] == "Cannot convert a table with mixin columns to a pandas DataFrame"

    def test_masking(self):

        t = table.Table(masked=True)

        t['a'] = [1, 2, 3]
        t['a'].mask = [True, False, True]

        t['b'] = [1., 2., 3.]
        t['b'].mask = [False, False, True]

        t['u'] = ['a', 'b', 'c']
        t['u'].mask = [False, True, False]

        t['s'] = ['a', 'b', 'c']
        t['s'].mask = [False, True, False]

        d = t.to_pandas()

        t2 = table.Table.from_pandas(d)

        for name, column in t.columns.items():
            assert np.all(column.data == t2[name].data)
            assert np.all(column.mask == t2[name].mask)
            # Masked integer type comes back as float.  Nothing we can do about this.
            if column.dtype.kind == 'i':
                assert t2[name].dtype.kind == 'f'
            else:
                if column.dtype.byteorder in ('=', '|'):
                    assert column.dtype == t2[name].dtype
                else:
                    assert column.byteswap().newbyteorder().dtype == t2[name].dtype

@pytest.mark.usefixtures('table_types')
class TestReplaceColumn(SetupData):
    def test_fail_replace_column(self, table_types):
        """Raise exception when trying to replace column via table.columns object"""
        self._setup(table_types)
        t = table_types.Table([self.a, self.b])

        with pytest.raises(ValueError):
            t.columns['a'] = [1, 2, 3]

        with pytest.raises(ValueError):
            t.replace_column('not there', [1, 2, 3])

    def test_replace_column(self, table_types):
        """Replace existing column with a new column"""
        self._setup(table_types)
        t = table_types.Table([self.a, self.b])
        ta = t['a']
        tb = t['b']

        vals = [1.2, 3.4, 5.6]
        for col in (vals,
                    table_types.Column(vals),
                    table_types.Column(vals, name='a'),
                    table_types.Column(vals, name='b')):
            t.replace_column('a', col)
            assert np.all(t['a'] == vals)
            assert t['a'] is not ta  # New a column
            assert t['b'] is tb  # Original b column unchanged
            assert t.colnames == ['a', 'b']
            assert t['a'].meta == {}
            assert t['a'].format is None

    def test_replace_index_column(self, table_types):
        """Replace index column and generate expected exception"""
        self._setup(table_types)
        t = table_types.Table([self.a, self.b])
        t.add_index('a')

        with pytest.raises(ValueError) as err:
            t.replace_column('a', [1, 2, 3])
        assert err.value.args[0] == 'cannot replace a table index column'

class Test__Astropy_Table__():
    """
    Test initializing a Table subclass from a table-like object that
    implements the __astropy_table__ interface method.
    """

    class SimpleTable:
        def __init__(self):
            self.columns = [[1, 2, 3],
                            [4, 5, 6],
                            [7, 8, 9] * u.m]
            self.names = ['a', 'b', 'c']
            self.meta = OrderedDict([('a', 1), ('b', 2)])

        def __astropy_table__(self, cls, copy, **kwargs):
            a, b, c = self.columns
            c.info.name = 'c'
            cols = [table.Column(a, name='a'),
                    table.MaskedColumn(b, name='b'),
                    c]
            names = [col.info.name for col in cols]
            return cls(cols, names=names, copy=copy, meta=kwargs or self.meta)

    def test_simple_1(self):
        """Make a SimpleTable and convert to Table, QTable with copy=False, True"""
        for table_cls in (table.Table, table.QTable):
            col_c_class = u.Quantity if table_cls is table.QTable else table.MaskedColumn
            for cpy in (False, True):
                st = self.SimpleTable()
                # Test putting in a non-native kwarg `extra_meta` to Table initializer
                t = table_cls(st, copy=cpy, extra_meta='extra!')
                assert t.colnames == ['a', 'b', 'c']
                assert t.meta == {'extra_meta': 'extra!'}
                assert np.all(t['a'] == st.columns[0])
                assert np.all(t['b'] == st.columns[1])
                vals = t['c'].value if table_cls is table.QTable else t['c']
                assert np.all(st.columns[2].value == vals)

                assert isinstance(t['a'], table.MaskedColumn)
                assert isinstance(t['b'], table.MaskedColumn)
                assert isinstance(t['c'], col_c_class)
                assert t['c'].unit is u.m
                assert type(t) is table_cls

                # Copy being respected?
                t['a'][0] = 10
                assert st.columns[0][0] == 1 if cpy else 10

    def test_simple_2(self):
        """Test converting a SimpleTable and changing column names and types"""
        st = self.SimpleTable()
        dtypes = [np.int32, np.float32, np.float16]
        names = ['a', 'b', 'c']
        t = table.Table(st, dtype=dtypes, names=names, meta=OrderedDict([('c', 3)]))
        assert t.colnames == names
        assert all(col.dtype.type is dtype
                   for col, dtype in zip(t.columns.values(), dtypes))

        # The supplied meta is ignored.  This is consistent with current
        # behavior when initializing from an existing astropy Table.
        assert t.meta == st.meta

    def test_kwargs_exception(self):
        """If extra kwargs provided but without initializing with a table-like
        object, exception is raised"""
        with pytest.raises(TypeError) as err:
            table.Table([[1]], extra_meta='extra!')
        assert '__init__() got unexpected keyword argument' in str(err)

def test_replace_column_qtable():
    """Replace existing Quantity column with a new column in a QTable"""
    a = [1, 2, 3] * u.m
    b = [4, 5, 6]
    t = table.QTable([a, b], names=['a', 'b'])

    ta = t['a']
    tb = t['b']
    ta.info.meta = {'aa': [0, 1, 2, 3, 4]}
    ta.info.format = '%f'

    t.replace_column('a', a.to('cm'))
    assert np.all(t['a'] == ta)
    assert t['a'] is not ta  # New a column
    assert t['b'] is tb  # Original b column unchanged
    assert t.colnames == ['a', 'b']
    assert t['a'].info.meta is None
    assert t['a'].info.format is None

def test_replace_update_column_via_setitem():
    """
    Test table update like ``t['a'] = value``.  This leverages off the
    already well-tested ``replace_column`` and in-place update
    ``t['a'][:] = value``, so this testing is fairly light.
    """
    a = [1, 2] * u.m
    b = [3, 4]
    t = table.QTable([a, b], names=['a', 'b'])
    assert isinstance(t['a'], u.Quantity)

    # Inplace update
    ta = t['a']
    t['a'] = 5 * u.m
    assert np.all(t['a'] == [5, 5] * u.m)
    assert t['a'] is ta

    # Replace
    t['a'] = [5, 6]
    assert np.all(t['a'] == [5, 6])
    assert isinstance(t['a'], table.Column)
    assert t['a'] is not ta

def test_replace_update_column_via_setitem_warnings_normal():
    """
    Test warnings related to table replace change in #5556:
    Normal warning-free replace
    """
    t = table.Table([[1, 2, 3], [4, 5, 6]], names=['a', 'b'])
    with catch_warnings() as w:
        with table.conf.set_temp('replace_warnings',
                                 ['refcount', 'attributes', 'slice']):
            t['a'] = 0  # in-place update
            assert len(w) == 0

            t['a'] = [10, 20, 30]  # replace column
            assert len(w) == 0

def test_replace_update_column_via_setitem_warnings_slice():
    """
    Test warnings related to table replace change in #5556:
    Replace a slice, one warning.
    """
    t = table.Table([[1, 2, 3], [4, 5, 6]], names=['a', 'b'])
    with catch_warnings() as w:
        with table.conf.set_temp('replace_warnings',
                                 ['refcount', 'attributes', 'slice']):
            t2 = t[:2]

            t2['a'] = 0  # in-place slice update
            assert np.all(t['a'] == [0, 0, 3])
            assert len(w) == 0

            t2['a'] = [10, 20]  # replace slice
            assert len(w) == 1
            assert "replaced column 'a' which looks like an array slice" in str(w[0].message)

def test_replace_update_column_via_setitem_warnings_attributes():
    """
    Test warnings related to table replace change in #5556:
    Lost attributes.
    """
    t = table.Table([[1, 2, 3], [4, 5, 6]], names=['a', 'b'])
    t['a'].unit = 'm'

    with catch_warnings() as w:
        with table.conf.set_temp('replace_warnings',
                                 ['refcount', 'attributes', 'slice']):
            t['a'] = [10, 20, 30]
        assert len(w) == 1
        assert "replaced column 'a' and column attributes ['unit']" in str(w[0].message)

def test_replace_update_column_via_setitem_warnings_refcount():
    """
    Test warnings related to table replace change in #5556:
    Reference count changes.
    """
    t = table.Table([[1, 2, 3], [4, 5, 6]], names=['a', 'b'])
    ta = t['a']  # Generate an extra reference to original column

    with catch_warnings() as w:
        with table.conf.set_temp('replace_warnings',
                                 ['refcount', 'attributes', 'slice']):
            t['a'] = [10, 20, 30]
        assert len(w) == 1
        assert "replaced column 'a' and the number of references" in str(w[0].message)

def test_replace_update_column_via_setitem_warnings_always():
    """
    Test warnings related to table replace change in #5556:
    Test 'always' setting that raises warning for any replace.
    """
    t = table.Table([[1, 2, 3], [4, 5, 6]], names=['a', 'b'])

    with catch_warnings() as w:
        with table.conf.set_temp('replace_warnings', ['always']):
            t['a'] = 0  # in-place slice update
            assert len(w) == 0

            from inspect import currentframe, getframeinfo
            frameinfo = getframeinfo(currentframe())
            t['a'] = [10, 20, 30]  # replace column
            assert len(w) == 1
            assert "replaced column 'a'" == str(w[0].message)

            # Make sure the warning points back to the user code line
            assert w[0].lineno == frameinfo.lineno + 1
            assert w[0].category is table.TableReplaceWarning
            assert 'test_table' in w[0].filename

def test_replace_update_column_via_setitem_replace_inplace():
    """
    Test the replace_inplace config option related to #5556.  In this
    case no replace is done.
    """
    t = table.Table([[1, 2, 3], [4, 5, 6]], names=['a', 'b'])
    ta = t['a']
    t['a'].unit = 'm'

    with catch_warnings() as w:
        with table.conf.set_temp('replace_inplace', True):
            with table.conf.set_temp('replace_warnings',
                                     ['always', 'refcount', 'attributes', 'slice']):
                t['a'] = 0  # in-place update
                assert len(w) == 0
                assert ta is t['a']

                t['a'] = [10, 20, 30]  # normally replaces column, but not now
                assert len(w) == 0
                assert ta is t['a']
                assert np.all(t['a'] == [10, 20, 30])

def test_primary_key_is_inherited():
    """Test whether a new Table inherits the primary_key attribute from
    its parent Table. Issue #4672"""

    t = table.Table([(2, 3, 2, 1), (8, 7, 6, 5)], names=('a', 'b'))
    t.add_index('a')
    original_key = t.primary_key

    # can't test if tuples are equal, so just check content
    assert original_key[0] is 'a'

    t2 = t[:]
    t3 = t.copy()
    t4 = table.Table(t)

    # test whether the reference is the same in the following
    assert original_key == t2.primary_key
    assert original_key == t3.primary_key
    assert original_key == t4.primary_key

    # just test one element, assume rest are equal if assert passes
    assert t.loc[1] == t2.loc[1]
    assert t.loc[1] == t3.loc[1]
    assert t.loc[1] == t4.loc[1]

def test_qtable_read_for_ipac_table_with_char_columns():
    '''Test that a char column of a QTable is assigned no unit and not
    a dimensionless unit, otherwise conversion of reader output to
    QTable fails.'''
    t1 = table.QTable([["A"]], names="B")
    out = StringIO()
    t1.write(out, format="ascii.ipac")
    t2 = table.QTable.read(out.getvalue(), format="ascii.ipac", guess=False)
    assert t2["B"].unit is None