# -*- coding: utf-8 -*-

try:
    import json
except ImportError:
    import simplejson as json
import math
import pytz
import pytest
import time
import datetime
import calendar
import re
import decimal
import dateutil
from functools import partial
from pandas.compat import range, zip, StringIO, u
import pandas._libs.json as ujson
import pandas.compat as compat

import numpy as np
from pandas import DataFrame, Series, Index, NaT, DatetimeIndex
import pandas.util.testing as tm

json_unicode = (json.dumps if compat.PY3
                else partial(json.dumps, encoding="utf-8"))

class TestUltraJSONTests(object):

    @pytest.mark.skipif(compat.is_platform_32bit(),
                        reason="not compliant on 32-bit, xref #15865")
    def test_encodeDecimal(self):
        sut = decimal.Decimal("1337.1337")
        encoded = ujson.encode(sut, double_precision=15)
        decoded = ujson.decode(encoded)
        assert decoded == 1337.1337

        sut = decimal.Decimal("0.95")
        encoded = ujson.encode(sut, double_precision=1)
        assert encoded == "1.0"
        decoded = ujson.decode(encoded)
        assert decoded == 1.0

        sut = decimal.Decimal("0.94")
        encoded = ujson.encode(sut, double_precision=1)
        assert encoded == "0.9"
        decoded = ujson.decode(encoded)
        assert decoded == 0.9

        sut = decimal.Decimal("1.95")
        encoded = ujson.encode(sut, double_precision=1)
        assert encoded == "2.0"
        decoded = ujson.decode(encoded)
        assert decoded == 2.0

        sut = decimal.Decimal("-1.95")
        encoded = ujson.encode(sut, double_precision=1)
        assert encoded == "-2.0"
        decoded = ujson.decode(encoded)
        assert decoded == -2.0

        sut = decimal.Decimal("0.995")
        encoded = ujson.encode(sut, double_precision=2)
        assert encoded == "1.0"
        decoded = ujson.decode(encoded)
        assert decoded == 1.0

        sut = decimal.Decimal("0.9995")
        encoded = ujson.encode(sut, double_precision=3)
        assert encoded == "1.0"
        decoded = ujson.decode(encoded)
        assert decoded == 1.0

        sut = decimal.Decimal("0.99999999999999944")
        encoded = ujson.encode(sut, double_precision=15)
        assert encoded == "1.0"
        decoded = ujson.decode(encoded)
        assert decoded == 1.0

    def test_encodeStringConversion(self):
        input = "A string \\ / \b \f \n \r \t </script> &"
        not_html_encoded = ('"A string \\\\ \\/ \\b \\f \\n '
                            '\\r \\t <\\/script> &"')
        html_encoded = ('"A string \\\\ \\/ \\b \\f \\n \\r \\t '
                        '\\u003c\\/script\\u003e \\u0026"')

        def helper(expected_output, **encode_kwargs):
            output = ujson.encode(input, **encode_kwargs)
            assert input == json.loads(output)
            assert output == expected_output
            assert input == ujson.decode(output)

        # Default behavior assumes encode_html_chars=False.
        helper(not_html_encoded, ensure_ascii=True)
        helper(not_html_encoded, ensure_ascii=False)

        # Make sure explicit encode_html_chars=False works.
        helper(not_html_encoded, ensure_ascii=True, encode_html_chars=False)
        helper(not_html_encoded, ensure_ascii=False, encode_html_chars=False)

        # Make sure explicit encode_html_chars=True does the encoding.
        helper(html_encoded, ensure_ascii=True, encode_html_chars=True)
        helper(html_encoded, ensure_ascii=False, encode_html_chars=True)

    def test_doubleLongIssue(self):
        sut = {u('a'): -4342969734183514}
        encoded = json.dumps(sut)
        decoded = json.loads(encoded)
        assert sut == decoded
        encoded = ujson.encode(sut, double_precision=15)
        decoded = ujson.decode(encoded)
        assert sut == decoded

    def test_doubleLongDecimalIssue(self):
        sut = {u('a'): -12345678901234.56789012}
        encoded = json.dumps(sut)
        decoded = json.loads(encoded)
        assert sut == decoded
        encoded = ujson.encode(sut, double_precision=15)
        decoded = ujson.decode(encoded)
        assert sut == decoded

    def test_encodeNonCLocale(self):
        import locale
        savedlocale = locale.getlocale(locale.LC_NUMERIC)
        try:
            locale.setlocale(locale.LC_NUMERIC, 'it_IT.UTF-8')
        except:
            try:
                locale.setlocale(locale.LC_NUMERIC, 'Italian_Italy')
            except:
                pytest.skip('Could not set locale for testing')
        assert ujson.loads(ujson.dumps(4.78e60)) == 4.78e60
        assert ujson.loads('4.78', precise_float=True) == 4.78
        locale.setlocale(locale.LC_NUMERIC, savedlocale)

    def test_encodeDecodeLongDecimal(self):
        sut = {u('a'): -528656961.4399388}
        encoded = ujson.dumps(sut, double_precision=15)
        ujson.decode(encoded)

    def test_decimalDecodeTestPrecise(self):
        sut = {u('a'): 4.56}
        encoded = ujson.encode(sut)
        decoded = ujson.decode(encoded, precise_float=True)
        assert sut == decoded

    @pytest.mark.skipif(compat.is_platform_windows() and not compat.PY3,
                        reason="buggy on win-64 for py2")
    def test_encodeDoubleTinyExponential(self):
        num = 1e-40
        assert num == ujson.decode(ujson.encode(num))
        num = 1e-100
        assert num == ujson.decode(ujson.encode(num))
        num = -1e-45
        assert num == ujson.decode(ujson.encode(num))
        num = -1e-145
        assert np.allclose(num, ujson.decode(ujson.encode(num)))

    def test_encodeDictWithUnicodeKeys(self):
        input = {u("key1"): u("value1"), u("key1"):
                 u("value1"), u("key1"): u("value1"),
                 u("key1"): u("value1"), u("key1"):
                 u("value1"), u("key1"): u("value1")}
        output = ujson.encode(input)

        input = {u("بن"): u("value1"), u("بن"): u("value1"),
                 u("بن"): u("value1"), u("بن"): u("value1"),
                 u("بن"): u("value1"), u("بن"): u("value1"),
                 u("بن"): u("value1")}
        output = ujson.encode(input)  # noqa

    def test_encodeDoubleConversion(self):
        input = math.pi
        output = ujson.encode(input)
        assert round(input, 5) == round(json.loads(output), 5)
        assert round(input, 5) == round(ujson.decode(output), 5)

    def test_encodeWithDecimal(self):
        input = 1.0
        output = ujson.encode(input)
        assert output == "1.0"

    def test_encodeDoubleNegConversion(self):
        input = -math.pi
        output = ujson.encode(input)

        assert round(input, 5) == round(json.loads(output), 5)
        assert round(input, 5) == round(ujson.decode(output), 5)

    def test_encodeArrayOfNestedArrays(self):
        input = [[[[]]]] * 20
        output = ujson.encode(input)
        assert input == json.loads(output)
        # assert output == json.dumps(input)
        assert input == ujson.decode(output)
        input = np.array(input)
        tm.assert_numpy_array_equal(input, ujson.decode(
            output, numpy=True, dtype=input.dtype))

    def test_encodeArrayOfDoubles(self):
        input = [31337.31337, 31337.31337, 31337.31337, 31337.31337] * 10
        output = ujson.encode(input)
        assert input == json.loads(output)
        # assert output == json.dumps(input)
        assert input == ujson.decode(output)
        tm.assert_numpy_array_equal(
            np.array(input), ujson.decode(output, numpy=True))

    def test_doublePrecisionTest(self):
        input = 30.012345678901234
        output = ujson.encode(input, double_precision=15)
        assert input == json.loads(output)
        assert input == ujson.decode(output)

        output = ujson.encode(input, double_precision=9)
        assert round(input, 9) == json.loads(output)
        assert round(input, 9) == ujson.decode(output)

        output = ujson.encode(input, double_precision=3)
        assert round(input, 3) == json.loads(output)
        assert round(input, 3) == ujson.decode(output)

    def test_invalidDoublePrecision(self):
        input = 30.12345678901234567890

        pytest.raises(ValueError, ujson.encode, input, double_precision=20)
        pytest.raises(ValueError, ujson.encode, input, double_precision=-1)

        # will throw typeError
        pytest.raises(TypeError, ujson.encode, input, double_precision='9')
        # will throw typeError
        pytest.raises(TypeError, ujson.encode,
                      input, double_precision=None)

    def test_encodeStringConversion2(self):
        input = "A string \\ / \b \f \n \r \t"
        output = ujson.encode(input)
        assert input == json.loads(output)
        assert output == '"A string \\\\ \\/ \\b \\f \\n \\r \\t"'
        assert input == ujson.decode(output)
        pass

    def test_decodeUnicodeConversion(self):
        pass

    def test_encodeUnicodeConversion1(self):
        input = "Räksmörgås اسامة بن محمد بن عوض بن لادن"
        enc = ujson.encode(input)
        dec = ujson.decode(enc)
        assert enc == json_unicode(input)
        assert dec == json.loads(enc)

    def test_encodeControlEscaping(self):
        input = "\x19"
        enc = ujson.encode(input)
        dec = ujson.decode(enc)
        assert input == dec
        assert enc == json_unicode(input)

    def test_encodeUnicodeConversion2(self):
        input = "\xe6\x97\xa5\xd1\x88"
        enc = ujson.encode(input)
        dec = ujson.decode(enc)
        assert enc == json_unicode(input)
        assert dec == json.loads(enc)

    def test_encodeUnicodeSurrogatePair(self):
        input = "\xf0\x90\x8d\x86"
        enc = ujson.encode(input)
        dec = ujson.decode(enc)

        assert enc == json_unicode(input)
        assert dec == json.loads(enc)

    def test_encodeUnicode4BytesUTF8(self):
        input = "\xf0\x91\x80\xb0TRAILINGNORMAL"
        enc = ujson.encode(input)
        dec = ujson.decode(enc)

        assert enc == json_unicode(input)
        assert dec == json.loads(enc)

    def test_encodeUnicode4BytesUTF8Highest(self):
        input = "\xf3\xbf\xbf\xbfTRAILINGNORMAL"
        enc = ujson.encode(input)

        dec = ujson.decode(enc)

        assert enc == json_unicode(input)
        assert dec == json.loads(enc)

    def test_encodeArrayInArray(self):
        input = [[[[]]]]
        output = ujson.encode(input)

        assert input == json.loads(output)
        assert output == json.dumps(input)
        assert input == ujson.decode(output)
        tm.assert_numpy_array_equal(
            np.array(input), ujson.decode(output, numpy=True))
        pass

    def test_encodeIntConversion(self):
        input = 31337
        output = ujson.encode(input)
        assert input == json.loads(output)
        assert output == json.dumps(input)
        assert input == ujson.decode(output)
        pass

    def test_encodeIntNegConversion(self):
        input = -31337
        output = ujson.encode(input)
        assert input == json.loads(output)
        assert output == json.dumps(input)
        assert input == ujson.decode(output)
        pass

    def test_encodeLongNegConversion(self):
        input = -9223372036854775808
        output = ujson.encode(input)

        assert input == json.loads(output)
        assert output == json.dumps(input)
        assert input == ujson.decode(output)

    def test_encodeListConversion(self):
        input = [1, 2, 3, 4]
        output = ujson.encode(input)
        assert input == json.loads(output)
        assert input == ujson.decode(output)
        tm.assert_numpy_array_equal(
            np.array(input), ujson.decode(output, numpy=True))
        pass

    def test_encodeDictConversion(self):
        input = {"k1": 1, "k2": 2, "k3": 3, "k4": 4}
        output = ujson.encode(input)  # noqa
        assert input == json.loads(output)
        assert input == ujson.decode(output)
        assert input == ujson.decode(output)
        pass

    def test_encodeNoneConversion(self):
        input = None
        output = ujson.encode(input)
        assert input == json.loads(output)
        assert output == json.dumps(input)
        assert input == ujson.decode(output)
        pass

    def test_encodeTrueConversion(self):
        input = True
        output = ujson.encode(input)
        assert input == json.loads(output)
        assert output == json.dumps(input)
        assert input == ujson.decode(output)
        pass

    def test_encodeFalseConversion(self):
        input = False
        output = ujson.encode(input)
        assert input == json.loads(output)
        assert output == json.dumps(input)
        assert input == ujson.decode(output)

    def test_encodeDatetimeConversion(self):
        ts = time.time()
        input = datetime.datetime.fromtimestamp(ts)
        output = ujson.encode(input, date_unit='s')
        expected = calendar.timegm(input.utctimetuple())
        assert int(expected) == json.loads(output)
        assert int(expected) == ujson.decode(output)

    def test_encodeDateConversion(self):
        ts = time.time()
        input = datetime.date.fromtimestamp(ts)

        output = ujson.encode(input, date_unit='s')
        tup = (input.year, input.month, input.day, 0, 0, 0)

        expected = calendar.timegm(tup)
        assert int(expected) == json.loads(output)
        assert int(expected) == ujson.decode(output)

    def test_encodeTimeConversion(self):
        tests = [
            datetime.time(),
            datetime.time(1, 2, 3),
            datetime.time(10, 12, 15, 343243),
        ]
        for test in tests:
            output = ujson.encode(test)
            expected = '"{iso}"'.format(iso=test.isoformat())
            assert expected == output

    def test_encodeTimeConversion_pytz(self):
        # see gh-11473: to_json segfaults with timezone-aware datetimes
        test = datetime.time(10, 12, 15, 343243, pytz.utc)
        output = ujson.encode(test)
        expected = '"{iso}"'.format(iso=test.isoformat())
        assert expected == output

    def test_encodeTimeConversion_dateutil(self):
        # see gh-11473: to_json segfaults with timezone-aware datetimes
        test = datetime.time(10, 12, 15, 343243, dateutil.tz.tzutc())
        output = ujson.encode(test)
        expected = '"{iso}"'.format(iso=test.isoformat())
        assert expected == output

    def test_nat(self):
        input = NaT
        assert ujson.encode(input) == 'null', "Expected null"

    def test_npy_nat(self):
        from distutils.version import LooseVersion
        if LooseVersion(np.__version__) < LooseVersion('1.7.0'):
            pytest.skip("numpy version < 1.7.0, is "
                        "{0}".format(np.__version__))

        input = np.datetime64('NaT')
        assert ujson.encode(input) == 'null', "Expected null"

    def test_datetime_units(self):
        from pandas._libs.lib import Timestamp

        val = datetime.datetime(2013, 8, 17, 21, 17, 12, 215504)
        stamp = Timestamp(val)

        roundtrip = ujson.decode(ujson.encode(val, date_unit='s'))
        assert roundtrip == stamp.value // 10**9

        roundtrip = ujson.decode(ujson.encode(val, date_unit='ms'))
        assert roundtrip == stamp.value // 10**6

        roundtrip = ujson.decode(ujson.encode(val, date_unit='us'))
        assert roundtrip == stamp.value // 10**3

        roundtrip = ujson.decode(ujson.encode(val, date_unit='ns'))
        assert roundtrip == stamp.value

        pytest.raises(ValueError, ujson.encode, val, date_unit='foo')

    def test_encodeToUTF8(self):
        input = "\xe6\x97\xa5\xd1\x88"
        enc = ujson.encode(input, ensure_ascii=False)
        dec = ujson.decode(enc)
        assert enc == json_unicode(input, ensure_ascii=False)
        assert dec == json.loads(enc)

    def test_decodeFromUnicode(self):
        input = u("{\"obj\": 31337}")
        dec1 = ujson.decode(input)
        dec2 = ujson.decode(str(input))
        assert dec1 == dec2

    def test_encodeRecursionMax(self):
        # 8 is the max recursion depth

        class O2:
            member = 0
            pass

        class O1:
            member = 0
            pass

        input = O1()
        input.member = O2()
        input.member.member = input

        try:
            output = ujson.encode(input)  # noqa
            assert False, "Expected overflow exception"
        except(OverflowError):
            pass

    def test_encodeDoubleNan(self):
        input = np.nan
        assert ujson.encode(input) == 'null', "Expected null"

    def test_encodeDoubleInf(self):
        input = np.inf
        assert ujson.encode(input) == 'null', "Expected null"

    def test_encodeDoubleNegInf(self):
        input = -np.inf
        assert ujson.encode(input) == 'null', "Expected null"

    def test_decodeJibberish(self):
        input = "fdsa sda v9sa fdsa"
        try:
            ujson.decode(input)
            assert False, "Expected exception!"
        except(ValueError):
            return
        assert False, "Wrong exception"

    def test_decodeBrokenArrayStart(self):
        input = "["
        try:
            ujson.decode(input)
            assert False, "Expected exception!"
        except(ValueError):
            return
        assert False, "Wrong exception"

    def test_decodeBrokenObjectStart(self):
        input = "{"
        try:
            ujson.decode(input)
            assert False, "Expected exception!"
        except(ValueError):
            return
        assert False, "Wrong exception"

    def test_decodeBrokenArrayEnd(self):
        input = "]"
        try:
            ujson.decode(input)
            assert False, "Expected exception!"
        except(ValueError):
            return
        assert False, "Wrong exception"

    def test_decodeArrayDepthTooBig(self):
        input = '[' * (1024 * 1024)
        try:
            ujson.decode(input)
            assert False, "Expected exception!"
        except(ValueError):
            return
        assert False, "Wrong exception"

    def test_decodeBrokenObjectEnd(self):
        input = "}"
        try:
            ujson.decode(input)
            assert False, "Expected exception!"
        except(ValueError):
            return
        assert False, "Wrong exception"

    def test_decodeObjectDepthTooBig(self):
        input = '{' * (1024 * 1024)
        try:
            ujson.decode(input)
            assert False, "Expected exception!"
        except(ValueError):
            return
        assert False, "Wrong exception"

    def test_decodeStringUnterminated(self):
        input = "\"TESTING"
        try:
            ujson.decode(input)
            assert False, "Expected exception!"
        except(ValueError):
            return
        assert False, "Wrong exception"

    def test_decodeStringUntermEscapeSequence(self):
        input = "\"TESTING\\\""
        try:
            ujson.decode(input)
            assert False, "Expected exception!"
        except(ValueError):
            return
        assert False, "Wrong exception"

    def test_decodeStringBadEscape(self):
        input = "\"TESTING\\\""
        try:
            ujson.decode(input)
            assert False, "Expected exception!"
        except(ValueError):
            return
        assert False, "Wrong exception"

    def test_decodeTrueBroken(self):
        input = "tru"
        try:
            ujson.decode(input)
            assert False, "Expected exception!"
        except(ValueError):
            return
        assert False, "Wrong exception"

    def test_decodeFalseBroken(self):
        input = "fa"
        try:
            ujson.decode(input)
            assert False, "Expected exception!"
        except(ValueError):
            return
        assert False, "Wrong exception"

    def test_decodeNullBroken(self):
        input = "n"
        try:
            ujson.decode(input)
            assert False, "Expected exception!"
        except(ValueError):
            return
        assert False, "Wrong exception"

    def test_decodeBrokenDictKeyTypeLeakTest(self):
        input = '{{1337:""}}'
        for x in range(1000):
            try:
                ujson.decode(input)
                assert False, "Expected exception!"
            except ValueError:
                continue

            assert False, "Wrong exception"

    def test_decodeBrokenDictLeakTest(self):
        input = '{{"key":"}'
        for x in range(1000):
            try:
                ujson.decode(input)
                assert False, "Expected exception!"
            except(ValueError):
                continue

            assert False, "Wrong exception"

    def test_decodeBrokenListLeakTest(self):
        input = '[[[true'
        for x in range(1000):
            try:
                ujson.decode(input)
                assert False, "Expected exception!"
            except(ValueError):
                continue

            assert False, "Wrong exception"

    def test_decodeDictWithNoKey(self):
        input = "{{{{31337}}}}"
        try:
            ujson.decode(input)
            assert False, "Expected exception!"
        except(ValueError):
            return

        assert False, "Wrong exception"

    def test_decodeDictWithNoColonOrValue(self):
        input = "{{{{\"key\"}}}}"
        try:
            ujson.decode(input)
            assert False, "Expected exception!"
        except(ValueError):
            return

        assert False, "Wrong exception"

    def test_decodeDictWithNoValue(self):
        input = "{{{{\"key\":}}}}"
        try:
            ujson.decode(input)
            assert False, "Expected exception!"
        except(ValueError):
            return

        assert False, "Wrong exception"

    def test_decodeNumericIntPos(self):
        input = "31337"
        assert 31337 == ujson.decode(input)

    def test_decodeNumericIntNeg(self):
        input = "-31337"
        assert -31337 == ujson.decode(input)

    @pytest.mark.skipif(compat.PY3, reason="only PY2")
    def test_encodeUnicode4BytesUTF8Fail(self):
        input = "\xfd\xbf\xbf\xbf\xbf\xbf"
        try:
            enc = ujson.encode(input)  # noqa
            assert False, "Expected exception"
        except OverflowError:
            pass

    def test_encodeNullCharacter(self):
        input = "31337 \x00 1337"
        output = ujson.encode(input)
        assert input == json.loads(output)
        assert output == json.dumps(input)
        assert input == ujson.decode(output)

        input = "\x00"
        output = ujson.encode(input)
        assert input == json.loads(output)
        assert output == json.dumps(input)
        assert input == ujson.decode(output)

        assert '"  \\u0000\\r\\n "' == ujson.dumps(u("  \u0000\r\n "))
        pass

    def test_decodeNullCharacter(self):
        input = "\"31337 \\u0000 31337\""
        assert ujson.decode(input) == json.loads(input)

    def test_encodeListLongConversion(self):
        input = [9223372036854775807, 9223372036854775807, 9223372036854775807,
                 9223372036854775807, 9223372036854775807, 9223372036854775807]
        output = ujson.encode(input)
        assert input == json.loads(output)
        assert input == ujson.decode(output)
        tm.assert_numpy_array_equal(np.array(input),
                                    ujson.decode(output, numpy=True,
                                                 dtype=np.int64))
        pass

    def test_encodeLongConversion(self):
        input = 9223372036854775807
        output = ujson.encode(input)
        assert input == json.loads(output)
        assert output == json.dumps(input)
        assert input == ujson.decode(output)
        pass

    def test_numericIntExp(self):
        input = "1337E40"
        output = ujson.decode(input)
        assert output == json.loads(input)

    def test_numericIntFrcExp(self):
        input = "1.337E40"
        output = ujson.decode(input)
        tm.assert_almost_equal(output, json.loads(input))

    def test_decodeNumericIntExpEPLUS(self):
        input = "1337E+9"
        output = ujson.decode(input)
        tm.assert_almost_equal(output, json.loads(input))

    def test_decodeNumericIntExpePLUS(self):
        input = "1.337e+40"
        output = ujson.decode(input)
        tm.assert_almost_equal(output, json.loads(input))

    def test_decodeNumericIntExpE(self):
        input = "1337E40"
        output = ujson.decode(input)
        tm.assert_almost_equal(output, json.loads(input))

    def test_decodeNumericIntExpe(self):
        input = "1337e40"
        output = ujson.decode(input)
        tm.assert_almost_equal(output, json.loads(input))

    def test_decodeNumericIntExpEMinus(self):
        input = "1.337E-4"
        output = ujson.decode(input)
        tm.assert_almost_equal(output, json.loads(input))

    def test_decodeNumericIntExpeMinus(self):
        input = "1.337e-4"
        output = ujson.decode(input)
        tm.assert_almost_equal(output, json.loads(input))

    def test_dumpToFile(self):
        f = StringIO()
        ujson.dump([1, 2, 3], f)
        assert "[1,2,3]" == f.getvalue()

    def test_dumpToFileLikeObject(self):
        class filelike:

            def __init__(self):
                self.bytes = ''

            def write(self, bytes):
                self.bytes += bytes
        f = filelike()
        ujson.dump([1, 2, 3], f)
        assert "[1,2,3]" == f.bytes

    def test_dumpFileArgsError(self):
        try:
            ujson.dump([], '')
        except TypeError:
            pass
        else:
            assert False, 'expected TypeError'

    def test_loadFile(self):
        f = StringIO("[1,2,3,4]")
        assert [1, 2, 3, 4] == ujson.load(f)

        f = StringIO("[1,2,3,4]")
        tm.assert_numpy_array_equal(
            np.array([1, 2, 3, 4]), ujson.load(f, numpy=True))

    def test_loadFileLikeObject(self):
        class filelike:

            def read(self):
                try:
                    self.end
                except AttributeError:
                    self.end = True
                    return "[1,2,3,4]"
        f = filelike()
        assert [1, 2, 3, 4] == ujson.load(f)

        f = filelike()
        tm.assert_numpy_array_equal(
            np.array([1, 2, 3, 4]), ujson.load(f, numpy=True))

    def test_loadFileArgsError(self):
        try:
            ujson.load("[]")
        except TypeError:
            pass
        else:
            assert False, "expected TypeError"

    def test_version(self):
        assert re.match(r'^\d+\.\d+(\.\d+)?$', ujson.__version__), \
            "ujson.__version__ must be a string like '1.4.0'"

    def test_encodeNumericOverflow(self):
        try:
            ujson.encode(12839128391289382193812939)
        except OverflowError:
            pass
        else:
            assert False, "expected OverflowError"

    def test_encodeNumericOverflowNested(self):
        for n in range(0, 100):
            class Nested:
                x = 12839128391289382193812939

            nested = Nested()

            try:
                ujson.encode(nested)
            except OverflowError:
                pass
            else:
                assert False, "expected OverflowError"

    def test_decodeNumberWith32bitSignBit(self):
        # Test that numbers that fit within 32 bits but would have the
        # sign bit set (2**31 <= x < 2**32) are decoded properly.
        boundary1 = 2**31  # noqa
        boundary2 = 2**32  # noqa
        docs = (
            '{"id": 3590016419}',
            '{{"id": {low}}}'.format(low=2**31),
            '{{"id": {high}}}'.format(high=2**32),
            '{{"id": {one_less}}}'.format(one_less=(2**32) - 1),
        )
        results = (3590016419, 2**31, 2**32, 2**32 - 1)
        for doc, result in zip(docs, results):
            assert ujson.decode(doc)['id'] == result

    def test_encodeBigEscape(self):
        for x in range(10):
            if compat.PY3:
                base = '\u00e5'.encode('utf-8')
            else:
                base = "\xc3\xa5"
            input = base * 1024 * 1024 * 2
            output = ujson.encode(input)  # noqa

    def test_decodeBigEscape(self):
        for x in range(10):
            if compat.PY3:
                base = '\u00e5'.encode('utf-8')
            else:
                base = "\xc3\xa5"
            quote = compat.str_to_bytes("\"")
            input = quote + (base * 1024 * 1024 * 2) + quote
            output = ujson.decode(input)  # noqa

    def test_toDict(self):
        d = {u("key"): 31337}

        class DictTest:

            def toDict(self):
                return d

        o = DictTest()
        output = ujson.encode(o)
        dec = ujson.decode(output)
        assert dec == d

    def test_defaultHandler(self):

        class _TestObject(object):

            def __init__(self, val):
                self.val = val

            @property
            def recursive_attr(self):
                return _TestObject("recursive_attr")

            def __str__(self):
                return str(self.val)

        pytest.raises(OverflowError, ujson.encode, _TestObject("foo"))
        assert '"foo"' == ujson.encode(_TestObject("foo"),
                                       default_handler=str)

        def my_handler(obj):
            return "foobar"

        assert '"foobar"' == ujson.encode(_TestObject("foo"),
                                          default_handler=my_handler)

        def my_handler_raises(obj):
            raise TypeError("I raise for anything")

        with tm.assert_raises_regex(TypeError, "I raise for anything"):
            ujson.encode(_TestObject("foo"), default_handler=my_handler_raises)

        def my_int_handler(obj):
            return 42

        assert ujson.decode(ujson.encode(
            _TestObject("foo"), default_handler=my_int_handler)) == 42

        def my_obj_handler(obj):
            return datetime.datetime(2013, 2, 3)

        assert (ujson.decode(ujson.encode(datetime.datetime(2013, 2, 3))) ==
                ujson.decode(ujson.encode(_TestObject("foo"),
                                          default_handler=my_obj_handler)))

        l = [_TestObject("foo"), _TestObject("bar")]
        assert (json.loads(json.dumps(l, default=str)) ==
                ujson.decode(ujson.encode(l, default_handler=str)))

class TestNumpyJSONTests(object):

    def test_Bool(self):
        b = np.bool(True)
        assert ujson.decode(ujson.encode(b)) == b

    def test_BoolArray(self):
        inpt = np.array([True, False, True, True, False, True, False, False],
                        dtype=np.bool)
        outp = np.array(ujson.decode(ujson.encode(inpt)), dtype=np.bool)
        tm.assert_numpy_array_equal(inpt, outp)

    def test_Int(self):
        num = np.int(2562010)
        assert np.int(ujson.decode(ujson.encode(num))) == num

        num = np.int8(127)
        assert np.int8(ujson.decode(ujson.encode(num))) == num

        num = np.int16(2562010)
        assert np.int16(ujson.decode(ujson.encode(num))) == num

        num = np.int32(2562010)
        assert np.int32(ujson.decode(ujson.encode(num))) == num

        num = np.int64(2562010)
        assert np.int64(ujson.decode(ujson.encode(num))) == num

        num = np.uint8(255)
        assert np.uint8(ujson.decode(ujson.encode(num))) == num

        num = np.uint16(2562010)
        assert np.uint16(ujson.decode(ujson.encode(num))) == num

        num = np.uint32(2562010)
        assert np.uint32(ujson.decode(ujson.encode(num))) == num

        num = np.uint64(2562010)
        assert np.uint64(ujson.decode(ujson.encode(num))) == num

    def test_IntArray(self):
        arr = np.arange(100, dtype=np.int)
        dtypes = (np.int, np.int8, np.int16, np.int32, np.int64,
                  np.uint, np.uint8, np.uint16, np.uint32, np.uint64)
        for dtype in dtypes:
            inpt = arr.astype(dtype)
            outp = np.array(ujson.decode(ujson.encode(inpt)), dtype=dtype)
            tm.assert_numpy_array_equal(inpt, outp)

    def test_IntMax(self):
        num = np.int(np.iinfo(np.int).max)
        assert np.int(ujson.decode(ujson.encode(num))) == num

        num = np.int8(np.iinfo(np.int8).max)
        assert np.int8(ujson.decode(ujson.encode(num))) == num

        num = np.int16(np.iinfo(np.int16).max)
        assert np.int16(ujson.decode(ujson.encode(num))) == num

        num = np.int32(np.iinfo(np.int32).max)
        assert np.int32(ujson.decode(ujson.encode(num))) == num

        num = np.uint8(np.iinfo(np.uint8).max)
        assert np.uint8(ujson.decode(ujson.encode(num))) == num

        num = np.uint16(np.iinfo(np.uint16).max)
        assert np.uint16(ujson.decode(ujson.encode(num))) == num

        num = np.uint32(np.iinfo(np.uint32).max)
        assert np.uint32(ujson.decode(ujson.encode(num))) == num

        if not compat.is_platform_32bit():
            num = np.int64(np.iinfo(np.int64).max)
            assert np.int64(ujson.decode(ujson.encode(num))) == num

            # uint64 max will always overflow as it's encoded to signed
            num = np.uint64(np.iinfo(np.int64).max)
            assert np.uint64(ujson.decode(ujson.encode(num))) == num

    def test_Float(self):
        num = np.float(256.2013)
        assert np.float(ujson.decode(ujson.encode(num))) == num

        num = np.float32(256.2013)
        assert np.float32(ujson.decode(ujson.encode(num))) == num

        num = np.float64(256.2013)
        assert np.float64(ujson.decode(ujson.encode(num))) == num

    def test_FloatArray(self):
        arr = np.arange(12.5, 185.72, 1.7322, dtype=np.float)
        dtypes = (np.float, np.float32, np.float64)

        for dtype in dtypes:
            inpt = arr.astype(dtype)
            outp = np.array(ujson.decode(ujson.encode(
                inpt, double_precision=15)), dtype=dtype)
            tm.assert_almost_equal(inpt, outp)

    def test_FloatMax(self):
        num = np.float(np.finfo(np.float).max / 10)
        tm.assert_almost_equal(np.float(ujson.decode(
            ujson.encode(num, double_precision=15))), num, 15)

        num = np.float32(np.finfo(np.float32).max / 10)
        tm.assert_almost_equal(np.float32(ujson.decode(
            ujson.encode(num, double_precision=15))), num, 15)

        num = np.float64(np.finfo(np.float64).max / 10)
        tm.assert_almost_equal(np.float64(ujson.decode(
            ujson.encode(num, double_precision=15))), num, 15)

    def test_Arrays(self):
        arr = np.arange(100)

        arr = arr.reshape((10, 10))
        tm.assert_numpy_array_equal(
            np.array(ujson.decode(ujson.encode(arr))), arr)
        tm.assert_numpy_array_equal(ujson.decode(
            ujson.encode(arr), numpy=True), arr)

        arr = arr.reshape((5, 5, 4))
        tm.assert_numpy_array_equal(
            np.array(ujson.decode(ujson.encode(arr))), arr)
        tm.assert_numpy_array_equal(ujson.decode(
            ujson.encode(arr), numpy=True), arr)

        arr = arr.reshape((100, 1))
        tm.assert_numpy_array_equal(
            np.array(ujson.decode(ujson.encode(arr))), arr)
        tm.assert_numpy_array_equal(ujson.decode(
            ujson.encode(arr), numpy=True), arr)

        arr = np.arange(96)
        arr = arr.reshape((2, 2, 2, 2, 3, 2))
        tm.assert_numpy_array_equal(
            np.array(ujson.decode(ujson.encode(arr))), arr)
        tm.assert_numpy_array_equal(ujson.decode(
            ujson.encode(arr), numpy=True), arr)

        l = ['a', list(), dict(), dict(), list(),
             42, 97.8, ['a', 'b'], {'key': 'val'}]
        arr = np.array(l)
        tm.assert_numpy_array_equal(
            np.array(ujson.decode(ujson.encode(arr))), arr)

        arr = np.arange(100.202, 200.202, 1, dtype=np.float32)
        arr = arr.reshape((5, 5, 4))
        outp = np.array(ujson.decode(ujson.encode(arr)), dtype=np.float32)
        tm.assert_almost_equal(arr, outp)
        outp = ujson.decode(ujson.encode(arr), numpy=True, dtype=np.float32)
        tm.assert_almost_equal(arr, outp)

    def test_OdArray(self):
        def will_raise():
            ujson.encode(np.array(1))

        pytest.raises(TypeError, will_raise)

    def test_ArrayNumpyExcept(self):

        input = ujson.dumps([42, {}, 'a'])
        try:
            ujson.decode(input, numpy=True)
            assert False, "Expected exception!"
        except(TypeError):
            pass
        except:
            assert False, "Wrong exception"

        input = ujson.dumps(['a', 'b', [], 'c'])
        try:
            ujson.decode(input, numpy=True)
            assert False, "Expected exception!"
        except(ValueError):
            pass
        except:
            assert False, "Wrong exception"

        input = ujson.dumps([['a'], 42])
        try:
            ujson.decode(input, numpy=True)
            assert False, "Expected exception!"
        except(ValueError):
            pass
        except:
            assert False, "Wrong exception"

        input = ujson.dumps([42, ['a'], 42])
        try:
            ujson.decode(input, numpy=True)
            assert False, "Expected exception!"
        except(ValueError):
            pass
        except:
            assert False, "Wrong exception"

        input = ujson.dumps([{}, []])
        try:
            ujson.decode(input, numpy=True)
            assert False, "Expected exception!"
        except(ValueError):
            pass
        except:
            assert False, "Wrong exception"

        input = ujson.dumps([42, None])
        try:
            ujson.decode(input, numpy=True)
            assert False, "Expected exception!"
        except(TypeError):
            pass
        except:
            assert False, "Wrong exception"

        input = ujson.dumps([{'a': 'b'}])
        try:
            ujson.decode(input, numpy=True, labelled=True)
            assert False, "Expected exception!"
        except(ValueError):
            pass
        except:
            assert False, "Wrong exception"

        input = ujson.dumps({'a': {'b': {'c': 42}}})
        try:
            ujson.decode(input, numpy=True, labelled=True)
            assert False, "Expected exception!"
        except(ValueError):
            pass
        except:
            assert False, "Wrong exception"

        input = ujson.dumps([{'a': 42, 'b': 23}, {'c': 17}])
        try:
            ujson.decode(input, numpy=True, labelled=True)
            assert False, "Expected exception!"
        except(ValueError):
            pass
        except:
            assert False, "Wrong exception"

    def test_ArrayNumpyLabelled(self):
        input = {'a': []}
        output = ujson.loads(ujson.dumps(input), numpy=True, labelled=True)
        assert (np.empty((1, 0)) == output[0]).all()
        assert (np.array(['a']) == output[1]).all()
        assert output[2] is None

        input = [{'a': 42}]
        output = ujson.loads(ujson.dumps(input), numpy=True, labelled=True)
        assert (np.array([42]) == output[0]).all()
        assert output[1] is None
        assert (np.array([u('a')]) == output[2]).all()

        # Write out the dump explicitly so there is no dependency on iteration
        # order GH10837
        input_dumps = ('[{"a": 42, "b":31}, {"a": 24, "c": 99}, '
                       '{"a": 2.4, "b": 78}]')
        output = ujson.loads(input_dumps, numpy=True, labelled=True)
        expectedvals = np.array(
            [42, 31, 24, 99, 2.4, 78], dtype=int).reshape((3, 2))
        assert (expectedvals == output[0]).all()
        assert output[1] is None
        assert (np.array([u('a'), 'b']) == output[2]).all()

        input_dumps = ('{"1": {"a": 42, "b":31}, "2": {"a": 24, "c": 99}, '
                       '"3": {"a": 2.4, "b": 78}}')
        output = ujson.loads(input_dumps, numpy=True, labelled=True)
        expectedvals = np.array(
            [42, 31, 24, 99, 2.4, 78], dtype=int).reshape((3, 2))
        assert (expectedvals == output[0]).all()
        assert (np.array(['1', '2', '3']) == output[1]).all()
        assert (np.array(['a', 'b']) == output[2]).all()

class TestPandasJSONTests(object):

    def test_DataFrame(self):
        df = DataFrame([[1, 2, 3], [4, 5, 6]], index=[
                       'a', 'b'], columns=['x', 'y', 'z'])

        # column indexed
        outp = DataFrame(ujson.decode(ujson.encode(df)))
        assert (df == outp).values.all()
        tm.assert_index_equal(df.columns, outp.columns)
        tm.assert_index_equal(df.index, outp.index)

        dec = _clean_dict(ujson.decode(ujson.encode(df, orient="split")))
        outp = DataFrame(**dec)
        assert (df == outp).values.all()
        tm.assert_index_equal(df.columns, outp.columns)
        tm.assert_index_equal(df.index, outp.index)

        outp = DataFrame(ujson.decode(ujson.encode(df, orient="records")))
        outp.index = df.index
        assert (df == outp).values.all()
        tm.assert_index_equal(df.columns, outp.columns)

        outp = DataFrame(ujson.decode(ujson.encode(df, orient="values")))
        outp.index = df.index
        assert (df.values == outp.values).all()

        outp = DataFrame(ujson.decode(ujson.encode(df, orient="index")))
        assert (df.transpose() == outp).values.all()
        tm.assert_index_equal(df.transpose().columns, outp.columns)
        tm.assert_index_equal(df.transpose().index, outp.index)

    def test_DataFrameNumpy(self):
        df = DataFrame([[1, 2, 3], [4, 5, 6]], index=[
                       'a', 'b'], columns=['x', 'y', 'z'])

        # column indexed
        outp = DataFrame(ujson.decode(ujson.encode(df), numpy=True))
        assert (df == outp).values.all()
        tm.assert_index_equal(df.columns, outp.columns)
        tm.assert_index_equal(df.index, outp.index)

        dec = _clean_dict(ujson.decode(ujson.encode(df, orient="split"),
                                       numpy=True))
        outp = DataFrame(**dec)
        assert (df == outp).values.all()
        tm.assert_index_equal(df.columns, outp.columns)
        tm.assert_index_equal(df.index, outp.index)

        outp = DataFrame(ujson.decode(ujson.encode(df, orient="index"),
                                      numpy=True))
        assert (df.transpose() == outp).values.all()
        tm.assert_index_equal(df.transpose().columns, outp.columns)
        tm.assert_index_equal(df.transpose().index, outp.index)

    def test_DataFrameNested(self):
        df = DataFrame([[1, 2, 3], [4, 5, 6]], index=[
                       'a', 'b'], columns=['x', 'y', 'z'])

        nested = {'df1': df, 'df2': df.copy()}

        exp = {'df1': ujson.decode(ujson.encode(df)),
               'df2': ujson.decode(ujson.encode(df))}
        assert ujson.decode(ujson.encode(nested)) == exp

        exp = {'df1': ujson.decode(ujson.encode(df, orient="index")),
               'df2': ujson.decode(ujson.encode(df, orient="index"))}
        assert ujson.decode(ujson.encode(nested, orient="index")) == exp

        exp = {'df1': ujson.decode(ujson.encode(df, orient="records")),
               'df2': ujson.decode(ujson.encode(df, orient="records"))}
        assert ujson.decode(ujson.encode(nested, orient="records")) == exp

        exp = {'df1': ujson.decode(ujson.encode(df, orient="values")),
               'df2': ujson.decode(ujson.encode(df, orient="values"))}
        assert ujson.decode(ujson.encode(nested, orient="values")) == exp

        exp = {'df1': ujson.decode(ujson.encode(df, orient="split")),
               'df2': ujson.decode(ujson.encode(df, orient="split"))}
        assert ujson.decode(ujson.encode(nested, orient="split")) == exp

    def test_DataFrameNumpyLabelled(self):
        df = DataFrame([[1, 2, 3], [4, 5, 6]], index=[
                       'a', 'b'], columns=['x', 'y', 'z'])

        # column indexed
        outp = DataFrame(*ujson.decode(ujson.encode(df),
                                       numpy=True, labelled=True))
        assert (df.T == outp).values.all()
        tm.assert_index_equal(df.T.columns, outp.columns)
        tm.assert_index_equal(df.T.index, outp.index)

        outp = DataFrame(*ujson.decode(ujson.encode(df, orient="records"),
                                       numpy=True, labelled=True))
        outp.index = df.index
        assert (df == outp).values.all()
        tm.assert_index_equal(df.columns, outp.columns)

        outp = DataFrame(*ujson.decode(ujson.encode(df, orient="index"),
                                       numpy=True, labelled=True))
        assert (df == outp).values.all()
        tm.assert_index_equal(df.columns, outp.columns)
        tm.assert_index_equal(df.index, outp.index)

    def test_Series(self):
        s = Series([10, 20, 30, 40, 50, 60], name="series",
                   index=[6, 7, 8, 9, 10, 15]).sort_values()

        # column indexed
        outp = Series(ujson.decode(ujson.encode(s))).sort_values()
        exp = Series([10, 20, 30, 40, 50, 60],
                     index=['6', '7', '8', '9', '10', '15'])
        tm.assert_series_equal(outp, exp)

        outp = Series(ujson.decode(ujson.encode(s), numpy=True)).sort_values()
        tm.assert_series_equal(outp, exp)

        dec = _clean_dict(ujson.decode(ujson.encode(s, orient="split")))
        outp = Series(**dec)
        tm.assert_series_equal(outp, s)

        dec = _clean_dict(ujson.decode(ujson.encode(s, orient="split"),
                                       numpy=True))
        outp = Series(**dec)

        exp_np = Series(np.array([10, 20, 30, 40, 50, 60]))
        exp_pd = Series([10, 20, 30, 40, 50, 60])
        outp = Series(ujson.decode(ujson.encode(s, orient="records"),
                                   numpy=True))
        tm.assert_series_equal(outp, exp_np)

        outp = Series(ujson.decode(ujson.encode(s, orient="records")))
        exp = Series([10, 20, 30, 40, 50, 60])
        tm.assert_series_equal(outp, exp_pd)

        outp = Series(ujson.decode(ujson.encode(s, orient="values"),
                                   numpy=True))
        tm.assert_series_equal(outp, exp_np)

        outp = Series(ujson.decode(ujson.encode(s, orient="values")))
        tm.assert_series_equal(outp, exp_pd)

        outp = Series(ujson.decode(ujson.encode(
            s, orient="index"))).sort_values()
        exp = Series([10, 20, 30, 40, 50, 60],
                     index=['6', '7', '8', '9', '10', '15'])
        tm.assert_series_equal(outp, exp)

        outp = Series(ujson.decode(ujson.encode(
            s, orient="index"), numpy=True)).sort_values()
        tm.assert_series_equal(outp, exp)

    def test_SeriesNested(self):
        s = Series([10, 20, 30, 40, 50, 60], name="series",
                   index=[6, 7, 8, 9, 10, 15]).sort_values()

        nested = {'s1': s, 's2': s.copy()}

        exp = {'s1': ujson.decode(ujson.encode(s)),
               's2': ujson.decode(ujson.encode(s))}
        assert ujson.decode(ujson.encode(nested)) == exp

        exp = {'s1': ujson.decode(ujson.encode(s, orient="split")),
               's2': ujson.decode(ujson.encode(s, orient="split"))}
        assert ujson.decode(ujson.encode(nested, orient="split")) == exp

        exp = {'s1': ujson.decode(ujson.encode(s, orient="records")),
               's2': ujson.decode(ujson.encode(s, orient="records"))}
        assert ujson.decode(ujson.encode(nested, orient="records")) == exp

        exp = {'s1': ujson.decode(ujson.encode(s, orient="values")),
               's2': ujson.decode(ujson.encode(s, orient="values"))}
        assert ujson.decode(ujson.encode(nested, orient="values")) == exp

        exp = {'s1': ujson.decode(ujson.encode(s, orient="index")),
               's2': ujson.decode(ujson.encode(s, orient="index"))}
        assert ujson.decode(ujson.encode(nested, orient="index")) == exp

    def test_Index(self):
        i = Index([23, 45, 18, 98, 43, 11], name="index")

        # column indexed
        outp = Index(ujson.decode(ujson.encode(i)), name='index')
        tm.assert_index_equal(i, outp)

        outp = Index(ujson.decode(ujson.encode(i), numpy=True), name='index')
        tm.assert_index_equal(i, outp)

        dec = _clean_dict(ujson.decode(ujson.encode(i, orient="split")))
        outp = Index(**dec)
        tm.assert_index_equal(i, outp)
        assert i.name == outp.name

        dec = _clean_dict(ujson.decode(ujson.encode(i, orient="split"),
                                       numpy=True))
        outp = Index(**dec)
        tm.assert_index_equal(i, outp)
        assert i.name == outp.name

        outp = Index(ujson.decode(ujson.encode(i, orient="values")),
                     name='index')
        tm.assert_index_equal(i, outp)

        outp = Index(ujson.decode(ujson.encode(i, orient="values"),
                                  numpy=True), name='index')
        tm.assert_index_equal(i, outp)

        outp = Index(ujson.decode(ujson.encode(i, orient="records")),
                     name='index')
        tm.assert_index_equal(i, outp)

        outp = Index(ujson.decode(ujson.encode(i, orient="records"),
                                  numpy=True), name='index')
        tm.assert_index_equal(i, outp)

        outp = Index(ujson.decode(ujson.encode(i, orient="index")),
                     name='index')
        tm.assert_index_equal(i, outp)

        outp = Index(ujson.decode(ujson.encode(i, orient="index"),
                                  numpy=True), name='index')
        tm.assert_index_equal(i, outp)

    def test_datetimeindex(self):
        from pandas.core.indexes.datetimes import date_range

        rng = date_range('1/1/2000', periods=20)

        encoded = ujson.encode(rng, date_unit='ns')
        decoded = DatetimeIndex(np.array(ujson.decode(encoded)))

        tm.assert_index_equal(rng, decoded)

        ts = Series(np.random.randn(len(rng)), index=rng)
        decoded = Series(ujson.decode(ujson.encode(ts, date_unit='ns')))
        idx_values = decoded.index.values.astype(np.int64)
        decoded.index = DatetimeIndex(idx_values)
        tm.assert_series_equal(ts, decoded)

    def test_decodeArrayTrailingCommaFail(self):
        input = "[31337,]"
        try:
            ujson.decode(input)
        except ValueError:
            pass
        else:
            assert False, "expected ValueError"

    def test_decodeArrayLeadingCommaFail(self):
        input = "[,31337]"
        try:
            ujson.decode(input)
        except ValueError:
            pass
        else:
            assert False, "expected ValueError"

    def test_decodeArrayOnlyCommaFail(self):
        input = "[,]"
        try:
            ujson.decode(input)
        except ValueError:
            pass
        else:
            assert False, "expected ValueError"

    def test_decodeArrayUnmatchedBracketFail(self):
        input = "[]]"
        try:
            ujson.decode(input)
        except ValueError:
            pass
        else:
            assert False, "expected ValueError"

    def test_decodeArrayEmpty(self):
        input = "[]"
        ujson.decode(input)

    def test_decodeArrayOneItem(self):
        input = "[31337]"
        ujson.decode(input)

    def test_decodeBigValue(self):
        input = "9223372036854775807"
        ujson.decode(input)

    def test_decodeSmallValue(self):
        input = "-9223372036854775808"
        ujson.decode(input)

    def test_decodeTooBigValue(self):
        try:
            input = "9223372036854775808"
            ujson.decode(input)
        except ValueError:
            pass
        else:
            assert False, "expected ValueError"

    def test_decodeTooSmallValue(self):
        try:
            input = "-90223372036854775809"
            ujson.decode(input)
        except ValueError:
            pass
        else:
            assert False, "expected ValueError"

    def test_decodeVeryTooBigValue(self):
        try:
            input = "9223372036854775808"
            ujson.decode(input)
        except ValueError:
            pass
        else:
            assert False, "expected ValueError"

    def test_decodeVeryTooSmallValue(self):
        try:
            input = "-90223372036854775809"
            ujson.decode(input)
        except ValueError:
            pass
        else:
            assert False, "expected ValueError"

    def test_decodeWithTrailingWhitespaces(self):
        input = "{}\n\t "
        ujson.decode(input)

    def test_decodeWithTrailingNonWhitespaces(self):
        try:
            input = "{}\n\t a"
            ujson.decode(input)
        except ValueError:
            pass
        else:
            assert False, "expected ValueError"

    def test_decodeArrayWithBigInt(self):
        try:
            ujson.loads('[18446098363113800555]')
        except ValueError:
            pass
        else:
            assert False, "expected ValueError"

    def test_decodeArrayFaultyUnicode(self):
        try:
            ujson.loads('[18446098363113800555]')
        except ValueError:
            pass
        else:
            assert False, "expected ValueError"

    def test_decodeFloatingPointAdditionalTests(self):
        places = 15

        tm.assert_almost_equal(-1.1234567893,
                               ujson.loads("-1.1234567893"),
                               check_less_precise=places)
        tm.assert_almost_equal(-1.234567893,
                               ujson.loads("-1.234567893"),
                               check_less_precise=places)
        tm.assert_almost_equal(-1.34567893,
                               ujson.loads("-1.34567893"),
                               check_less_precise=places)
        tm.assert_almost_equal(-1.4567893,
                               ujson.loads("-1.4567893"),
                               check_less_precise=places)
        tm.assert_almost_equal(-1.567893,
                               ujson.loads("-1.567893"),
                               check_less_precise=places)
        tm.assert_almost_equal(-1.67893,
                               ujson.loads("-1.67893"),
                               check_less_precise=places)
        tm.assert_almost_equal(-1.7893, ujson.loads("-1.7893"),
                               check_less_precise=places)
        tm.assert_almost_equal(-1.893, ujson.loads("-1.893"),
                               check_less_precise=places)
        tm.assert_almost_equal(-1.3, ujson.loads("-1.3"),
                               check_less_precise=places)

        tm.assert_almost_equal(1.1234567893, ujson.loads(
            "1.1234567893"), check_less_precise=places)
        tm.assert_almost_equal(1.234567893, ujson.loads(
            "1.234567893"), check_less_precise=places)
        tm.assert_almost_equal(
            1.34567893, ujson.loads("1.34567893"), check_less_precise=places)
        tm.assert_almost_equal(
            1.4567893, ujson.loads("1.4567893"), check_less_precise=places)
        tm.assert_almost_equal(
            1.567893, ujson.loads("1.567893"), check_less_precise=places)
        tm.assert_almost_equal(1.67893, ujson.loads("1.67893"),
                               check_less_precise=places)
        tm.assert_almost_equal(1.7893, ujson.loads("1.7893"),
                               check_less_precise=places)
        tm.assert_almost_equal(1.893, ujson.loads("1.893"),
                               check_less_precise=places)
        tm.assert_almost_equal(1.3, ujson.loads("1.3"),
                               check_less_precise=places)

    def test_encodeBigSet(self):
        s = set()
        for x in range(0, 100000):
            s.add(x)
        ujson.encode(s)

    def test_encodeEmptySet(self):
        s = set()
        assert "[]" == ujson.encode(s)

    def test_encodeSet(self):
        s = set([1, 2, 3, 4, 5, 6, 7, 8, 9])
        enc = ujson.encode(s)
        dec = ujson.decode(enc)

        for v in dec:
            assert v in s

def _clean_dict(d):
    return {str(k): v for k, v in compat.iteritems(d)}


# -*- coding: utf-8 -*-
"""
Sphinx directive to support embedded IPython code.

This directive allows pasting of entire interactive IPython sessions, prompts
and all, and their code will actually get re-executed at doc build time, with
all prompts renumbered sequentially. It also allows you to input code as a pure
python input by giving the argument python to the directive. The output looks
like an interactive ipython section.

To enable this directive, simply list it in your Sphinx ``conf.py`` file
(making sure the directory where you placed it is visible to sphinx, as is
needed for all Sphinx directives). For example, to enable syntax highlighting
and the IPython directive::

    extensions = ['IPython.sphinxext.ipython_console_highlighting',
                  'IPython.sphinxext.ipython_directive']

The IPython directive outputs code-blocks with the language 'ipython'. So
if you do not have the syntax highlighting extension enabled as well, then
all rendered code-blocks will be uncolored. By default this directive assumes
that your prompts are unchanged IPython ones, but this can be customized.
The configurable options that can be placed in conf.py are:

ipython_savefig_dir:
    The directory in which to save the figures. This is relative to the
    Sphinx source directory. The default is `html_static_path`.
ipython_rgxin:
    The compiled regular expression to denote the start of IPython input
    lines. The default is re.compile('In \[(\d+)\]:\s?(.*)\s*'). You
    shouldn't need to change this.
ipython_rgxout:
    The compiled regular expression to denote the start of IPython output
    lines. The default is re.compile('Out\[(\d+)\]:\s?(.*)\s*'). You
    shouldn't need to change this.
ipython_promptin:
    The string to represent the IPython input prompt in the generated ReST.
    The default is 'In [%d]:'. This expects that the line numbers are used
    in the prompt.
ipython_promptout:
    The string to represent the IPython prompt in the generated ReST. The
    default is 'Out [%d]:'. This expects that the line numbers are used
    in the prompt.
ipython_mplbackend:
    The string which specifies if the embedded Sphinx shell should import
    Matplotlib and set the backend. The value specifies a backend that is
    passed to `matplotlib.use()` before any lines in `ipython_execlines` are
    executed. If not specified in conf.py, then the default value of 'agg' is
    used. To use the IPython directive without matplotlib as a dependency, set
    the value to `None`. It may end up that matplotlib is still imported
    if the user specifies so in `ipython_execlines` or makes use of the
    @savefig pseudo decorator.
ipython_execlines:
    A list of strings to be exec'd in the embedded Sphinx shell. Typical
    usage is to make certain packages always available. Set this to an empty
    list if you wish to have no imports always available. If specified in
    conf.py as `None`, then it has the effect of making no imports available.
    If omitted from conf.py altogether, then the default value of
    ['import numpy as np', 'import matplotlib.pyplot as plt'] is used.
ipython_holdcount
    When the @suppress pseudo-decorator is used, the execution count can be
    incremented or not. The default behavior is to hold the execution count,
    corresponding to a value of `True`. Set this to `False` to increment
    the execution count after each suppressed command.

As an example, to use the IPython directive when `matplotlib` is not available,
one sets the backend to `None`::

    ipython_mplbackend = None

An example usage of the directive is:

.. code-block:: rst

    .. ipython::

        In [1]: x = 1

        In [2]: y = x**2

        In [3]: print(y)

See http://matplotlib.org/sampledoc/ipython_directive.html for additional
documentation.

Pseudo-Decorators
=================

Note: Only one decorator is supported per input. If more than one decorator
is specified, then only the last one is used.

In addition to the Pseudo-Decorators/options described at the above link,
several enhancements have been made. The directive will emit a message to the
console at build-time if code-execution resulted in an exception or warning.
You can suppress these on a per-block basis by specifying the :okexcept:
or :okwarning: options:

.. code-block:: rst

    .. ipython::
        :okexcept:
        :okwarning:

        In [1]: 1/0
        In [2]: # raise warning.

ToDo
----

- Turn the ad-hoc test() function into a real test suite.
- Break up ipython-specific functionality from matplotlib stuff into better
  separated code.

Authors
-------

- John D Hunter: orignal author.
- Fernando Perez: refactoring, documentation, cleanups, port to 0.11.
- VáclavŠmilauer <eudoxos-AT-arcig.cz>: Prompt generalizations.
- Skipper Seabold, refactoring, cleanups, pure python addition
"""
from __future__ import print_function

#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------

# Stdlib
import atexit
import os
import re
import sys
import tempfile
import ast
import warnings
import shutil

# Third-party
from docutils.parsers.rst import directives
from sphinx.util.compat import Directive

# Our own
from traitlets.config import Config
from IPython import InteractiveShell
from IPython.core.profiledir import ProfileDir
from IPython.utils import io
from IPython.utils.py3compat import PY3

if PY3:
    from io import StringIO
else:
    from StringIO import StringIO

#-----------------------------------------------------------------------------
# Globals
#-----------------------------------------------------------------------------
# for tokenizing blocks
COMMENT, INPUT, OUTPUT =  range(3)

#-----------------------------------------------------------------------------
# Functions and class declarations
#-----------------------------------------------------------------------------

def block_parser(part, rgxin, rgxout, fmtin, fmtout):
    """
    part is a string of ipython text, comprised of at most one
    input, one output, comments, and blank lines.  The block parser
    parses the text into a list of::

      blocks = [ (TOKEN0, data0), (TOKEN1, data1), ...]

    where TOKEN is one of [COMMENT | INPUT | OUTPUT ] and
    data is, depending on the type of token::

      COMMENT : the comment string

      INPUT: the (DECORATOR, INPUT_LINE, REST) where
         DECORATOR: the input decorator (or None)
         INPUT_LINE: the input as string (possibly multi-line)
         REST : any stdout generated by the input line (not OUTPUT)

      OUTPUT: the output string, possibly multi-line

    """
    block = []
    lines = part.split('\n')
    N = len(lines)
    i = 0
    decorator = None
    while 1:

        if i==N:
            # nothing left to parse -- the last line
            break

        line = lines[i]
        i += 1
        line_stripped = line.strip()
        if line_stripped.startswith('#'):
            block.append((COMMENT, line))
            continue

        if line_stripped.startswith('@'):
            # Here is where we assume there is, at most, one decorator.
            # Might need to rethink this.
            decorator = line_stripped
            continue

        # does this look like an input line?
        matchin = rgxin.match(line)
        if matchin:
            lineno, inputline = int(matchin.group(1)), matchin.group(2)

            # the ....: continuation string
            continuation = '   %s:'%''.join(['.']*(len(str(lineno))+2))
            Nc = len(continuation)
            # input lines can continue on for more than one line, if
            # we have a '\' line continuation char or a function call
            # echo line 'print'.  The input line can only be
            # terminated by the end of the block or an output line, so
            # we parse out the rest of the input line if it is
            # multiline as well as any echo text

            rest = []
            while i<N:

                # look ahead; if the next line is blank, or a comment, or
                # an output line, we're done

                nextline = lines[i]
                matchout = rgxout.match(nextline)
                #print "nextline=%s, continuation=%s, starts=%s"%(nextline, continuation, nextline.startswith(continuation))
                if matchout or nextline.startswith('#'):
                    break
                elif nextline.startswith(continuation):
                    # The default ipython_rgx* treat the space following the colon as optional.
                    # However, If the space is there we must consume it or code
                    # employing the cython_magic extension will fail to execute.
                    #
                    # This works with the default ipython_rgx* patterns,
                    # If you modify them, YMMV.
                    nextline = nextline[Nc:]
                    if nextline and nextline[0] == ' ':
                        nextline = nextline[1:]

                    inputline += '\n' +  nextline
                else:
                    rest.append(nextline)
                i+= 1

            block.append((INPUT, (decorator, inputline, '\n'.join(rest))))
            continue

        # if it looks like an output line grab all the text to the end
        # of the block
        matchout = rgxout.match(line)
        if matchout:
            lineno, output = int(matchout.group(1)), matchout.group(2)
            if i<N-1:
                output = '\n'.join([output] + lines[i:])

            block.append((OUTPUT, output))
            break

    return block

class EmbeddedSphinxShell(object):
    """An embedded IPython instance to run inside Sphinx"""

    def __init__(self, exec_lines=None):

        self.cout = StringIO()

        if exec_lines is None:
            exec_lines = []

        # Create config object for IPython
        config = Config()
        config.HistoryManager.hist_file = ':memory:'
        config.InteractiveShell.autocall = False
        config.InteractiveShell.autoindent = False
        config.InteractiveShell.colors = 'NoColor'

        # create a profile so instance history isn't saved
        tmp_profile_dir = tempfile.mkdtemp(prefix='profile_')
        profname = 'auto_profile_sphinx_build'
        pdir = os.path.join(tmp_profile_dir,profname)
        profile = ProfileDir.create_profile_dir(pdir)

        # Create and initialize global ipython, but don't start its mainloop.
        # This will persist across different EmbededSphinxShell instances.
        IP = InteractiveShell.instance(config=config, profile_dir=profile)
        atexit.register(self.cleanup)

        # io.stdout redirect must be done after instantiating InteractiveShell
        io.stdout = self.cout
        io.stderr = self.cout

        # For debugging, so we can see normal output, use this:
        #from IPython.utils.io import Tee
        #io.stdout = Tee(self.cout, channel='stdout') # dbg
        #io.stderr = Tee(self.cout, channel='stderr') # dbg

        # Store a few parts of IPython we'll need.
        self.IP = IP
        self.user_ns = self.IP.user_ns
        self.user_global_ns = self.IP.user_global_ns

        self.input = ''
        self.output = ''
        self.tmp_profile_dir = tmp_profile_dir

        self.is_verbatim = False
        self.is_doctest = False
        self.is_suppress = False

        # Optionally, provide more detailed information to shell.
        # this is assigned by the SetUp method of IPythonDirective
        # to point at itself.
        #
        # So, you can access handy things at self.directive.state
        self.directive = None

        # on the first call to the savefig decorator, we'll import
        # pyplot as plt so we can make a call to the plt.gcf().savefig
        self._pyplot_imported = False

        # Prepopulate the namespace.
        for line in exec_lines:
            self.process_input_line(line, store_history=False)

    def cleanup(self):
        shutil.rmtree(self.tmp_profile_dir, ignore_errors=True)

    def clear_cout(self):
        self.cout.seek(0)
        self.cout.truncate(0)

    def process_input_line(self, line, store_history=True):
        """process the input, capturing stdout"""

        stdout = sys.stdout
        splitter = self.IP.input_splitter
        try:
            sys.stdout = self.cout
            splitter.push(line)
            more = splitter.push_accepts_more()
            if not more:
                source_raw = splitter.raw_reset()
                self.IP.run_cell(source_raw, store_history=store_history)
        finally:
            sys.stdout = stdout

    def process_image(self, decorator):
        """
        # build out an image directive like
        # .. image:: somefile.png
        #    :width 4in
        #
        # from an input like
        # savefig somefile.png width=4in
        """
        savefig_dir = self.savefig_dir
        source_dir = self.source_dir
        saveargs = decorator.split(' ')
        filename = saveargs[1]
        # insert relative path to image file in source
        outfile = os.path.relpath(os.path.join(savefig_dir,filename),
                    source_dir)

        imagerows = ['.. image:: %s'%outfile]

        for kwarg in saveargs[2:]:
            arg, val = kwarg.split('=')
            arg = arg.strip()
            val = val.strip()
            imagerows.append('   :%s: %s'%(arg, val))

        image_file = os.path.basename(outfile) # only return file name
        image_directive = '\n'.join(imagerows)
        return image_file, image_directive

    # Callbacks for each type of token
    def process_input(self, data, input_prompt, lineno):
        """
        Process data block for INPUT token.

        """
        decorator, input, rest = data
        image_file = None
        image_directive = None

        is_verbatim = decorator=='@verbatim' or self.is_verbatim
        is_doctest = (decorator is not None and \
                     decorator.startswith('@doctest')) or self.is_doctest
        is_suppress = decorator=='@suppress' or self.is_suppress
        is_okexcept = decorator=='@okexcept' or self.is_okexcept
        is_okwarning = decorator=='@okwarning' or self.is_okwarning
        is_savefig = decorator is not None and \
                     decorator.startswith('@savefig')

        input_lines = input.split('\n')
        if len(input_lines) > 1:
            if input_lines[-1] != "":
                input_lines.append('') # make sure there's a blank line
                                       # so splitter buffer gets reset

        continuation = '   %s:'%''.join(['.']*(len(str(lineno))+2))

        if is_savefig:
            image_file, image_directive = self.process_image(decorator)

        ret = []
        is_semicolon = False

        # Hold the execution count, if requested to do so.
        if is_suppress and self.hold_count:
            store_history = False
        else:
            store_history = True

        # Note: catch_warnings is not thread safe
        with warnings.catch_warnings(record=True) as ws:
            for i, line in enumerate(input_lines):
                if line.endswith(';'):
                    is_semicolon = True

                if i == 0:
                    # process the first input line
                    if is_verbatim:
                        self.process_input_line('')
                        self.IP.execution_count += 1 # increment it anyway
                    else:
                        # only submit the line in non-verbatim mode
                        self.process_input_line(line, store_history=store_history)
                    formatted_line = '%s %s'%(input_prompt, line)
                else:
                    # process a continuation line
                    if not is_verbatim:
                        self.process_input_line(line, store_history=store_history)

                    formatted_line = '%s %s'%(continuation, line)

                if not is_suppress:
                    ret.append(formatted_line)

        if not is_suppress and len(rest.strip()) and is_verbatim:
            # The "rest" is the standard output of the input. This needs to be
            # added when in verbatim mode. If there is no "rest", then we don't
            # add it, as the new line will be added by the processed output.
            ret.append(rest)

        # Fetch the processed output. (This is not the submitted output.)
        self.cout.seek(0)
        processed_output = self.cout.read()
        if not is_suppress and not is_semicolon:
            #
            # In IPythonDirective.run, the elements of `ret` are eventually
            # combined such that '' entries correspond to newlines. So if
            # `processed_output` is equal to '', then the adding it to `ret`
            # ensures that there is a blank line between consecutive inputs
            # that have no outputs, as in:
            #
            #    In [1]: x = 4
            #
            #    In [2]: x = 5
            #
            # When there is processed output, it has a '\n' at the tail end. So
            # adding the output to `ret` will provide the necessary spacing
            # between consecutive input/output blocks, as in:
            #
            #   In [1]: x
            #   Out[1]: 5
            #
            #   In [2]: x
            #   Out[2]: 5
            #
            # When there is stdout from the input, it also has a '\n' at the
            # tail end, and so this ensures proper spacing as well. E.g.:
            #
            #   In [1]: print x
            #   5
            #
            #   In [2]: x = 5
            #
            # When in verbatim mode, `processed_output` is empty (because
            # nothing was passed to IP. Sometimes the submitted code block has
            # an Out[] portion and sometimes it does not. When it does not, we
            # need to ensure proper spacing, so we have to add '' to `ret`.
            # However, if there is an Out[] in the submitted code, then we do
            # not want to add a newline as `process_output` has stuff to add.
            # The difficulty is that `process_input` doesn't know if
            # `process_output` will be called---so it doesn't know if there is
            # Out[] in the code block. The requires that we include a hack in
            # `process_block`. See the comments there.
            #
            ret.append(processed_output)
        elif is_semicolon:
            # Make sure there is a newline after the semicolon.
            ret.append('')

        # context information
        filename = "Unknown"
        lineno = 0
        if self.directive.state:
            filename = self.directive.state.document.current_source
            lineno = self.directive.state.document.current_line

        # output any exceptions raised during execution to stdout
        # unless :okexcept: has been specified.
        if not is_okexcept and "Traceback" in processed_output:
            s =  "\nException in %s at block ending on line %s\n" % (filename, lineno)
            s += "Specify :okexcept: as an option in the ipython:: block to suppress this message\n"
            sys.stdout.write('\n\n>>>' + ('-' * 73))
            sys.stdout.write(s)
            sys.stdout.write(processed_output)
            sys.stdout.write('<<<' + ('-' * 73) + '\n\n')

        # output any warning raised during execution to stdout
        # unless :okwarning: has been specified.
        if not is_okwarning:
            for w in ws:
                s =  "\nWarning in %s at block ending on line %s\n" % (filename, lineno)
                s += "Specify :okwarning: as an option in the ipython:: block to suppress this message\n"
                sys.stdout.write('\n\n>>>' + ('-' * 73))
                sys.stdout.write(s)
                sys.stdout.write(('-' * 76) + '\n')
                s=warnings.formatwarning(w.message, w.category,
                                         w.filename, w.lineno, w.line)
                sys.stdout.write(s)
                sys.stdout.write('<<<' + ('-' * 73) + '\n')

        self.cout.truncate(0)

        return (ret, input_lines, processed_output,
                is_doctest, decorator, image_file, image_directive)

    def process_output(self, data, output_prompt, input_lines, output,
                       is_doctest, decorator, image_file):
        """
        Process data block for OUTPUT token.

        """
        # Recall: `data` is the submitted output, and `output` is the processed
        # output from `input_lines`.

        TAB = ' ' * 4

        if is_doctest and output is not None:

            found = output # This is the processed output
            found = found.strip()
            submitted = data.strip()

            if self.directive is None:
                source = 'Unavailable'
                content = 'Unavailable'
            else:
                source = self.directive.state.document.current_source
                content = self.directive.content
                # Add tabs and join into a single string.
                content = '\n'.join([TAB + line for line in content])

            # Make sure the output contains the output prompt.
            ind = found.find(output_prompt)
            if ind < 0:
                e = ('output does not contain output prompt\n\n'
                     'Document source: {0}\n\n'
                     'Raw content: \n{1}\n\n'
                     'Input line(s):\n{TAB}{2}\n\n'
                     'Output line(s):\n{TAB}{3}\n\n')
                e = e.format(source, content, '\n'.join(input_lines),
                             repr(found), TAB=TAB)
                raise RuntimeError(e)
            found = found[len(output_prompt):].strip()

            # Handle the actual doctest comparison.
            if decorator.strip() == '@doctest':
                # Standard doctest
                if found != submitted:
                    e = ('doctest failure\n\n'
                         'Document source: {0}\n\n'
                         'Raw content: \n{1}\n\n'
                         'On input line(s):\n{TAB}{2}\n\n'
                         'we found output:\n{TAB}{3}\n\n'
                         'instead of the expected:\n{TAB}{4}\n\n')
                    e = e.format(source, content, '\n'.join(input_lines),
                                 repr(found), repr(submitted), TAB=TAB)
                    raise RuntimeError(e)
            else:
                self.custom_doctest(decorator, input_lines, found, submitted)

        # When in verbatim mode, this holds additional submitted output
        # to be written in the final Sphinx output.
        # https://github.com/ipython/ipython/issues/5776
        out_data = []

        is_verbatim = decorator=='@verbatim' or self.is_verbatim
        if is_verbatim and data.strip():
            # Note that `ret` in `process_block` has '' as its last element if
            # the code block was in verbatim mode. So if there is no submitted
            # output, then we will have proper spacing only if we do not add
            # an additional '' to `out_data`. This is why we condition on
            # `and data.strip()`.

            # The submitted output has no output prompt. If we want the
            # prompt and the code to appear, we need to join them now
            # instead of adding them separately---as this would create an
            # undesired newline. How we do this ultimately depends on the
            # format of the output regex. I'll do what works for the default
            # prompt for now, and we might have to adjust if it doesn't work
            # in other cases. Finally, the submitted output does not have
            # a trailing newline, so we must add it manually.
            out_data.append("{0} {1}\n".format(output_prompt, data))

        return out_data

    def process_comment(self, data):
        """Process data fPblock for COMMENT token."""
        if not self.is_suppress:
            return [data]

    def save_image(self, image_file):
        """
        Saves the image file to disk.
        """
        self.ensure_pyplot()
        command = 'plt.gcf().savefig("%s")'%image_file
        #print 'SAVEFIG', command  # dbg
        self.process_input_line('bookmark ipy_thisdir', store_history=False)
        self.process_input_line('cd -b ipy_savedir', store_history=False)
        self.process_input_line(command, store_history=False)
        self.process_input_line('cd -b ipy_thisdir', store_history=False)
        self.process_input_line('bookmark -d ipy_thisdir', store_history=False)
        self.clear_cout()

    def process_block(self, block):
        """
        process block from the block_parser and return a list of processed lines
        """
        ret = []
        output = None
        input_lines = None
        lineno = self.IP.execution_count

        input_prompt = self.promptin % lineno
        output_prompt = self.promptout % lineno
        image_file = None
        image_directive = None

        found_input = False
        for token, data in block:
            if token == COMMENT:
                out_data = self.process_comment(data)
            elif token == INPUT:
                found_input = True
                (out_data, input_lines, output, is_doctest,
                 decorator, image_file, image_directive) = \
                          self.process_input(data, input_prompt, lineno)
            elif token == OUTPUT:
                if not found_input:

                    TAB = ' ' * 4
                    linenumber = 0
                    source = 'Unavailable'
                    content = 'Unavailable'
                    if self.directive:
                        linenumber = self.directive.state.document.current_line
                        source = self.directive.state.document.current_source
                        content = self.directive.content
                        # Add tabs and join into a single string.
                        content = '\n'.join([TAB + line for line in content])

                    e = ('\n\nInvalid block: Block contains an output prompt '
                         'without an input prompt.\n\n'
                         'Document source: {0}\n\n'
                         'Content begins at line {1}: \n\n{2}\n\n'
                         'Problematic block within content: \n\n{TAB}{3}\n\n')
                    e = e.format(source, linenumber, content, block, TAB=TAB)

                    # Write, rather than include in exception, since Sphinx
                    # will truncate tracebacks.
                    sys.stdout.write(e)
                    raise RuntimeError('An invalid block was detected.')

                out_data = \
                    self.process_output(data, output_prompt, input_lines,
                                        output, is_doctest, decorator,
                                        image_file)
                if out_data:
                    # Then there was user submitted output in verbatim mode.
                    # We need to remove the last element of `ret` that was
                    # added in `process_input`, as it is '' and would introduce
                    # an undesirable newline.
                    assert(ret[-1] == '')
                    del ret[-1]

            if out_data:
                ret.extend(out_data)

        # save the image files
        if image_file is not None:
            self.save_image(image_file)

        return ret, image_directive

    def ensure_pyplot(self):
        """
        Ensures that pyplot has been imported into the embedded IPython shell.

        Also, makes sure to set the backend appropriately if not set already.

        """
        # We are here if the @figure pseudo decorator was used. Thus, it's
        # possible that we could be here even if python_mplbackend were set to
        # `None`. That's also strange and perhaps worthy of raising an
        # exception, but for now, we just set the backend to 'agg'.

        if not self._pyplot_imported:
            if 'matplotlib.backends' not in sys.modules:
                # Then ipython_matplotlib was set to None but there was a
                # call to the @figure decorator (and ipython_execlines did
                # not set a backend).
                #raise Exception("No backend was set, but @figure was used!")
                import matplotlib
                matplotlib.use('agg')

            # Always import pyplot into embedded shell.
            self.process_input_line('import matplotlib.pyplot as plt',
                                    store_history=False)
            self._pyplot_imported = True

    def process_pure_python(self, content):
        """
        content is a list of strings. it is unedited directive content

        This runs it line by line in the InteractiveShell, prepends
        prompts as needed capturing stderr and stdout, then returns
        the content as a list as if it were ipython code
        """
        output = []
        savefig = False # keep up with this to clear figure
        multiline = False # to handle line continuation
        multiline_start = None
        fmtin = self.promptin

        ct = 0

        for lineno, line in enumerate(content):

            line_stripped = line.strip()
            if not len(line):
                output.append(line)
                continue

            # handle decorators
            if line_stripped.startswith('@'):
                output.extend([line])
                if 'savefig' in line:
                    savefig = True # and need to clear figure
                continue

            # handle comments
            if line_stripped.startswith('#'):
                output.extend([line])
                continue

            # deal with lines checking for multiline
            continuation  = u'   %s:'% ''.join(['.']*(len(str(ct))+2))
            if not multiline:
                modified = u"%s %s" % (fmtin % ct, line_stripped)
                output.append(modified)
                ct += 1
                try:
                    ast.parse(line_stripped)
                    output.append(u'')
                except Exception: # on a multiline
                    multiline = True
                    multiline_start = lineno
            else: # still on a multiline
                modified = u'%s %s' % (continuation, line)
                output.append(modified)

                # if the next line is indented, it should be part of multiline
                if len(content) > lineno + 1:
                    nextline = content[lineno + 1]
                    if len(nextline) - len(nextline.lstrip()) > 3:
                        continue
                try:
                    mod = ast.parse(
                            '\n'.join(content[multiline_start:lineno+1]))
                    if isinstance(mod.body[0], ast.FunctionDef):
                        # check to see if we have the whole function
                        for element in mod.body[0].body:
                            if isinstance(element, ast.Return):
                                multiline = False
                    else:
                        output.append(u'')
                        multiline = False
                except Exception:
                    pass

            if savefig: # clear figure if plotted
                self.ensure_pyplot()
                self.process_input_line('plt.clf()', store_history=False)
                self.clear_cout()
                savefig = False

        return output

    def custom_doctest(self, decorator, input_lines, found, submitted):
        """
        Perform a specialized doctest.

        """
        from .custom_doctests import doctests

        args = decorator.split()
        doctest_type = args[1]
        if doctest_type in doctests:
            doctests[doctest_type](self, args, input_lines, found, submitted)
        else:
            e = "Invalid option to @doctest: {0}".format(doctest_type)
            raise Exception(e)

class IPythonDirective(Directive):

    has_content = True
    required_arguments = 0
    optional_arguments = 4 # python, suppress, verbatim, doctest
    final_argumuent_whitespace = True
    option_spec = { 'python': directives.unchanged,
                    'suppress' : directives.flag,
                    'verbatim' : directives.flag,
                    'doctest' : directives.flag,
                    'okexcept': directives.flag,
                    'okwarning': directives.flag
                  }

    shell = None

    seen_docs = set()

    def get_config_options(self):
        # contains sphinx configuration variables
        config = self.state.document.settings.env.config

        # get config variables to set figure output directory
        outdir = self.state.document.settings.env.app.outdir
        savefig_dir = config.ipython_savefig_dir
        source_dir = os.path.dirname(self.state.document.current_source)
        if savefig_dir is None:
            savefig_dir = config.html_static_path or '_static'
        if isinstance(savefig_dir, list):
            savefig_dir = os.path.join(*savefig_dir)
        savefig_dir = os.path.join(outdir, savefig_dir)

        # get regex and prompt stuff
        rgxin      = config.ipython_rgxin
        rgxout     = config.ipython_rgxout
        promptin   = config.ipython_promptin
        promptout  = config.ipython_promptout
        mplbackend = config.ipython_mplbackend
        exec_lines = config.ipython_execlines
        hold_count = config.ipython_holdcount

        return (savefig_dir, source_dir, rgxin, rgxout,
                promptin, promptout, mplbackend, exec_lines, hold_count)

    def setup(self):
        # Get configuration values.
        (savefig_dir, source_dir, rgxin, rgxout, promptin, promptout,
         mplbackend, exec_lines, hold_count) = self.get_config_options()

        if self.shell is None:
            # We will be here many times.  However, when the
            # EmbeddedSphinxShell is created, its interactive shell member
            # is the same for each instance.

            if mplbackend:
                import matplotlib
                # Repeated calls to use() will not hurt us since `mplbackend`
                # is the same each time.
                matplotlib.use(mplbackend)

            # Must be called after (potentially) importing matplotlib and
            # setting its backend since exec_lines might import pylab.
            self.shell = EmbeddedSphinxShell(exec_lines)

            # Store IPython directive to enable better error messages
            self.shell.directive = self

        # reset the execution count if we haven't processed this doc
        #NOTE: this may be borked if there are multiple seen_doc tmp files
        #check time stamp?
        if not self.state.document.current_source in self.seen_docs:
            self.shell.IP.history_manager.reset()
            self.shell.IP.execution_count = 1
            self.shell.IP.prompt_manager.width = 0
            self.seen_docs.add(self.state.document.current_source)

        # and attach to shell so we don't have to pass them around
        self.shell.rgxin = rgxin
        self.shell.rgxout = rgxout
        self.shell.promptin = promptin
        self.shell.promptout = promptout
        self.shell.savefig_dir = savefig_dir
        self.shell.source_dir = source_dir
        self.shell.hold_count = hold_count

        # setup bookmark for saving figures directory
        self.shell.process_input_line('bookmark ipy_savedir %s'%savefig_dir,
                                      store_history=False)
        self.shell.clear_cout()

        return rgxin, rgxout, promptin, promptout

    def teardown(self):
        # delete last bookmark
        self.shell.process_input_line('bookmark -d ipy_savedir',
                                      store_history=False)
        self.shell.clear_cout()

    def run(self):
        debug = False

        #TODO, any reason block_parser can't be a method of embeddable shell
        # then we wouldn't have to carry these around
        rgxin, rgxout, promptin, promptout = self.setup()

        options = self.options
        self.shell.is_suppress = 'suppress' in options
        self.shell.is_doctest = 'doctest' in options
        self.shell.is_verbatim = 'verbatim' in options
        self.shell.is_okexcept = 'okexcept' in options
        self.shell.is_okwarning = 'okwarning' in options

        # handle pure python code
        if 'python' in self.arguments:
            content = self.content
            self.content = self.shell.process_pure_python(content)

        # parts consists of all text within the ipython-block.
        # Each part is an input/output block.
        parts = '\n'.join(self.content).split('\n\n')

        lines = ['.. code-block:: ipython', '']
        figures = []

        for part in parts:
            block = block_parser(part, rgxin, rgxout, promptin, promptout)
            if len(block):
                rows, figure = self.shell.process_block(block)
                for row in rows:
                    lines.extend(['   {0}'.format(line)
                                  for line in row.split('\n')])

                if figure is not None:
                    figures.append(figure)

        for figure in figures:
            lines.append('')
            lines.extend(figure.split('\n'))
            lines.append('')

        if len(lines) > 2:
            if debug:
                print('\n'.join(lines))
            else:
                # This has to do with input, not output. But if we comment
                # these lines out, then no IPython code will appear in the
                # final output.
                self.state_machine.insert_input(
                    lines, self.state_machine.input_lines.source(0))

        # cleanup
        self.teardown()

        return []

# Enable as a proper Sphinx directive
def setup(app):
    setup.app = app

    app.add_directive('ipython', IPythonDirective)
    app.add_config_value('ipython_savefig_dir', None, 'env')
    app.add_config_value('ipython_rgxin',
                         re.compile('In \[(\d+)\]:\s?(.*)\s*'), 'env')
    app.add_config_value('ipython_rgxout',
                         re.compile('Out\[(\d+)\]:\s?(.*)\s*'), 'env')
    app.add_config_value('ipython_promptin', 'In [%d]:', 'env')
    app.add_config_value('ipython_promptout', 'Out[%d]:', 'env')

    # We could just let matplotlib pick whatever is specified as the default
    # backend in the matplotlibrc file, but this would cause issues if the
    # backend didn't work in headless environments. For this reason, 'agg'
    # is a good default backend choice.
    app.add_config_value('ipython_mplbackend', 'agg', 'env')

    # If the user sets this config value to `None`, then EmbeddedSphinxShell's
    # __init__ method will treat it as [].
    execlines = ['import numpy as np', 'import matplotlib.pyplot as plt']
    app.add_config_value('ipython_execlines', execlines, 'env')

    app.add_config_value('ipython_holdcount', True, 'env')

    metadata = {'parallel_read_safe': True, 'parallel_write_safe': True}
    return metadata

# Simple smoke test, needs to be converted to a proper automatic test.
def test():

    examples = [
        r"""
In [9]: pwd
Out[9]: '/home/jdhunter/py4science/book'

In [10]: cd bookdata/
/home/jdhunter/py4science/book/bookdata

In [2]: from pylab import *

In [2]: ion()

In [3]: im = imread('stinkbug.png')

@savefig mystinkbug.png width=4in
In [4]: imshow(im)
Out[4]: <matplotlib.image.AxesImage object at 0x39ea850>

""",
        r"""

In [1]: x = 'hello world'

# string methods can be
# used to alter the string
@doctest
In [2]: x.upper()
Out[2]: 'HELLO WORLD'

@verbatim
In [3]: x.st<TAB>
x.startswith  x.strip
""",
    r"""

In [130]: url = 'http://ichart.finance.yahoo.com/table.csv?s=CROX\
   .....: &d=9&e=22&f=2009&g=d&a=1&br=8&c=2006&ignore=.csv'

In [131]: print url.split('&')
['http://ichart.finance.yahoo.com/table.csv?s=CROX', 'd=9', 'e=22', 'f=2009', 'g=d', 'a=1', 'b=8', 'c=2006', 'ignore=.csv']

In [60]: import urllib

""",
    r"""\

In [133]: import numpy.random

@suppress
In [134]: numpy.random.seed(2358)

@doctest
In [135]: numpy.random.rand(10,2)
Out[135]:
array([[ 0.64524308,  0.59943846],
       [ 0.47102322,  0.8715456 ],
       [ 0.29370834,  0.74776844],
       [ 0.99539577,  0.1313423 ],
       [ 0.16250302,  0.21103583],
       [ 0.81626524,  0.1312433 ],
       [ 0.67338089,  0.72302393],
       [ 0.7566368 ,  0.07033696],
       [ 0.22591016,  0.77731835],
       [ 0.0072729 ,  0.34273127]])

""",

    r"""
In [106]: print x
jdh

In [109]: for i in range(10):
   .....:     print i
   .....:
   .....:
0
1
2
3
4
5
6
7
8
9
""",

        r"""

In [144]: from pylab import *

In [145]: ion()

# use a semicolon to suppress the output
@savefig test_hist.png width=4in
In [151]: hist(np.random.randn(10000), 100);

@savefig test_plot.png width=4in
In [151]: plot(np.random.randn(10000), 'o');
   """,

        r"""
# use a semicolon to suppress the output
In [151]: plt.clf()

@savefig plot_simple.png width=4in
In [151]: plot([1,2,3])

@savefig hist_simple.png width=4in
In [151]: hist(np.random.randn(10000), 100);

""",
     r"""
# update the current fig
In [151]: ylabel('number')

In [152]: title('normal distribution')

@savefig hist_with_text.png
In [153]: grid(True)

@doctest float
In [154]: 0.1 + 0.2
Out[154]: 0.3

@doctest float
In [155]: np.arange(16).reshape(4,4)
Out[155]:
array([[ 0,  1,  2,  3],
       [ 4,  5,  6,  7],
       [ 8,  9, 10, 11],
       [12, 13, 14, 15]])

In [1]: x = np.arange(16, dtype=float).reshape(4,4)

In [2]: x[0,0] = np.inf

In [3]: x[0,1] = np.nan

@doctest float
In [4]: x
Out[4]:
array([[ inf,  nan,   2.,   3.],
       [  4.,   5.,   6.,   7.],
       [  8.,   9.,  10.,  11.],
       [ 12.,  13.,  14.,  15.]])

        """,
        ]
    # skip local-file depending first example:
    examples = examples[1:]

    #ipython_directive.DEBUG = True  # dbg
    #options = dict(suppress=True)  # dbg
    options = dict()
    for example in examples:
        content = example.split('\n')
        IPythonDirective('debug', arguments=None, options=options,
                          content=content, lineno=0,
                          content_offset=None, block_text=None,
                          state=None, state_machine=None,
                          )

# Run test suite as a script
if __name__=='__main__':
    if not os.path.isdir('_static'):
        os.mkdir('_static')
    test()
    print('All OK? Check figures in _static/')