Repository: pandas-dev/pandas

Issue:
OverflowError for string of large numbers (int64) while reading json


```python
import json
import pandas as pd
json_data = {"articleId":"1404366058080022500245"}
pd.read_json(json.dumps(json_data), typ='series')

Python int too large to convert to C long: OverflowError
...
File "/var/task/pandas/io/json/json.py", line 366, in read_json
return json_reader.read()
File "/var/task/pandas/io/json/json.py", line 467, in read
obj = self._get_object_parser(self.data)
File "/var/task/pandas/io/json/json.py", line 489, in _get_object_parser
obj = SeriesParser(json, **kwargs).parse()
File "/var/task/pandas/io/json/json.py", line 582, in parse
self._try_convert_types()
File "/var/task/pandas/io/json/json.py", line 752, in _try_convert_types
'data', self.obj, convert_dates=self.convert_dates)
File "/var/task/pandas/io/json/json.py", line 621, in _try_convert_data
new_data, result = self._try_convert_to_date(data)
File "/var/task/pandas/io/json/json.py", line 684, in _try_convert_to_date
new_data = data.astype('int64')
File "/var/task/pandas/util/_decorators.py", line 118, in wrapper
return func(*args, **kwargs)
File "/var/task/pandas/core/generic.py", line 4004, in astype
**kwargs)
File "/var/task/pandas/core/internals.py", line 3462, in astype
return self.apply('astype', dtype=dtype, **kwargs)
File "/var/task/pandas/core/internals.py", line 3329, in apply
applied = getattr(b, f)(**kwargs)
File "/var/task/pandas/core/internals.py", line 544, in astype
**kwargs)
File "/var/task/pandas/core/internals.py", line 625, in _astype
values = astype_nansafe(values.ravel(), dtype, copy=True)
File "/var/task/pandas/core/dtypes/cast.py", line 692, in astype_nansafe
return lib.astype_intsafe(arr.ravel(), dtype).reshape(arr.shape)
File "pandas/_libs/lib.pyx", line 854, in pandas._libs.lib.astype_intsafe
File "pandas/_libs/src/util.pxd", line 91, in util.set_value_at_unsafe
OverflowError: Python int too large to convert to C long
```

Reading json containing stringified numbers larger than int64 causes OverflowError in pandas 0.21.1. Appears to be working fine in 0.21.0

Max





Discussion:
It looks like previous versions would automatically convert that large number to a float, but you're getting an overflow error in 0.21.1 because it is attempting to fit that number into an int.

FWIW you can work around this by explicitly specifying the dtype:

    pd.read_json(json.dumps(json_data), typ='series', dtype={"articleId": np.float64})

Resolving patch:
diff --git a/pandas/io/json/json.py b/pandas/io/json/json.py
--- a/pandas/io/json/json.py
+++ b/pandas/io/json/json.py
@@ -724,7 +724,7 @@ def _try_convert_to_date(self, data):
         if new_data.dtype == 'object':
             try:
                 new_data = data.astype('int64')
-            except (TypeError, ValueError):
+            except (TypeError, ValueError, OverflowError):
                 pass
 
         # ignore numbers that are out of range


Test patch:
diff --git a/pandas/tests/io/json/test_pandas.py b/pandas/tests/io/json/test_pandas.py
--- a/pandas/tests/io/json/test_pandas.py
+++ b/pandas/tests/io/json/test_pandas.py
@@ -1074,6 +1074,20 @@ def test_read_jsonl_unicode_chars(self):
                              columns=['a', 'b'])
         assert_frame_equal(result, expected)
 
+    def test_read_json_large_numbers(self):
+        # GH18842
+        json = '{"articleId": "1404366058080022500245"}'
+        json = StringIO(json)
+        result = read_json(json, typ="series")
+        expected = Series(1.404366e+21, index=['articleId'])
+        assert_series_equal(result, expected)
+
+        json = '{"0": {"articleId": "1404366058080022500245"}}'
+        json = StringIO(json)
+        result = read_json(json)
+        expected = DataFrame(1.404366e+21, index=['articleId'], columns=[0])
+        assert_frame_equal(result, expected)
+
     def test_to_jsonl(self):
         # GH9180
         df = DataFrame([[1, 2], [1, 2]], columns=['a', 'b'])