Repository: pandas-dev/pandas

Issue:
BUG: read_csv fails with uint64
`master` at <a href="https://github.com/pandas-dev/pandas/commit/aba7d255a165bbc221be106839c59a876655c0c2">aba7d2</a>:
~~~python
>>> from pandas import read_csv
>>> from pandas.compat import StringIO
>>> data = 'a\n' + str(2**63)
>>>
>>> read_csv(StringIO(data), engine='c').info()
<class 'pandas.core.frame.DataFrame'>
RangeIndex: 1 entries, 0 to 0
Data columns (total 1 columns):
a    1 non-null object
dtypes: object(1)
memory usage: 88.0+ bytes
>>>
>>> read_csv(StringIO(data), engine='python').info()
<class 'pandas.core.frame.DataFrame'>
RangeIndex: 1 entries, 0 to 0
Data columns (total 1 columns):
a    1 non-null object
dtypes: object(1)
memory usage: 88.0+ bytes
~~~

We should be able to handle `uint64`, and tests like this one <a href="https://github.com/pandas-dev/pandas/blob/aba7d255a165bbc221be106839c59a876655c0c2/pandas/io/tests/parser/common.py#L937">here</a> should not be enforcing buggy behavior.

The buggy behavior for the C engine traces to <a href="https://github.com/pandas-dev/pandas/blob/master/pandas/parser.pyx#L1126">here</a>, where we attempt to cast according to this order defined <a href="https://github.com/pandas-dev/pandas/blob/master/pandas/parser.pyx#L417">here</a>.  Note for starters that `uint64` is not in that list.  This `try-except` is due to `OverflowError` with `int64`, after which we immediately convert to an `object` array of strings.  At first, I thought inserting `uint64` to the list would be good, but that can cause bad casting in the other direction, i.e. negative numbers get converted to their `uint64` equivalents.

The buggy behavior for the Python engine traces to <a href="https://github.com/pandas-dev/pandas/blob/master/pandas/io/parsers.py#L1345">here</a>, where we attempt to infer the `dtype` <a href="https://github.com/pandas-dev/pandas/blob/master/pandas/io/parsers.py#L1392">here</a>.  However, as I pointed out in #14982, this function fails with `uint64` with a similar (and non-sensical) `try-except` for `OverflowError` in `int64`.

The questions that I posed in #14982 are also relevant here, since they should be consistent across both engines that also is performant.  Patching the Python engine probably requires fixing #14982 first, and patching the C engine probably requires adding new functions to `parser.pyx` to parse `uint64` and `tokenizer.c`.  However, in light of the questions that I posed in #14982, I'm not really sure what is best.


Discussion:
I think response in #14982 answers this. Key idea is to make sure this is performant though.
also in this issue can make sure that passing ``dtype='uint64'`` works properly (e.g. explict user casting)

Resolving patch:
diff --git a/asv_bench/benchmarks/io_bench.py b/asv_bench/benchmarks/io_bench.py
--- a/asv_bench/benchmarks/io_bench.py
+++ b/asv_bench/benchmarks/io_bench.py
@@ -128,6 +128,29 @@ def time_read_parse_dates_iso8601(self):
         read_csv(StringIO(self.data), header=None, names=['foo'], parse_dates=['foo'])
 
 
+class read_uint64_integers(object):
+    goal_time = 0.2
+
+    def setup(self):
+        self.na_values = [2**63 + 500]
+
+        self.arr1 = np.arange(10000).astype('uint64') + 2**63
+        self.data1 = '\n'.join(map(lambda x: str(x), self.arr1))
+
+        self.arr2 = self.arr1.copy().astype(object)
+        self.arr2[500] = -1
+        self.data2 = '\n'.join(map(lambda x: str(x), self.arr2))
+
+    def time_read_uint64(self):
+        read_csv(StringIO(self.data1), header=None)
+
+    def time_read_uint64_neg_values(self):
+        read_csv(StringIO(self.data2), header=None)
+
+    def time_read_uint64_na_values(self):
+        read_csv(StringIO(self.data1), header=None, na_values=self.na_values)
+
+
 class write_csv_standard(object):
     goal_time = 0.2
 


Test patch:
diff --git a/pandas/io/tests/parser/common.py b/pandas/io/tests/parser/common.py
--- a/pandas/io/tests/parser/common.py
+++ b/pandas/io/tests/parser/common.py
@@ -921,29 +921,39 @@ def test_int64_overflow(self):
             self.assertRaises(OverflowError, self.read_csv,
                               StringIO(data), converters={'ID': conv})
 
-        # These numbers fall right inside the int64 range,
+        # These numbers fall right inside the int64-uint64 range,
         # so they should be parsed as string.
+        ui_max = np.iinfo(np.uint64).max
         i_max = np.iinfo(np.int64).max
         i_min = np.iinfo(np.int64).min
 
-        for x in [i_max, i_min]:
+        for x in [i_max, i_min, ui_max]:
             result = self.read_csv(StringIO(str(x)), header=None)
             expected = DataFrame([x])
             tm.assert_frame_equal(result, expected)
 
-        # These numbers fall just outside the int64 range,
+        # These numbers fall just outside the int64-uint64 range,
         # so they should be parsed as string.
-        too_big = i_max + 1
+        too_big = ui_max + 1
         too_small = i_min - 1
 
         for x in [too_big, too_small]:
             result = self.read_csv(StringIO(str(x)), header=None)
-            if self.engine == 'python' and x == too_big:
-                expected = DataFrame([x])
-            else:
-                expected = DataFrame([str(x)])
+            expected = DataFrame([str(x)])
             tm.assert_frame_equal(result, expected)
 
+        # No numerical dtype can hold both negative and uint64 values,
+        # so they should be cast as string.
+        data = '-1\n' + str(2**63)
+        expected = DataFrame([str(-1), str(2**63)])
+        result = self.read_csv(StringIO(data), header=None)
+        tm.assert_frame_equal(result, expected)
+
+        data = str(2**63) + '\n-1'
+        expected = DataFrame([str(2**63), str(-1)])
+        result = self.read_csv(StringIO(data), header=None)
+        tm.assert_frame_equal(result, expected)
+
     def test_empty_with_nrows_chunksize(self):
         # see gh-9535
         expected = DataFrame([], columns=['foo', 'bar'])
diff --git a/pandas/io/tests/parser/dtypes.py b/pandas/io/tests/parser/dtypes.py
--- a/pandas/io/tests/parser/dtypes.py
+++ b/pandas/io/tests/parser/dtypes.py
@@ -275,3 +275,11 @@ def test_empty_dtype(self):
         result = self.read_csv(StringIO(data), header=0,
                                dtype={'a': np.int32, 1: np.float64})
         tm.assert_frame_equal(result, expected)
+
+    def test_numeric_dtype(self):
+        data = '0\n1'
+
+        for dt in np.typecodes['AllInteger'] + np.typecodes['Float']:
+            expected = pd.DataFrame([0, 1], dtype=dt)
+            result = self.read_csv(StringIO(data), header=None, dtype=dt)
+            tm.assert_frame_equal(expected, result)
diff --git a/pandas/io/tests/parser/na_values.py b/pandas/io/tests/parser/na_values.py
--- a/pandas/io/tests/parser/na_values.py
+++ b/pandas/io/tests/parser/na_values.py
@@ -289,3 +289,17 @@ def test_na_values_dict_col_index(self):
         out = self.read_csv(StringIO(data), na_values=na_values)
         expected = DataFrame({'a': [np.nan, 1]})
         tm.assert_frame_equal(out, expected)
+
+    def test_na_values_uint64(self):
+        # see gh-14983
+
+        na_values = [2**63]
+        data = str(2**63) + '\n' + str(2**63 + 1)
+        expected = DataFrame([str(2**63), str(2**63 + 1)])
+        out = self.read_csv(StringIO(data), header=None, na_values=na_values)
+        tm.assert_frame_equal(out, expected)
+
+        data = str(2**63) + ',1' + '\n,2'
+        expected = DataFrame([[str(2**63), 1], ['', 2]])
+        out = self.read_csv(StringIO(data), header=None)
+        tm.assert_frame_equal(out, expected)