Repository: numpy/numpy

Issue:
recarray fromfile doesn't support file like objects (Trac #1911)
_Original ticket http://projects.scipy.org/numpy/ticket/1911 on 2011-07-18 by trac user sbreuss, assigned to unknown._

The current implementation of the record array fromfile method doesn't support file like objects (e.g. StringIO, io.BytesIO, ...)

By two minor changes in the source this would be possible:
1) change the get_remaining_size method to use seek
2) not using the readinto method of the file object

I'm open to any suggestions, a patch for the current master version showing my changes is attached.

Cheers,
Simon Breuss



Discussion:
Attachment added by trac user sbreuss on 2011-07-18: [records.py.diff](http://projects.scipy.org/numpy/attachment/ticket/1911/records.py.diff)

Note that even for simple float arrays text fromfile requires an actual file descriptor, so it doesn't work with StringIO.

Is this also related to #6246?

I don't think so. I think this is an issue about record arrays, binary files, and seeking, so there's no reason it shouldn't be fixable to work with BytesIO. 

The following bug looks to be related: fromfile can take an io class object on Python 3, but not on Python 2. Specifically, the following code runs perfectly on Python 3.4, but produces an IOError on Python 2.7. I'm using numpy 1.9.2. Originally reported at https://github.com/mcmtroffaes/pathlib2/pull/7 by @scienceopen

``` python
import numpy as np
import io
np.array([1.,2.,3.]).tofile("test.dat")
with io.open("test.dat", mode='rb') as f:
    print(np.fromfile(f))
```

EDIT: also see https://github.com/numpy/numpy/pull/6632

Is there intention to fix this?
Another change needed here - `isfileobj` would need to become `hasattr(fd, 'readinto')`, which is more consistent with some other parts of numpy.

Supporting `io.StringIO` is a bad idea, only `io.BytesIO` should be supported.



Resolving patch:
diff --git a/numpy/core/records.py b/numpy/core/records.py
--- a/numpy/core/records.py
+++ b/numpy/core/records.py
@@ -40,7 +40,7 @@
 from . import numeric as sb
 from . import numerictypes as nt
 from numpy.compat import (
-    isfileobj, os_fspath, contextlib_nullcontext
+    os_fspath, contextlib_nullcontext
 )
 from numpy.core.overrides import set_module
 from .arrayprint import get_printoptions
@@ -847,13 +847,12 @@ def fromstring(datastring, dtype=None, shape=None, offset=0, formats=None,
     return _array
 
 def get_remaining_size(fd):
+    pos = fd.tell()
     try:
-        fn = fd.fileno()
-    except AttributeError:
-        return os.path.getsize(fd.name) - fd.tell()
-    st = os.fstat(fn)
-    size = st.st_size - fd.tell()
-    return size
+        fd.seek(0, 2)
+        return fd.tell() - pos
+    finally:
+        fd.seek(pos, 0)
 
 def fromfile(fd, dtype=None, shape=None, offset=0, formats=None,
              names=None, titles=None, aligned=False, byteorder=None):
@@ -911,7 +910,9 @@ def fromfile(fd, dtype=None, shape=None, offset=0, formats=None,
     elif isinstance(shape, int):
         shape = (shape,)
 
-    if isfileobj(fd):
+    if hasattr(fd, 'readinto'):
+        # GH issue 2504. fd supports io.RawIOBase or io.BufferedIOBase interface. 
+        # Example of fd: gzip, BytesIO, BufferedReader
         # file already opened
         ctx = contextlib_nullcontext(fd)
     else:
@@ -1036,7 +1037,7 @@ def array(obj, dtype=None, shape=None, offset=0, strides=None, formats=None,
     array('def', dtype='<U3')
     """
 
-    if ((isinstance(obj, (type(None), str)) or isfileobj(obj)) and
+    if ((isinstance(obj, (type(None), str)) or hasattr(obj, 'readinto')) and
            formats is None and dtype is None):
         raise ValueError("Must define formats (or dtype) if object is "
                          "None, string, or an open file")
@@ -1078,7 +1079,7 @@ def array(obj, dtype=None, shape=None, offset=0, strides=None, formats=None,
             new = new.copy()
         return new
 
-    elif isfileobj(obj):
+    elif hasattr(obj, 'readinto'):
         return fromfile(obj, dtype=dtype, shape=shape, offset=offset)
 
     elif isinstance(obj, ndarray):


Test patch:
diff --git a/numpy/core/tests/test_records.py b/numpy/core/tests/test_records.py
--- a/numpy/core/tests/test_records.py
+++ b/numpy/core/tests/test_records.py
@@ -1,5 +1,6 @@
 import collections.abc
 import textwrap
+from io import BytesIO
 from os import path
 from pathlib import Path
 import pytest
@@ -79,8 +80,14 @@ def test_recarray_fromfile(self):
         r1 = np.rec.fromfile(fd, formats='f8,i4,a5', shape=3, byteorder='big')
         fd.seek(2880 * 2)
         r2 = np.rec.array(fd, formats='f8,i4,a5', shape=3, byteorder='big')
+        fd.seek(2880 * 2)
+        bytes_array = BytesIO()
+        bytes_array.write(fd.read())
+        bytes_array.seek(0)
+        r3 = np.rec.fromfile(bytes_array, formats='f8,i4,a5', shape=3, byteorder='big')
         fd.close()
         assert_equal(r1, r2)
+        assert_equal(r2, r3)
 
     def test_recarray_from_obj(self):
         count = 10