Repository: yt-project/yt

Issue:
Big endian error for the Gadget data
### Bug report

**Bug summary**

This is a big endian convert error for the gadget data in the latest version of yt.

**Code for reproduction**
```python
In [1]: import yt

In [2]: yt.__version__
Out[2]: '3.5.dev0'

In [3]: ds=yt.load("/tmp/BigEndianGadgetBinary/BigEndianGadgetBinary")
yt : [INFO     ] 2018-12-07 15:42:38,876 Calculating time from 9.500e-01 to be 4.152e+17 seconds
yt : [INFO     ] 2018-12-07 15:42:38,877 Assuming length units are in kpc/h (comoving)
yt : [INFO     ] 2018-12-07 15:42:38,924 Parameters: current_time              = 4.152191296890778e+17 s
yt : [INFO     ] 2018-12-07 15:42:38,925 Parameters: domain_dimensions         = [2 2 2]
yt : [INFO     ] 2018-12-07 15:42:38,925 Parameters: domain_left_edge          = [0. 0. 0.]
yt : [INFO     ] 2018-12-07 15:42:38,925 Parameters: domain_right_edge         = [1000000. 1000000. 1000000.]
yt : [INFO     ] 2018-12-07 15:42:38,925 Parameters: cosmological_simulation   = 1
yt : [INFO     ] 2018-12-07 15:42:38,926 Parameters: current_redshift          = 0.0526315833812
yt : [INFO     ] 2018-12-07 15:42:38,926 Parameters: omega_lambda              = 0.73
yt : [INFO     ] 2018-12-07 15:42:38,926 Parameters: omega_matter              = 0.27
yt : [INFO     ] 2018-12-07 15:42:38,926 Parameters: hubble_constant           = 0.7

In [4]: data=ds.all_data()

In [5]: data['Halo', 'Velocities']
yt : [INFO     ] 2018-12-07 15:42:52,382 Allocating for 2.210e+07 particles (index particle type 'all')
yt : [INFO     ] 2018-12-07 15:42:57,871 Identified 1.548e+06 octs
---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-5-59be19dacdd8> in <module>()
----> 1 data['Halo', 'Velocities']

/home/weiguang/.local/lib/python2.7/site-packages/yt-3.6.dev0-py2.7-linux-x86_64.egg/yt/data_objects/data_containers.pyc in __getitem__(self, key)
    253                 return self.field_data[f]
    254             else:
--> 255                 self.get_data(f)
    256         # fi.units is the unit expression string. We depend on the registry
    257         # hanging off the dataset to define this unit object.

/home/weiguang/.local/lib/python2.7/site-packages/yt-3.6.dev0-py2.7-linux-x86_64.egg/yt/data_objects/data_containers.pyc in get_data(self, fields)
   1374 
   1375         read_particles, gen_particles = self.index._read_particle_fields(
-> 1376                                         particles, self, self._current_chunk)
   1377         for f, v in read_particles.items():
   1378             self.field_data[f] = self.ds.arr(v, input_units = finfos[f].units)

/home/weiguang/.local/lib/python2.7/site-packages/yt-3.6.dev0-py2.7-linux-x86_64.egg/yt/geometry/geometry_handler.pyc in _read_particle_fields(self, fields, dobj, chunk)
    225             self._chunk_io(dobj, cache = False),
    226             selector,
--> 227             fields_to_read)
    228         return fields_to_return, fields_to_generate
    229 

/home/weiguang/.local/lib/python2.7/site-packages/yt-3.6.dev0-py2.7-linux-x86_64.egg/yt/utilities/io_handler.pyc in _read_particle_selection(self, chunks, selector, fields)
    197         # Here, ptype_map means which particles contribute to a given type.
    198         # And ptf is the actual fields from disk to read.
--> 199         psize = self._count_particles_chunks(chunks, ptf, selector)
    200         # Now we allocate
    201         # ptf, remember, is our mapping of what we want to read

/home/weiguang/.local/lib/python2.7/site-packages/yt-3.6.dev0-py2.7-linux-x86_64.egg/yt/utilities/io_handler.pyc in _count_particles_chunks(self, chunks, ptf, selector)
    169         psize = defaultdict(lambda: 0) # COUNT PTYPES ON DISK
    170         for ptype, (x, y, z) in self._read_particle_coords(chunks, ptf):
--> 171             psize[ptype] += selector.count_points(x, y, z, 0.0)
    172         return dict(psize.items())
    173 

/home/weiguang/.local/lib/python2.7/site-packages/yt-3.6.dev0-py2.7-linux-x86_64.egg/yt/geometry/selection_routines.pyx in yt.geometry.selection_routines.SelectorObject.count_points (yt/geometry/selection_routines.c:11940)()
    549     @cython.wraparound(False)
    550     @cython.cdivision(True)
--> 551     def count_points(self, np.ndarray[floating, ndim=1] x,
    552                            np.ndarray[floating, ndim=1] y,
    553                            np.ndarray[floating, ndim=1] z,

ValueError: Big-endian buffer not supported on little-endian compiler
```

**Version Information**
<!--Please specify your platform and versions of the relevant libraries you are
using:-->
  * Operating System: Ubuntu
  * Python Version: 2.7
  * yt version: 3.5dev (but it should be the latest one downloaded from github, and installed from source) 




Discussion:
Argh, sorry about the version number confusion, see https://github.com/yt-project/yt/pull/2118.

Thanks for the report!

Resolving patch:
diff --git a/yt/frontends/gadget/io.py b/yt/frontends/gadget/io.py
--- a/yt/frontends/gadget/io.py
+++ b/yt/frontends/gadget/io.py
@@ -270,8 +270,8 @@ def _read_particle_coords(self, chunks, ptf):
             for ptype in ptf:
                 # This is where we could implement sub-chunking
                 f.seek(poff[ptype, "Coordinates"], os.SEEK_SET)
-                pos = self._read_field_from_file(f,
-                                                 tp[ptype], "Coordinates")
+                pos = self._read_field_from_file(
+                    f, tp[ptype], "Coordinates")
                 yield ptype, (pos[:, 0], pos[:, 1], pos[:, 2])
             f.close()
 
@@ -286,8 +286,8 @@ def _read_particle_fields(self, chunks, ptf, selector):
             f = open(data_file.filename, "rb")
             for ptype, field_list in sorted(ptf.items()):
                 f.seek(poff[ptype, "Coordinates"], os.SEEK_SET)
-                pos = self._read_field_from_file(f,
-                                                 tp[ptype], "Coordinates")
+                pos = self._read_field_from_file(
+                    f, tp[ptype], "Coordinates")
                 mask = selector.select_points(
                     pos[:, 0], pos[:, 1], pos[:, 2], 0.0)
                 del pos
@@ -314,9 +314,14 @@ def _read_field_from_file(self, f, count, name):
             dt = self._endian + "u4"
         else:
             dt = self._endian + self._float_type
+        dt = np.dtype(dt)
         if name in self._vector_fields:
             count *= self._vector_fields[name]
         arr = np.fromfile(f, dtype=dt, count=count)
+        # ensure data are in native endianness to avoid errors
+        # when field data are passed to cython
+        dt = dt.newbyteorder('N')
+        arr = arr.astype(dt)
         if name in self._vector_fields:
             factor = self._vector_fields[name]
             arr = arr.reshape((count // factor, factor), order="C")


Test patch:
diff --git a/yt/frontends/gadget/tests/test_outputs.py b/yt/frontends/gadget/tests/test_outputs.py
--- a/yt/frontends/gadget/tests/test_outputs.py
+++ b/yt/frontends/gadget/tests/test_outputs.py
@@ -108,3 +108,9 @@ def test_pid_uniqueness():
     ad = ds.all_data()
     pid = ad['ParticleIDs']
     assert len(pid) == len(set(pid.v))
+
+@requires_ds(BE_Gadget)
+def test_bigendian_field_access():
+    ds = data_dir_load(BE_Gadget)
+    data = ds.all_data()
+    data['Halo', 'Velocities']