Repository: yt-project/yt

Issue:
Chunksize does not seem to work in some cases with tipsy
Debugging an issue with @claytonstrawn led to us determining that there is some strange interaction between `CHUNKSIZE` and tipsy that results in particle offsets that are off the end of the file, thus leading to zero-size arrays being read.  It seems that there's potentially an interaction with the smoothing length calculation here as well.

In his case, setting `CHUNKSIZE` to `256**3`, rather than `64**3`, sidestepped the problem -- so my guess is that something with the way that chunksize is splitting the file up is not being carried through correctly in the different chunking operations.


Discussion:
I believe that the logic in `_calculate_particle_offsets` is somehow, subtly, wrong, but I can't figure out why it was done the way that it was done -- which makes me nervous.  Here is the snippet:

```python
    def _calculate_particle_offsets(self, data_file, pcounts):
        field_offsets = {}
        pos = data_file.ds._header_offset
        import pdb;pdb.set_trace()
        for i, ptype in enumerate(self._ptypes):
            if ptype not in self._pdtypes:
                continue
            size = self._pdtypes[ptype].itemsize
            field_offsets[ptype] = pos
            params = self.ds.parameters
            if params[npart_mapping[ptype]] > CHUNKSIZE:
                 for j in range(i):
                     npart = params[npart_mapping[self._ptypes[j]]]
                     spart = self._pdtypes[self._ptypes[j]].itemsize
                     if npart > CHUNKSIZE:
                         field_offsets[ptype] += npart*spart
                 field_offsets[ptype] += data_file.start*size
            pos += data_file.total_particles[ptype] * size
        return field_offsets
```

What I don't understand is why we iterate `for j in range(i)`.  I don't see how this could work for multiple chunks where there are also multiple particle types.  The specific case here is with:

`'nbodies': 5184504, 'nsph': 1430243, 'ndark': 3383411, 'nstar': 370850`

so I believe that the issue is related to the ordering and counts of the particle types.

Resolving patch:
diff --git a/yt/frontends/tipsy/io.py b/yt/frontends/tipsy/io.py
--- a/yt/frontends/tipsy/io.py
+++ b/yt/frontends/tipsy/io.py
@@ -43,7 +43,7 @@ class IOHandlerTipsyBinary(IOHandlerSPH):
     _ptypes = ("Gas",
                "DarkMatter",
                "Stars")
-    _chunksize = 64 * 64 * 64
+    _chunksize = CHUNKSIZE
 
     _aux_fields = None
     _fields = (("Gas", "Mass"),
@@ -118,7 +118,9 @@ def _read_particle_coords(self, chunks, ptf):
             tp = data_file.total_particles
             f = open(data_file.filename, "rb")
             for ptype, field_list in sorted(ptf.items(),
-                                            key=lambda a: poff[a[0]]):
+                                            key=lambda a: poff.get(a[0], -1)):
+                if data_file.total_particles[ptype] == 0:
+                    continue
                 f.seek(poff[ptype])
                 total = 0
                 while total < tp[ptype]:
@@ -190,14 +192,14 @@ def _read_particle_fields(self, chunks, ptf, selector):
                 aux_fh[afield] = open(data_file.filename + '.' + afield, 'rb')
 
             for ptype, field_list in sorted(ptf.items(),
-                                            key=lambda a: poff[a[0]]):
+                                            key=lambda a: poff.get(a[0], -1)):
+                if data_file.total_particles[ptype] == 0:
+                    continue
                 f.seek(poff[ptype])
                 afields = list(set(field_list).intersection(self._aux_fields))
                 count = min(self._chunksize, tp[ptype])
                 p = np.fromfile(f, self._pdtypes[ptype], count=count)
                 auxdata = []
-                if data_file.total_particles[ptype] == 0:
-                    continue
                 for afield in afields:
                     aux_fh[afield].seek(aux_fields_offsets[afield][ptype])
                     if isinstance(self._aux_pdtypes[afield], np.dtype):
@@ -423,22 +425,36 @@ def _identify_fields(self, data_file):
         return self._field_list, {}
 
     def _calculate_particle_offsets(self, data_file, pcounts):
+        # This computes the offsets for each particle type into a "data_file."  Note that 
+        # the term "data_file" here is a bit overloaded, and also refers to a
+        # "chunk" of particles inside a data file.
+        # data_file.start represents the *particle count* that we should start at.
+        #
+        # At this point, data_file will have the total number of particles
+        # that this chunk represents located in the property total_particles.
+        # Because in tipsy files the particles are stored sequentially, we can
+        # figure out where each one starts.
+        # We first figure out the global offsets, then offset them by the count
+        # and size of each individual particle type.
         field_offsets = {}
+        # Initialize pos to the point the first particle type would start
         pos = data_file.ds._header_offset
+        global_offsets = {}
+        field_offsets = {}
         for i, ptype in enumerate(self._ptypes):
             if ptype not in self._pdtypes:
+                # This means we don't have any, I think, and so we shouldn't
+                # stick it in the offsets.
                 continue
+            # Note that much of this will be computed redundantly; future
+            # refactorings could fix this.
+            global_offsets[ptype] = pos
             size = self._pdtypes[ptype].itemsize
-            field_offsets[ptype] = pos
-            params = self.ds.parameters
-            if params[npart_mapping[ptype]] > CHUNKSIZE:
-                 for j in range(i):
-                     npart = params[npart_mapping[self._ptypes[j]]]
-                     spart = self._pdtypes[self._ptypes[j]].itemsize
-                     if npart > CHUNKSIZE:
-                         field_offsets[ptype] += npart*spart
-                 field_offsets[ptype] += data_file.start*size
-            pos += data_file.total_particles[ptype] * size
+            npart = self.ds.parameters[npart_mapping[ptype]]
+            # Get the offset into just this particle type, and start at data_file.start
+            if npart > data_file.start:
+                field_offsets[ptype] = pos + size * data_file.start
+            pos += npart * size
         return field_offsets
 
     def _calculate_particle_offsets_aux(self, data_file):


Test patch: