Repository: yt-project/yt

Issue:
RAMSES key error
### Bug report

**Bug summary**

When creating a derived field for a RAMSES dataset that depends on hydro and rt (any combination as long as the fields aren't stored in the same file), a `KeyError` is raised. This is due to the fact that the code is trying to read all the fields from the same file.

**Code for reproduction**

This requires [ramses_rt_00088](http://yt-project.org/data/ramses_rt_00088.tar.gz). See https://gist.github.com/cphyc/dc445db37e09bf2a89adf50409866f26.

**Version Information**
<!--Please specify your platform and versions of the relevant libraries you are
using:-->
  * Operating System: Arch linux
  * Python Version: 3.6.5
  * yt version: master
  * Other Libraries (if applicable): 



Discussion:
Hey Corentin, just checking do you want help with fixing this?
Hey,
I won't have time to look at this anytime soon, so if any help is welcome!

Resolving patch:
diff --git a/yt/frontends/ramses/io.py b/yt/frontends/ramses/io.py
--- a/yt/frontends/ramses/io.py
+++ b/yt/frontends/ramses/io.py
@@ -96,53 +96,46 @@ def _ramses_particle_file_handler(fname, foffsets, data_types,
 class IOHandlerRAMSES(BaseIOHandler):
     _dataset_type = "ramses"
 
-    def _generic_fluid_handler(self, chunks, selector, fields, size, ftype):
+    def _read_fluid_selection(self, chunks, selector, fields, size):
         tr = defaultdict(list)
 
+        # Set of field types
+        ftypes = set(f[0] for f in fields)
         for chunk in chunks:
-            for subset in chunk.objs:
-                fname = None
-                for fh in subset.domain.field_handlers:
-                    if fh.ftype == ftype:
-                        file_handler = fh
-                        fname = fh.fname
-                        break
-
-                if fname is None:
-                    raise YTFieldTypeNotFound(ftype)
-
-                # Now we read the entire thing
-                with FortranFile(fname) as fd:
-                    # This contains the boundary information, so we skim through
-                    # and pick off the right vectors
-                    rv = subset.fill(fd, fields, selector, file_handler)
-                for ft, f in fields:
-                    d = rv.pop(f)
-                    mylog.debug("Filling %s with %s (%0.3e %0.3e) (%s zones)",
-                        f, d.size, d.min(), d.max(), d.size)
-                    tr[(ft, f)].append(d)
+            # Gather fields by type to minimize i/o operations
+            for ft in ftypes:
+                # Get all the fields of the same type
+                field_subs = list(
+                    filter(lambda f: f[0]==ft, fields))
+
+                # Loop over subsets
+                for subset in chunk.objs:
+                    fname = None
+                    for fh in subset.domain.field_handlers:
+                        if fh.ftype == ft:
+                            file_handler = fh
+                            fname = fh.fname
+                            break
+
+                    if fname is None:
+                        raise YTFieldTypeNotFound(ft)
+
+                    # Now we read the entire thing
+                    with FortranFile(fname) as fd:
+                        # This contains the boundary information, so we skim through
+                        # and pick off the right vectors
+                        rv = subset.fill(fd, field_subs, selector, file_handler)
+                    for ft, f in field_subs:
+                        d = rv.pop(f)
+                        mylog.debug("Filling %s with %s (%0.3e %0.3e) (%s zones)",
+                            f, d.size, d.min(), d.max(), d.size)
+                        tr[(ft, f)].append(d)
         d = {}
         for field in fields:
             d[field] = np.concatenate(tr.pop(field))
 
         return d
 
-    def _read_fluid_selection(self, chunks, selector, fields, size):
-        d = {}
-
-        # Group fields by field type
-        for ft in set(f[0] for f in fields):
-            # Select the fields for the current reader
-            fields_subs = list(
-                filter(lambda f: f[0]==ft,
-                       fields))
-
-            newd = self._generic_fluid_handler(chunks, selector, fields_subs, size,
-                                               ft)
-            d.update(newd)
-
-        return d
-
     def _read_particle_coords(self, chunks, ptf):
         pn = "particle_position_%s"
         fields = [(ptype, "particle_position_%s" % ax)


Test patch:
diff --git a/yt/frontends/ramses/tests/test_outputs.py b/yt/frontends/ramses/tests/test_outputs.py
--- a/yt/frontends/ramses/tests/test_outputs.py
+++ b/yt/frontends/ramses/tests/test_outputs.py
@@ -412,3 +412,17 @@ def check_unit(array, unit):
     check_unit(ds.r[('gas','number_density')],'cm**(-3)')
     check_unit(ds.r[('gas','mu')],'dimensionless')
     check_unit(ds.r[('gas','Electron_number_density')],'cm**(-3)')
+
+
+ramses_rt = "ramses_rt_00088/output_00088/info_00088.txt"
+@requires_file(ramses_rt)
+def test_ramses_mixed_files():
+    # Test that one can use derived fields that depend on different
+    # files (here hydro and rt files)
+    ds = yt.load(ramses_rt)
+    def _mixed_field(field, data):
+        return data['rt', 'photon_density_1'] / data['gas', 'H_nuclei_density']
+    ds.add_field(('gas', 'mixed_files'), function=_mixed_field, sampling_type='cell')
+
+    # Access the field
+    ds.r[('gas', 'mixed_files')]