Repository: yt-project/yt

Issue:
Custom RAMSES gravity files
I have customize some RAMSES simulations and now I have extra gravity fields inside my file grav.out.
Trying to read them with YT causes a bug because, the file size is expected to be smaller.
```
ad['acceleration_x']
ValueError: Size obtained (8) does not match with the expected size (4) of record
```
Could you point out to me where/how to customize YT so that I can handle the new fields ?


Discussion:
Hi,

Many thanks for the report! The issue you are seeing seems to be due to some quantity in your `grav_XXXXX.outYYYYY` files being written in double-precision or 64 bit integer where yt expect them in single precision or 32 bit integer.

Could you provide additional information regarding how you customized the gravity files, and in particular whether the headers differ from the main version? Also, in what precision (single or double) are the fields written in the gravity files?

In principle, if we (yt) have done a good job and your gravity files are regular gravity files with extra fields written in double precision, it is expected it should work out of the box. If your case doesn't fall in these categories, there are many solutions depending on how exactly the gravity files are written! If it falls in these categories, the bug is probably on yt side.
Hello,
I wrote in in single precision however all the gravity quantity were duplicated. Here is the relevant pat of the RAMSES routine output_poisson.f90 that I customized to produce my files  `grav_XXXXX.outYYYYY
```
do ind=1,twotondim
              iskip=ncoarse+(ind-1)*ngridmax
              ! Write potential
              do i=1,ncache
                 xdp(i)=phi(ind_grid(i)+iskip)
                xdp2(i)=phi2(ind_grid(i)+iskip) !second phi
              end do
              write(ilun)xdp
              write(ilun)xdp2
              ! Write force
              do ivar=1,ndim
                 do i=1,ncache
                    xdp(i)=f(ind_grid(i)+iskip,ivar)
                 end do
                 write(ilun)xdp
              end do
              ! Write second force (after all the ndim component of the first force)
              do ivar=1,ndim
                 do i=1,ncache
                    xdp2(i)=f2(ind_grid(i)+iskip,ivar)
                 end do
                 write(ilun)xdp2
              end do
           end do
```
I did not touch neither the header of my gravity files nor the precision.

Resolving patch:
diff --git a/yt/frontends/ramses/field_handlers.py b/yt/frontends/ramses/field_handlers.py
--- a/yt/frontends/ramses/field_handlers.py
+++ b/yt/frontends/ramses/field_handlers.py
@@ -1,6 +1,7 @@
 import abc
 import glob
 import os
+from typing import List
 
 from yt.config import ytcfg
 from yt.funcs import mylog
@@ -244,25 +245,47 @@ def offset(self):
         if getattr(self, "_offset", None) is not None:
             return self._offset
 
+        nvars = len(self.field_list)
         with FortranFile(self.fname) as fd:
-
             # Skip headers
             nskip = len(self.attrs)
             fd.skip(nskip)
             min_level = self.domain.ds.min_level
 
+            # The file is as follows:
+            # > headers
+            # loop over levels
+            #   loop over cpu domains
+            #     > <ilevel>: current level
+            #     > <nocts>: number of octs in level, domain
+            #     loop over <nvars> variables (positions, velocities, density, ...)
+            #       loop over <2*2*2> cells in each oct
+            #          > <data> with shape (nocts, )
+            #
+            # So there are 8 * nvars records each with length (nocts, )
+            # at each (level, cpus)
             offset, level_count = read_offset(
                 fd,
                 min_level,
                 self.domain.domain_id,
                 self.parameters["nvar"],
                 self.domain.amr_header,
+                skip_len=nvars * 8,
             )
 
         self._offset = offset
         self._level_count = level_count
         return self._offset
 
+    @classmethod
+    def load_fields_from_yt_config(cls) -> List[str]:
+        if cls.config_field and ytcfg.has_section(cls.config_field):
+            cfg = ytcfg.get(cls.config_field, "fields")
+            fields = [_.strip() for _ in cfg if _.strip() != ""]
+            return fields
+
+        return []
+
 
 class HydroFieldFileHandler(FieldFileHandler):
     ftype = "ramses"
@@ -304,11 +327,17 @@ def detect_fields(cls, ds):
 
         ok = False
 
-        # Either the fields are given by dataset
         if ds._fields_in_file is not None:
+            # Case 1: fields are provided by users on construction of dataset
             fields = list(ds._fields_in_file)
             ok = True
-        elif os.path.exists(fname_desc):
+        else:
+            # Case 2: fields are provided by users in the config
+            fields = cls.load_fields_from_yt_config()
+            ok = len(fields) > 0
+
+        if not ok and os.path.exists(fname_desc):
+            # Case 3: there is a file descriptor
             # Or there is an hydro file descriptor
             mylog.debug("Reading hydro file descriptor.")
             # For now, we can only read double precision fields
@@ -317,18 +346,8 @@ def detect_fields(cls, ds):
             # We get no fields for old-style hydro file descriptor
             ok = len(fields) > 0
 
-        elif cls.config_field and ytcfg.has_section(cls.config_field):
-            # Or this is given by the config
-            cfg = ytcfg.get(cls.config_field, "fields")
-            known_fields = []
-            for field in (_.strip() for _ in cfg.split("\n") if _.strip() != ""):
-                known_fields.append(field.strip())
-            fields = known_fields
-
-            ok = True
-
-        # Else, attempt autodetection
         if not ok:
+            # Case 4: attempt autodetection with usual fields
             foldername = os.path.abspath(os.path.dirname(ds.parameter_filename))
             rt_flag = any(glob.glob(os.sep.join([foldername, "info_rt_*.txt"])))
             if rt_flag:  # rt run
@@ -460,18 +479,21 @@ def detect_fields(cls, ds):
         nvar = cls.parameters["nvar"]
         ndim = ds.dimensionality
 
-        if nvar == ndim + 1:
-            fields = ["Potential"] + [f"{k}-acceleration" for k in "xyz"[:ndim]]
-        else:
-            fields = [f"{k}-acceleration" for k in "xyz"[:ndim]]
-        ndetected = len(fields)
+        fields = cls.load_fields_from_yt_config()
+
+        if not fields:
+            if nvar == ndim + 1:
+                fields = ["Potential"] + [f"{k}-acceleration" for k in "xyz"[:ndim]]
+            else:
+                fields = [f"{k}-acceleration" for k in "xyz"[:ndim]]
+            ndetected = len(fields)
 
-        if ndetected != nvar and not ds._warned_extra_fields["gravity"]:
-            mylog.warning("Detected %s extra gravity fields.", nvar - ndetected)
-            ds._warned_extra_fields["gravity"] = True
+            if ndetected != nvar and not ds._warned_extra_fields["gravity"]:
+                mylog.info("Detected %s extra gravity fields.", nvar - ndetected)
+                ds._warned_extra_fields["gravity"] = True
 
-            for i in range(nvar - ndetected):
-                fields.append(f"var{i}")
+                for i in range(nvar - ndetected):
+                    fields.append(f"var{i}")
 
         cls.field_list = [(cls.ftype, e) for e in fields]
 
@@ -546,15 +568,18 @@ def read_rhs(cast):
         with FortranFile(fname) as fd:
             cls.parameters = fd.read_attrs(cls.attrs)
 
-        fields = []
-        for ng in range(ngroups):
+        fields = cls.load_fields_from_yt_config()
+
+        if not fields:
+            fields = []
             tmp = [
                 "Photon_density_%s",
                 "Photon_flux_x_%s",
                 "Photon_flux_y_%s",
                 "Photon_flux_z_%s",
             ]
-            fields.extend([t % (ng + 1) for t in tmp])
+            for ng in range(ngroups):
+                fields.extend([t % (ng + 1) for t in tmp])
 
         cls.field_list = [(cls.ftype, e) for e in fields]
 


Test patch:
diff --git a/tests/tests.yaml b/tests/tests.yaml
--- a/tests/tests.yaml
+++ b/tests/tests.yaml
@@ -181,18 +181,19 @@ answer_tests:
 
 other_tests:
   unittests:
-     - '--exclude=test_mesh_slices'  # disable randomly failing test
-     - '--ignore=test_outputs_pytest'
-     - '--ignore-files=test_load_errors.py'
-     - '--ignore-files=test_commons.py'
-     - '--ignore-files=test_line_annotation_unit.py'
-     - '--ignore-files=test_load_sample.py'
-     - '--ignore-files=test_field_access_pytest.py'
-     - '--ignore-files=test_ambiguous_fields.py'
-     - '--ignore-files=test_eps_writer.py'
+     - "--exclude=test_mesh_slices"  # disable randomly failing test
+     - "--ignore=test_outputs_pytest"
+     - "--ignore-files=test_load_errors.py"
+     - "--ignore-files=test_commons.py"
+     - "--ignore-files=test_line_annotation_unit.py"
+     - "--ignore-files=test_load_sample.py"
+     - "--ignore-files=test_field_access_pytest.py"
+     - "--ignore-files=test_ambiguous_fields.py"
+     - "--ignore-files=test_eps_writer.py"
      - "--ignore-files=test_save.py"
-     - '--ignore-files=test_registration.py'
+     - "--ignore-files=test_registration.py"
      - "--ignore-files=test_invalid_origin.py"
-     - '--exclude-test=yt.frontends.gdf.tests.test_outputs.TestGDF'
+     - "--ignore-files=test_outputs_pytest\\.py"
+     - "--exclude-test=yt.frontends.gdf.tests.test_outputs.TestGDF"
   cookbook:
      - 'doc/source/cookbook/tests/test_cookbook.py'
diff --git a/yt/frontends/ramses/tests/test_outputs.py b/yt/frontends/ramses/tests/test_outputs.py
--- a/yt/frontends/ramses/tests/test_outputs.py
+++ b/yt/frontends/ramses/tests/test_outputs.py
@@ -340,34 +340,6 @@ def check_unit(array, unit):
         ytcfg.remove_section("ramses-particles")
 
 
-@requires_file(ramsesCosmo)
-def test_custom_hydro_def():
-    ytcfg.add_section("ramses-hydro")
-    ytcfg[
-        "ramses-hydro", "fields"
-    ] = """
-    Density
-    x-velocity
-    y-velocity
-    z-velocity
-    Foo
-    Bar
-    """
-    ds = yt.load(ramsesCosmo)
-
-    def check_unit(array, unit):
-        assert str(array.in_cgs().units) == unit
-
-    try:
-        assert ("ramses", "Foo") in ds.derived_field_list
-        assert ("ramses", "Bar") in ds.derived_field_list
-
-        check_unit(ds.r["ramses", "Foo"], "dimensionless")
-        check_unit(ds.r["ramses", "Bar"], "dimensionless")
-    finally:
-        ytcfg.remove_section("ramses-hydro")
-
-
 @requires_file(output_00080)
 @requires_file(ramses_sink)
 def test_grav_detection():
diff --git a/yt/frontends/ramses/tests/test_outputs_pytest.py b/yt/frontends/ramses/tests/test_outputs_pytest.py
new file mode 100644
--- /dev/null
+++ b/yt/frontends/ramses/tests/test_outputs_pytest.py
@@ -0,0 +1,57 @@
+from copy import deepcopy
+
+import pytest
+
+import yt
+from yt.config import ytcfg
+from yt.testing import requires_file
+
+output_00080 = "output_00080/info_00080.txt"
+ramses_new_format = "ramses_new_format/output_00002/info_00002.txt"
+
+custom_hydro = [
+    "my-Density",
+    "my-x-velocity",
+    "my-y-velocity",
+    "my-z-velocity",
+    "my-Pressure",
+    "my-Metallicity",
+]
+custom_grav = [
+    "my-x-acceleration",
+    "my-y-acceleration",
+    "my-z-acceleration",
+]
+
+
+@pytest.fixture()
+def custom_ramses_fields_conf():
+    old_config = deepcopy(ytcfg.config_root.as_dict())
+    ytcfg.add_section("ramses-hydro")
+    ytcfg.add_section("ramses-grav")
+    ytcfg.set("ramses-hydro", "fields", custom_hydro)
+    ytcfg.set("ramses-grav", "fields", custom_grav)
+    yield
+    ytcfg.remove_section("ramses-hydro")
+    ytcfg.remove_section("ramses-grav")
+    ytcfg.update(old_config)
+
+
+@requires_file(output_00080)
+def test_field_config_1(custom_ramses_fields_conf):
+    ds = yt.load(output_00080)
+
+    for f in custom_hydro:
+        assert ("ramses", f) in ds.field_list
+    for f in custom_grav:
+        assert ("gravity", f) in ds.field_list
+
+
+@requires_file(ramses_new_format)
+def test_field_config_2(custom_ramses_fields_conf):
+    ds = yt.load(ramses_new_format)
+
+    for f in custom_hydro:
+        assert ("ramses", f) in ds.field_list
+    for f in custom_grav:
+        assert ("gravity", f) in ds.field_list