Repository: GridTools/gt4py
Pull request #1470 (2024-02-26 14:12:23)

refactor[next]: Refactor workflow based executors and move the backend class.
## Description

### Changed

- `OTFCompileExecutor` and `CachedOTFCompileExecutor` merged into `ModularExecutor`

### Moved

- `OTFBackend` moved and renamed to `next.backend.Backend`

### Reasoning

The two `*CompileExecutor` classes were identical, except for type hints. Since they are now almost exclusively used inside `OTFBackend`, which does not retain typing information about which of them it contains, this distinction is no longer helpful for static type checking.

`OTFBackend` has always been more general than it's naming and location suggested. It is the de-facto definition of a backend within `gt4py.next`: a wrapper around an executor and an allocator. This change makes the status quo visible.

## Requirements

- [x] All fixes and/or new features come with corresponding tests.
- [x] Important design decisions have been documented in the approriate ADR inside the [docs/development/ADRs/](docs/development/ADRs/Index.md) folder.


Patch:
diff --git a/src/gt4py/next/backend.py b/src/gt4py/next/backend.py
new file mode 100644
index 00000000..33fe96e7
--- /dev/null
+++ b/src/gt4py/next/backend.py
@@ -0,0 +1,46 @@
+# GT4Py - GridTools Framework
+#
+# Copyright (c) 2014-2023, ETH Zurich
+# All rights reserved.
+#
+# This file is part of the GT4Py project and the GridTools framework.
+# GT4Py is free software: you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the
+# Free Software Foundation, either version 3 of the License, or any later
+# version. See the LICENSE.txt file at the top-level directory of this
+# distribution for a copy of the license or check <https://www.gnu.org/licenses/>.
+#
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from __future__ import annotations
+
+import dataclasses
+from typing import Any, Generic
+
+from gt4py._core import definitions as core_defs
+from gt4py.next import allocators as next_allocators
+from gt4py.next.iterator import ir as itir
+from gt4py.next.program_processors import processor_interface as ppi
+
+
+@dataclasses.dataclass(frozen=True)
+class Backend(Generic[core_defs.DeviceTypeT]):
+    executor: ppi.ProgramExecutor
+    allocator: next_allocators.FieldBufferAllocatorProtocol[core_defs.DeviceTypeT]
+
+    def __call__(self, program: itir.FencilDefinition, *args, **kwargs: Any) -> None:
+        self.executor.__call__(program, *args, **kwargs)
+
+    @property
+    def __name__(self) -> str:
+        return getattr(self.executor, "__name__", None) or repr(self)
+
+    @property
+    def kind(self) -> type[ppi.ProgramExecutor]:
+        return self.executor.kind
+
+    @property
+    def __gt_allocator__(
+        self,
+    ) -> next_allocators.FieldBufferAllocatorProtocol[core_defs.DeviceTypeT]:
+        return self.allocator
diff --git a/src/gt4py/next/program_processors/modular_executor.py b/src/gt4py/next/program_processors/modular_executor.py
new file mode 100644
index 00000000..19708a65
--- /dev/null
+++ b/src/gt4py/next/program_processors/modular_executor.py
@@ -0,0 +1,43 @@
+# GT4Py - GridTools Framework
+#
+# Copyright (c) 2014-2023, ETH Zurich
+# All rights reserved.
+#
+# This file is part of the GT4Py project and the GridTools framework.
+# GT4Py is free software: you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the
+# Free Software Foundation, either version 3 of the License, or any later
+# version. See the LICENSE.txt file at the top-level directory of this
+# distribution for a copy of the license or check <https://www.gnu.org/licenses/>.
+#
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from __future__ import annotations
+
+import dataclasses
+from typing import Any, Optional, TypeVar
+
+import gt4py.next.iterator.ir as itir
+import gt4py.next.program_processors.processor_interface as ppi
+from gt4py.next.otf import languages, stages, workflow
+
+
+SrcL = TypeVar("SrcL", bound=languages.NanobindSrcL)
+TgtL = TypeVar("TgtL", bound=languages.LanguageTag)
+LS = TypeVar("LS", bound=languages.LanguageSettings)
+HashT = TypeVar("HashT")
+
+
+@dataclasses.dataclass(frozen=True)
+class ModularExecutor(ppi.ProgramExecutor):
+    otf_workflow: workflow.Workflow[stages.ProgramCall, stages.CompiledProgram]
+    name: Optional[str] = None
+
+    def __call__(self, program: itir.FencilDefinition, *args, **kwargs: Any) -> None:
+        self.otf_workflow(stages.ProgramCall(program, args, kwargs))(
+            *args, offset_provider=kwargs["offset_provider"]
+        )
+
+    @property
+    def __name__(self) -> str:
+        return self.name or repr(self)
diff --git a/src/gt4py/next/program_processors/otf_compile_executor.py b/src/gt4py/next/program_processors/otf_compile_executor.py
deleted file mode 100644
index 8dff34a3..00000000
--- a/src/gt4py/next/program_processors/otf_compile_executor.py
+++ /dev/null
@@ -1,83 +0,0 @@
-# GT4Py - GridTools Framework
-#
-# Copyright (c) 2014-2023, ETH Zurich
-# All rights reserved.
-#
-# This file is part of the GT4Py project and the GridTools framework.
-# GT4Py is free software: you can redistribute it and/or modify it under
-# the terms of the GNU General Public License as published by the
-# Free Software Foundation, either version 3 of the License, or any later
-# version. See the LICENSE.txt file at the top-level directory of this
-# distribution for a copy of the license or check <https://www.gnu.org/licenses/>.
-#
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-from __future__ import annotations
-
-import dataclasses
-from typing import Any, Generic, Optional, TypeVar
-
-import gt4py._core.definitions as core_defs
-import gt4py.next.allocators as next_allocators
-import gt4py.next.iterator.ir as itir
-import gt4py.next.program_processors.processor_interface as ppi
-from gt4py.next.otf import languages, recipes, stages, workflow
-
-
-SrcL = TypeVar("SrcL", bound=languages.NanobindSrcL)
-TgtL = TypeVar("TgtL", bound=languages.LanguageTag)
-LS = TypeVar("LS", bound=languages.LanguageSettings)
-HashT = TypeVar("HashT")
-
-
-@dataclasses.dataclass(frozen=True)
-class OTFCompileExecutor(ppi.ProgramExecutor):
-    otf_workflow: recipes.OTFCompileWorkflow
-    name: Optional[str] = None
-
-    def __call__(self, program: itir.FencilDefinition, *args, **kwargs: Any) -> None:
-        self.otf_workflow(stages.ProgramCall(program, args, kwargs))(
-            *args, offset_provider=kwargs["offset_provider"]
-        )
-
-    @property
-    def __name__(self) -> str:
-        return self.name or repr(self)
-
-
-@dataclasses.dataclass(frozen=True)
-class CachedOTFCompileExecutor(ppi.ProgramExecutor, Generic[HashT]):
-    otf_workflow: workflow.CachedStep[stages.ProgramCall, stages.CompiledProgram, HashT]
-    name: Optional[str] = None
-
-    def __call__(self, program: itir.FencilDefinition, *args, **kwargs: Any) -> None:
-        self.otf_workflow(stages.ProgramCall(program, args, kwargs))(
-            *args, offset_provider=kwargs["offset_provider"]
-        )
-
-    @property
-    def __name__(self) -> str:
-        return self.name or repr(self)
-
-
-@dataclasses.dataclass(frozen=True)
-class OTFBackend(Generic[core_defs.DeviceTypeT]):
-    executor: ppi.ProgramExecutor
-    allocator: next_allocators.FieldBufferAllocatorProtocol[core_defs.DeviceTypeT]
-
-    def __call__(self, program: itir.FencilDefinition, *args, **kwargs: Any) -> None:
-        self.executor.__call__(program, *args, **kwargs)
-
-    @property
-    def __name__(self) -> str:
-        return getattr(self.executor, "__name__", None) or repr(self)
-
-    @property
-    def kind(self) -> type[ppi.ProgramExecutor]:
-        return self.executor.kind
-
-    @property
-    def __gt_allocator__(
-        self,
-    ) -> next_allocators.FieldBufferAllocatorProtocol[core_defs.DeviceTypeT]:
-        return self.allocator
diff --git a/src/gt4py/next/program_processors/runners/dace_iterator/__init__.py b/src/gt4py/next/program_processors/runners/dace_iterator/__init__.py
index 9fe19ee2..ae0cfbfe 100644
--- a/src/gt4py/next/program_processors/runners/dace_iterator/__init__.py
+++ b/src/gt4py/next/program_processors/runners/dace_iterator/__init__.py
@@ -25,9 +25,8 @@ from dace.transformation.auto import auto_optimize as autoopt
 
 import gt4py.next.allocators as next_allocators
 import gt4py.next.iterator.ir as itir
-import gt4py.next.program_processors.otf_compile_executor as otf_exec
 import gt4py.next.program_processors.processor_interface as ppi
-from gt4py.next import common
+from gt4py.next import backend, common
 from gt4py.next.iterator import transforms as itir_transforms
 from gt4py.next.otf.compilation import cache as compilation_cache
 from gt4py.next.type_system import type_specifications as ts, type_translation
@@ -42,38 +41,37 @@ except ImportError:
     cp = None
 
 
-def get_sorted_dim_ranges(domain: common.Domain) -> Sequence[common.FiniteUnitRange]:
-    assert common.Domain.is_finite(domain)
-    sorted_dims = get_sorted_dims(domain.dims)
-    return [domain.ranges[dim_index] for dim_index, _ in sorted_dims]
-
-
 """ Default build configuration in DaCe backend """
 _build_type = "Release"
-
-
-def convert_arg(arg: Any, sdfg_param: str):
-    if common.is_field(arg):
-        # field domain offsets are not supported
-        non_zero_offsets = [
-            (dim, dim_range)
-            for dim, dim_range in zip(arg.domain.dims, arg.domain.ranges)
-            if dim_range.start != 0
-        ]
-        if non_zero_offsets:
-            dim, dim_range = non_zero_offsets[0]
-            raise RuntimeError(
-                f"Field '{sdfg_param}' passed as array slice with offset {dim_range.start} on dimension {dim.value}."
-            )
-        sorted_dims = get_sorted_dims(arg.domain.dims)
-        ndim = len(sorted_dims)
-        dim_indices = [dim_index for dim_index, _ in sorted_dims]
-        if isinstance(arg.ndarray, np.ndarray):
-            return np.moveaxis(arg.ndarray, range(ndim), dim_indices)
-        else:
-            assert cp is not None and isinstance(arg.ndarray, cp.ndarray)
-            return cp.moveaxis(arg.ndarray, range(ndim), dim_indices)
-    return arg
+_default_on_gpu = False
+_default_use_field_canonical_representation = False
+
+
+def convert_arg(arg: Any, sdfg_param: str, use_field_canonical_representation: bool):
+    if not common.is_field(arg):
+        return arg
+    # field domain offsets are not supported
+    non_zero_offsets = [
+        (dim, dim_range)
+        for dim, dim_range in zip(arg.domain.dims, arg.domain.ranges)
+        if dim_range.start != 0
+    ]
+    if non_zero_offsets:
+        dim, dim_range = non_zero_offsets[0]
+        raise RuntimeError(
+            f"Field '{sdfg_param}' passed as array slice with offset {dim_range.start} on dimension {dim.value}."
+        )
+    if not use_field_canonical_representation:
+        return arg.ndarray
+    # the canonical representation requires alphabetical ordering of the dimensions in field domain definition
+    sorted_dims = get_sorted_dims(arg.domain.dims)
+    ndim = len(sorted_dims)
+    dim_indices = [dim_index for dim_index, _ in sorted_dims]
+    if isinstance(arg.ndarray, np.ndarray):
+        return np.moveaxis(arg.ndarray, range(ndim), dim_indices)
+    else:
+        assert cp is not None and isinstance(arg.ndarray, cp.ndarray)
+        return cp.moveaxis(arg.ndarray, range(ndim), dim_indices)
 
 
 def preprocess_program(
@@ -107,9 +105,14 @@ def preprocess_program(
     return fencil_definition, tmps
 
 
-def get_args(sdfg: dace.SDFG, args: Sequence[Any]) -> dict[str, Any]:
+def get_args(
+    sdfg: dace.SDFG, args: Sequence[Any], use_field_canonical_representation: bool
+) -> dict[str, Any]:
     sdfg_params: Sequence[str] = sdfg.arg_names
-    return {sdfg_param: convert_arg(arg, sdfg_param) for sdfg_param, arg in zip(sdfg_params, args)}
+    return {
+        sdfg_param: convert_arg(arg, sdfg_param, use_field_canonical_representation)
+        for sdfg_param, arg in zip(sdfg_params, args)
+    }
 
 
 def _ensure_is_on_device(
@@ -162,8 +165,13 @@ def get_stride_args(
                 raise ValueError(
                     f"Stride ({stride_size} bytes) for argument '{sym}' must be a multiple of item size ({value.itemsize} bytes)."
                 )
-            stride_args[str(sym)] = stride
-
+            if isinstance(sym, dace.symbol):
+                assert sym.name not in stride_args
+                stride_args[str(sym)] = stride
+            elif sym != stride:
+                raise RuntimeError(
+                    f"Expected stride {arrays[name].strides} for arg {name}, got {value.strides}."
+                )
     return stride_args
 
 
@@ -221,12 +229,15 @@ def get_sdfg_args(sdfg: dace.SDFG, *args, check_args: bool = False, **kwargs) ->
         sdfg:               The SDFG for which we want to get the arguments.
     """
     offset_provider = kwargs["offset_provider"]
-    on_gpu = kwargs.get("on_gpu", False)
+    on_gpu = kwargs.get("on_gpu", _default_on_gpu)
+    use_field_canonical_representation = kwargs.get(
+        "use_field_canonical_representation", _default_use_field_canonical_representation
+    )
 
     neighbor_tables = filter_neighbor_tables(offset_provider)
     device = dace.DeviceType.GPU if on_gpu else dace.DeviceType.CPU
 
-    dace_args = get_args(sdfg, args)
+    dace_args = get_args(sdfg, args, use_field_canonical_representation)
     dace_field_args = {n: v for n, v in dace_args.items() if not np.isscalar(v)}
     dace_conn_args = get_connectivity_args(neighbor_tables, device)
     dace_shapes = get_shape_args(sdfg.arrays, dace_field_args)
@@ -261,6 +272,7 @@ def build_sdfg_from_itir(
     load_sdfg_from_file: bool = False,
     cache_id: Optional[str] = None,
     save_sdfg: bool = True,
+    use_field_canonical_representation: bool = True,
 ) -> dace.SDFG:
     """Translate a Fencil into an SDFG.
 
@@ -275,6 +287,7 @@ def build_sdfg_from_itir(
         load_sdfg_from_file: Allows to read the SDFG from file, instead of generating it, for debug only.
         cache_id:            The id of the cache entry, used to disambiguate stored sdfgs.
         save_sdfg:           If `True`, the default the SDFG is stored as a file and can be loaded, this allows to skip the lowering step, requires `load_sdfg_from_file` set to `True`.
+        use_field_canonical_representation: If `True`,  assume that the fields dimensions are sorted alphabetically.
 
     Notes:
         Currently only the `FORCE_INLINE` liftmode is supported and the value of `lift_mode` is ignored.
@@ -292,7 +305,9 @@ def build_sdfg_from_itir(
 
     # visit ITIR and generate SDFG
     program, tmps = preprocess_program(program, offset_provider, lift_mode)
-    sdfg_genenerator = ItirToSDFG(arg_types, offset_provider, tmps, column_axis)
+    sdfg_genenerator = ItirToSDFG(
+        arg_types, offset_provider, tmps, use_field_canonical_representation, column_axis
+    )
     sdfg = sdfg_genenerator.visit(program)
     if sdfg is None:
         raise RuntimeError(f"Visit failed for program {program.id}.")
@@ -343,9 +358,12 @@ def run_dace_iterator(program: itir.FencilDefinition, *args, **kwargs):
     build_cache = kwargs.get("build_cache", None)
     compiler_args = kwargs.get("compiler_args", None)  # `None` will take default.
     build_type = kwargs.get("build_type", "RelWithDebInfo")
-    on_gpu = kwargs.get("on_gpu", False)
+    on_gpu = kwargs.get("on_gpu", _default_on_gpu)
     auto_optimize = kwargs.get("auto_optimize", True)
     lift_mode = kwargs.get("lift_mode", itir_transforms.LiftMode.FORCE_INLINE)
+    use_field_canonical_representation = kwargs.get(
+        "use_field_canonical_representation", _default_use_field_canonical_representation
+    )
     # ITIR parameters
     column_axis = kwargs.get("column_axis", None)
     offset_provider = kwargs["offset_provider"]
@@ -374,6 +392,7 @@ def run_dace_iterator(program: itir.FencilDefinition, *args, **kwargs):
             load_sdfg_from_file=load_sdfg_from_file,
             cache_id=cache_id,
             save_sdfg=save_sdfg,
+            use_field_canonical_representation=use_field_canonical_representation,
         )
 
         sdfg.build_folder = compilation_cache._session_cache_dir_path / ".dacecache"
@@ -417,7 +436,7 @@ def _run_dace_cpu(program: itir.FencilDefinition, *args, **kwargs) -> None:
     )
 
 
-run_dace_cpu = otf_exec.OTFBackend(
+run_dace_cpu = backend.Backend(
     executor=ppi.program_executor(_run_dace_cpu, name="run_dace_cpu"),
     allocator=next_allocators.StandardCPUFieldBufferAllocator(),
 )
@@ -440,7 +459,7 @@ else:
         raise RuntimeError("Missing 'cupy' dependency for GPU execution.")
 
 
-run_dace_gpu = otf_exec.OTFBackend(
+run_dace_gpu = backend.Backend(
     executor=ppi.program_executor(_run_dace_gpu, name="run_dace_gpu"),
     allocator=next_allocators.StandardGPUFieldBufferAllocator(),
 )
diff --git a/src/gt4py/next/program_processors/runners/dace_iterator/itir_to_sdfg.py b/src/gt4py/next/program_processors/runners/dace_iterator/itir_to_sdfg.py
index 726966e0..cbcf6dca 100644
--- a/src/gt4py/next/program_processors/runners/dace_iterator/itir_to_sdfg.py
+++ b/src/gt4py/next/program_processors/runners/dace_iterator/itir_to_sdfg.py
@@ -43,8 +43,6 @@ from .utility import (
     as_dace_type,
     as_scalar_type,
     connectivity_identifier,
-    create_memlet_at,
-    create_memlet_full,
     dace_debuginfo,
     filter_neighbor_tables,
     flatten_list,
@@ -79,6 +77,7 @@ def _get_scan_dim(
     column_axis: Dimension,
     storage_types: dict[str, ts.TypeSpec],
     output: SymRef,
+    use_field_canonical_representation: bool,
 ) -> tuple[str, int, ts.ScalarType]:
     """
     Extract information about the scan dimension.
@@ -92,7 +91,14 @@ def _get_scan_dim(
             - scan_dim_dtype: data type along the scan dimension
     """
     output_type = cast(ts.FieldType, storage_types[output.id])
-    sorted_dims = [dim for _, dim in get_sorted_dims(output_type.dims)]
+    sorted_dims = [
+        dim
+        for _, dim in (
+            get_sorted_dims(output_type.dims)
+            if use_field_canonical_representation
+            else enumerate(output_type.dims)
+        )
+    ]
     return (
         column_axis.value,
         sorted_dims.index(column_axis),
@@ -118,7 +124,7 @@ def _make_array_shape_and_strides(
         The output tuple fields are arrays of dace symbolic expressions.
     """
     dtype = dace.int32
-    sorted_dims = get_sorted_dims(dims) if sort_dims else enumerate(dims)
+    sorted_dims = get_sorted_dims(dims) if sort_dims else list(enumerate(dims))
     shape = [
         (
             neighbor_tables[dim.value].max_neighbors
@@ -157,12 +163,14 @@ class ItirToSDFG(eve.NodeVisitor):
     offset_provider: dict[str, Any]
     node_types: dict[int, next_typing.Type]
     unique_id: int
+    use_field_canonical_representation: bool
 
     def __init__(
         self,
         param_types: list[ts.TypeSpec],
         offset_provider: dict[str, NeighborTable],
         tmps: list[itir_transforms.global_tmps.Temporary],
+        use_field_canonical_representation: bool,
         column_axis: Optional[Dimension] = None,
     ):
         self.param_types = param_types
@@ -170,6 +178,7 @@ class ItirToSDFG(eve.NodeVisitor):
         self.offset_provider = offset_provider
         self.storage_types = {}
         self.tmps = tmps
+        self.use_field_canonical_representation = use_field_canonical_representation
 
     def add_storage(
         self,
@@ -177,7 +186,7 @@ class ItirToSDFG(eve.NodeVisitor):
         name: str,
         type_: ts.TypeSpec,
         neighbor_tables: Mapping[str, NeighborTable],
-        sort_dimensions: bool = True,
+        sort_dimensions: bool,
     ):
         if isinstance(type_, ts.FieldType):
             shape, strides = _make_array_shape_and_strides(
@@ -273,6 +282,19 @@ class ItirToSDFG(eve.NodeVisitor):
 
         return tmp_symbols
 
+    def create_memlet_at(
+        self,
+        field_name: str,
+        index: dict[str, str],
+    ):
+        field_type = cast(ts.FieldType, self.storage_types[field_name])
+        if self.use_field_canonical_representation:
+            field_index = [index[dim.value] for _, dim in get_sorted_dims(field_type.dims)]
+        else:
+            field_index = [index[dim.value] for dim in field_type.dims]
+        subset = ", ".join(field_index)
+        return dace.Memlet(data=field_name, subset=subset)
+
     def get_output_nodes(
         self, closure: itir.StencilClosure, sdfg: dace.SDFG, state: dace.SDFGState
     ) -> dict[str, dace.nodes.AccessNode]:
@@ -281,7 +303,9 @@ class ItirToSDFG(eve.NodeVisitor):
         output_symbols_pass.visit(closure.output)
         # Visit output node again to generate the corresponding tasklet
         context = Context(sdfg, state, output_symbols_pass.symbol_refs)
-        translator = PythonTaskletCodegen(self.offset_provider, context, self.node_types)
+        translator = PythonTaskletCodegen(
+            self.offset_provider, context, self.node_types, self.use_field_canonical_representation
+        )
         output_nodes = flatten_list(translator.visit(closure.output))
         return {node.value.data: node.value for node in output_nodes}
 
@@ -296,7 +320,13 @@ class ItirToSDFG(eve.NodeVisitor):
 
         # Add program parameters as SDFG storages.
         for param, type_ in zip(node.params, self.param_types):
-            self.add_storage(program_sdfg, str(param.id), type_, neighbor_tables)
+            self.add_storage(
+                program_sdfg,
+                str(param.id),
+                type_,
+                neighbor_tables,
+                self.use_field_canonical_representation,
+            )
 
         if self.tmps:
             tmp_symbols = self.add_storage_for_temporaries(node.params, entry_state, program_sdfg)
@@ -339,10 +369,12 @@ class ItirToSDFG(eve.NodeVisitor):
 
             # Create memlets to transfer the program parameters
             input_mapping = {
-                name: create_memlet_full(name, program_sdfg.arrays[name]) for name in input_names
+                name: dace.Memlet.from_array(name, program_sdfg.arrays[name])
+                for name in input_names
             }
             output_mapping = {
-                name: create_memlet_full(name, program_sdfg.arrays[name]) for name in output_names
+                name: dace.Memlet.from_array(name, program_sdfg.arrays[name])
+                for name in output_names
             }
 
             symbol_mapping = map_nested_sdfg_symbols(program_sdfg, closure_sdfg, input_mapping)
@@ -410,7 +442,7 @@ class ItirToSDFG(eve.NodeVisitor):
                 closure_init_state.add_nedge(
                     closure_init_state.add_access(name, debuginfo=closure_sdfg.debuginfo),
                     closure_init_state.add_access(transient_name, debuginfo=closure_sdfg.debuginfo),
-                    create_memlet_full(name, closure_sdfg.arrays[name]),
+                    dace.Memlet.from_array(name, closure_sdfg.arrays[name]),
                 )
                 input_transients_mapping[name] = transient_name
             elif isinstance(self.storage_types[name], ts.FieldType):
@@ -476,7 +508,7 @@ class ItirToSDFG(eve.NodeVisitor):
             for input_name in input_names
         ]
         input_memlets = [
-            create_memlet_full(name, closure_sdfg.arrays[name])
+            dace.Memlet.from_array(name, closure_sdfg.arrays[name])
             for name in [*input_local_names, *connectivity_names]
         ]
 
@@ -496,19 +528,15 @@ class ItirToSDFG(eve.NodeVisitor):
             _, (scan_lb, scan_ub) = closure_domain[scan_dim_index]
             output_subset = f"{scan_lb.value}:{scan_ub.value}"
 
-            output_memlets = [
-                create_memlet_at(
-                    output_name,
-                    tuple(
-                        (
-                            f"i_{dim}"
-                            if f"i_{dim}" in map_ranges
-                            else f"0:{closure_sdfg.arrays[output_name].shape[scan_dim_index]}"
-                        )
-                        for dim, _ in closure_domain
-                    ),
+            domain_subset = {
+                dim: (
+                    f"i_{dim}"
+                    if f"i_{dim}" in map_ranges
+                    else f"0:{closure_sdfg.arrays[output_name].shape[scan_dim_index]}"
                 )
-            ]
+                for dim, _ in closure_domain
+            }
+            output_memlets = [self.create_memlet_at(output_name, domain_subset)]
         else:
             nsdfg, map_ranges, results = self._visit_parallel_stencil_closure(
                 node, closure_sdfg.arrays, closure_domain
@@ -517,7 +545,7 @@ class ItirToSDFG(eve.NodeVisitor):
             output_subset = "0"
 
             output_memlets = [
-                create_memlet_at(output_name, tuple(idx for idx in map_ranges.keys()))
+                self.create_memlet_at(output_name, {dim: f"i_{dim}" for dim, _ in closure_domain})
                 for output_name in output_connectors_mapping.values()
             ]
 
@@ -578,6 +606,7 @@ class ItirToSDFG(eve.NodeVisitor):
             self.column_axis,
             self.storage_types,
             node.output,
+            self.use_field_canonical_representation,
         )
 
         assert isinstance(node.output, SymRef)
@@ -683,16 +712,17 @@ class ItirToSDFG(eve.NodeVisitor):
             input_arrays,
             connectivity_arrays,
             self.node_types,
+            self.use_field_canonical_representation,
         )
 
         lambda_input_names = [name for name, _ in input_arrays]
         lambda_output_names = [connector.value.data for connector in lambda_outputs]
 
         input_memlets = [
-            create_memlet_full(name, scan_sdfg.arrays[name]) for name in lambda_input_names
+            dace.Memlet.from_array(name, scan_sdfg.arrays[name]) for name in lambda_input_names
         ]
         connectivity_memlets = [
-            create_memlet_full(name, scan_sdfg.arrays[name]) for name in connectivity_names
+            dace.Memlet.from_array(name, scan_sdfg.arrays[name]) for name in connectivity_names
         ]
         input_mapping = {param: arg for param, arg in zip(lambda_input_names, input_memlets)}
         connectivity_mapping = {
@@ -767,6 +797,7 @@ class ItirToSDFG(eve.NodeVisitor):
             input_arrays,
             connectivity_arrays,
             self.node_types,
+            self.use_field_canonical_representation,
         )
 
         return context.body, map_ranges, [r.value.data for r in results]
@@ -787,12 +818,17 @@ class ItirToSDFG(eve.NodeVisitor):
             assert isinstance(dimension, itir.AxisLiteral)
             lower_bound = named_range.args[1]
             upper_bound = named_range.args[2]
-            translator = PythonTaskletCodegen(self.offset_provider, context, self.node_types)
+            translator = PythonTaskletCodegen(
+                self.offset_provider,
+                context,
+                self.node_types,
+                self.use_field_canonical_representation,
+            )
             lb = translator.visit(lower_bound)[0]
             ub = translator.visit(upper_bound)[0]
             bounds.append((dimension.value, (lb, ub)))
 
-        return tuple(sorted(bounds, key=lambda item: item[0]))
+        return tuple(bounds)
 
     @staticmethod
     def _check_shift_offsets_are_literals(node: itir.StencilClosure):
diff --git a/src/gt4py/next/program_processors/runners/dace_iterator/itir_to_tasklet.py b/src/gt4py/next/program_processors/runners/dace_iterator/itir_to_tasklet.py
index d67d014b..6f9c5733 100644
--- a/src/gt4py/next/program_processors/runners/dace_iterator/itir_to_tasklet.py
+++ b/src/gt4py/next/program_processors/runners/dace_iterator/itir_to_tasklet.py
@@ -34,8 +34,6 @@ from .utility import (
     add_mapped_nested_sdfg,
     as_dace_type,
     connectivity_identifier,
-    create_memlet_at,
-    create_memlet_full,
     dace_debuginfo,
     filter_neighbor_tables,
     flatten_list,
@@ -243,11 +241,11 @@ def _visit_lift_in_neighbors_reduction(
     parent_state = transformer.context.state
 
     input_mapping = {
-        connector: create_memlet_full(node.data, node.desc(parent_sdfg))
+        connector: dace.Memlet.from_array(node.data, node.desc(parent_sdfg))
         for connector, node in input_nodes.items()
     }
     connectivity_mapping = {
-        name: create_memlet_full(name, parent_sdfg.arrays[name]) for name in connectivity_names
+        name: dace.Memlet.from_array(name, parent_sdfg.arrays[name]) for name in connectivity_names
     }
     array_mapping = {**input_mapping, **connectivity_mapping}
     symbol_mapping = map_nested_sdfg_symbols(parent_sdfg, lift_context.body, array_mapping)
@@ -396,7 +394,7 @@ def builtin_neighbors(
         state.add_access(table_name, debuginfo=di),
         me,
         shift_tasklet,
-        memlet=create_memlet_full(table_name, sdfg.arrays[table_name]),
+        memlet=dace.Memlet.from_array(table_name, sdfg.arrays[table_name]),
         dst_conn="__table",
     )
     state.add_memlet_path(
@@ -426,7 +424,8 @@ def builtin_neighbors(
             neighbor_value_node,
         )
     else:
-        data_access_index = ",".join(f"{dim}_v" for dim in sorted(iterator.dimensions))
+        sorted_dims = transformer.get_sorted_field_dimensions(iterator.dimensions)
+        data_access_index = ",".join(f"{dim}_v" for dim in sorted_dims)
         connector_neighbor_dim = f"{offset_provider.neighbor_axis.value}_v"
         data_access_tasklet = state.add_tasklet(
             "data_access",
@@ -444,7 +443,7 @@ def builtin_neighbors(
             iterator.field,
             me,
             data_access_tasklet,
-            memlet=create_memlet_full(iterator.field.data, field_desc),
+            memlet=dace.Memlet.from_array(iterator.field.data, field_desc),
             dst_conn="__field",
         )
         for dim in iterator.dimensions:
@@ -579,7 +578,10 @@ def builtin_if(
         symbol_map = copy.deepcopy(transformer.context.symbol_map)
         node_context = Context(sdfg, state, symbol_map)
         node_taskgen = PythonTaskletCodegen(
-            transformer.offset_provider, node_context, transformer.node_types
+            transformer.offset_provider,
+            node_context,
+            transformer.node_types,
+            transformer.use_field_canonical_representation,
         )
         return node_taskgen.visit(arg)
 
@@ -899,16 +901,22 @@ class PythonTaskletCodegen(gt4py.eve.codegen.TemplatedGenerator):
     offset_provider: dict[str, Any]
     context: Context
     node_types: dict[int, next_typing.Type]
+    use_field_canonical_representation: bool
 
     def __init__(
         self,
         offset_provider: dict[str, Any],
         context: Context,
         node_types: dict[int, next_typing.Type],
+        use_field_canonical_representation: bool,
     ):
         self.offset_provider = offset_provider
         self.context = context
         self.node_types = node_types
+        self.use_field_canonical_representation = use_field_canonical_representation
+
+    def get_sorted_field_dimensions(self, dims: Sequence[str]):
+        return sorted(dims) if self.use_field_canonical_representation else dims
 
     def visit_FunctionDefinition(self, node: itir.FunctionDefinition, **kwargs):
         raise NotImplementedError()
@@ -968,7 +976,12 @@ class PythonTaskletCodegen(gt4py.eve.codegen.TemplatedGenerator):
             lambda_symbols_pass.symbol_refs,
             reduce_identity=self.context.reduce_identity,
         )
-        lambda_taskgen = PythonTaskletCodegen(self.offset_provider, lambda_context, self.node_types)
+        lambda_taskgen = PythonTaskletCodegen(
+            self.offset_provider,
+            lambda_context,
+            self.node_types,
+            self.use_field_canonical_representation,
+        )
 
         results: list[ValueExpr] = []
         # We are flattening the returned list of value expressions because the multiple outputs of a lambda
@@ -1045,24 +1058,26 @@ class PythonTaskletCodegen(gt4py.eve.codegen.TemplatedGenerator):
         nsdfg_inputs = {}
         for name, value in func_inputs:
             if isinstance(value, ValueExpr):
-                nsdfg_inputs[name] = create_memlet_full(
+                nsdfg_inputs[name] = dace.Memlet.from_array(
                     value.value.data, self.context.body.arrays[value.value.data]
                 )
             else:
                 assert isinstance(value, IteratorExpr)
                 field = name[0]
                 indices = name[1]
-                nsdfg_inputs[field] = create_memlet_full(
+                nsdfg_inputs[field] = dace.Memlet.from_array(
                     value.field.data, self.context.body.arrays[value.field.data]
                 )
                 for dim, var in indices.items():
                     store = value.indices[dim].data
-                    nsdfg_inputs[var] = create_memlet_full(store, self.context.body.arrays[store])
+                    nsdfg_inputs[var] = dace.Memlet.from_array(
+                        store, self.context.body.arrays[store]
+                    )
 
         neighbor_tables = filter_neighbor_tables(self.offset_provider)
         for offset in neighbor_tables.keys():
             var = connectivity_identifier(offset)
-            nsdfg_inputs[var] = create_memlet_full(var, self.context.body.arrays[var])
+            nsdfg_inputs[var] = dace.Memlet.from_array(var, self.context.body.arrays[var])
 
         symbol_mapping = map_nested_sdfg_symbols(self.context.body, func_context.body, nsdfg_inputs)
 
@@ -1101,7 +1116,7 @@ class PythonTaskletCodegen(gt4py.eve.codegen.TemplatedGenerator):
             self.context.body.add_scalar(name, result.dtype, transient=True)
             result_access = self.context.state.add_access(name, debuginfo=nsdfg_node.debuginfo)
             result_exprs.append(ValueExpr(result_access, result.dtype))
-            memlet = create_memlet_full(name, self.context.body.arrays[name])
+            memlet = dace.Memlet.from_array(name, self.context.body.arrays[name])
             self.context.state.add_edge(nsdfg_node, result.value.data, result_access, None, memlet)
 
         return result_exprs
@@ -1113,7 +1128,7 @@ class PythonTaskletCodegen(gt4py.eve.codegen.TemplatedGenerator):
             # already a list of ValueExpr
             return iterator
 
-        sorted_dims = sorted(iterator.dimensions)
+        sorted_dims = self.get_sorted_field_dimensions(iterator.dimensions)
         if all([dim in iterator.indices for dim in iterator.dimensions]):
             # The deref iterator has index values on all dimensions: the result will be a scalar
             args = [ValueExpr(iterator.field, iterator.dtype)] + [
@@ -1334,8 +1349,9 @@ class PythonTaskletCodegen(gt4py.eve.codegen.TemplatedGenerator):
             input_args = [arg[0] for arg in args]
             input_valid_args = [arg[1] for arg in args if len(arg) == 2]
 
-            nreduce_index = tuple(f"_i{i}" for i in range(len(nreduce_shape)))
-            nreduce_domain = {idx: f"0:{size}" for idx, size in zip(nreduce_index, nreduce_shape)}
+            assert len(nreduce_shape) == 1
+            nreduce_index = unique_name("_i")
+            nreduce_domain = {nreduce_index: f"0:{nreduce_shape[0]}"}
 
             reduce_input_name = unique_var_name()
             self.context.body.add_array(
@@ -1353,12 +1369,14 @@ class PythonTaskletCodegen(gt4py.eve.codegen.TemplatedGenerator):
                 param: (
                     dace.Memlet(data=arg.value.data, subset="0")
                     if arg.value.desc(self.context.body).shape == (1,)
-                    else create_memlet_at(arg.value.data, nreduce_index)
+                    else dace.Memlet(data=arg.value.data, subset=nreduce_index)
                 )
                 for (param, _), arg in zip(inner_inputs, input_args)
             }
             output_mapping = {
-                inner_outputs[0].value.data: create_memlet_at(reduce_input_name, nreduce_index)
+                inner_outputs[0].value.data: dace.Memlet(
+                    data=reduce_input_name, subset=nreduce_index
+                )
             }
             symbol_mapping = map_nested_sdfg_symbols(
                 self.context.body, lambda_context.body, input_mapping
@@ -1377,8 +1395,8 @@ class PythonTaskletCodegen(gt4py.eve.codegen.TemplatedGenerator):
                 lambda_output_node = inner_outputs[0].value
                 # add input connector to nested sdfg
                 lambda_context.body.add_scalar("_valid_neighbor", dace.dtypes.bool)
-                input_mapping["_valid_neighbor"] = create_memlet_at(
-                    input_valid_node.data, nreduce_index
+                input_mapping["_valid_neighbor"] = dace.Memlet(
+                    data=input_valid_node.data, subset=nreduce_index
                 )
                 # add select tasklet before writing to output node
                 # TODO: consider replacing it with a select-memlet once it is supported by DaCe SDFG API
@@ -1507,7 +1525,7 @@ class PythonTaskletCodegen(gt4py.eve.codegen.TemplatedGenerator):
             if used:
                 continue
             elif not isinstance(arg, SymbolExpr):
-                memlet = create_memlet_full(
+                memlet = dace.Memlet.from_array(
                     arg.value.data, self.context.body.arrays[arg.value.data]
                 )
                 self.context.state.add_edge(arg.value, None, expr_tasklet, internal, memlet)
@@ -1529,6 +1547,7 @@ def closure_to_tasklet_sdfg(
     inputs: Sequence[tuple[str, ts.TypeSpec]],
     connectivities: Sequence[tuple[dace.ndarray, str]],
     node_types: dict[int, next_typing.Type],
+    use_field_canonical_representation: bool,
 ) -> tuple[Context, Sequence[ValueExpr]]:
     body = dace.SDFG("tasklet_toplevel")
     body.debuginfo = dace_debuginfo(node)
@@ -1565,7 +1584,9 @@ def closure_to_tasklet_sdfg(
         body.add_array(name, shape=shape, strides=strides, dtype=arr.dtype)
 
     context = Context(body, state, symbol_map)
-    translator = PythonTaskletCodegen(offset_provider, context, node_types)
+    translator = PythonTaskletCodegen(
+        offset_provider, context, node_types, use_field_canonical_representation
+    )
 
     args = [itir.SymRef(id=name) for name, _ in inputs]
     if is_scan(node.stencil):
diff --git a/src/gt4py/next/program_processors/runners/dace_iterator/utility.py b/src/gt4py/next/program_processors/runners/dace_iterator/utility.py
index d5ba62c7..77f86b40 100644
--- a/src/gt4py/next/program_processors/runners/dace_iterator/utility.py
+++ b/src/gt4py/next/program_processors/runners/dace_iterator/utility.py
@@ -71,15 +71,6 @@ def connectivity_identifier(name: str):
     return f"__connectivity_{name}"
 
 
-def create_memlet_full(source_identifier: str, source_array: dace.data.Array):
-    return dace.Memlet.from_array(source_identifier, source_array)
-
-
-def create_memlet_at(source_identifier: str, index: tuple[str, ...]):
-    subset = ", ".join(index)
-    return dace.Memlet(data=source_identifier, subset=subset)
-
-
 def get_sorted_dims(dims: Sequence[Dimension]) -> Sequence[tuple[int, Dimension]]:
     return sorted(enumerate(dims), key=lambda v: v[1].value)
 
diff --git a/src/gt4py/next/program_processors/runners/double_roundtrip.py b/src/gt4py/next/program_processors/runners/double_roundtrip.py
index 2f06d17c..36620202 100644
--- a/src/gt4py/next/program_processors/runners/double_roundtrip.py
+++ b/src/gt4py/next/program_processors/runners/double_roundtrip.py
@@ -16,9 +16,9 @@ from __future__ import annotations
 
 from typing import TYPE_CHECKING, Any
 
-import gt4py.next.program_processors.otf_compile_executor as otf_compile_executor
 import gt4py.next.program_processors.processor_interface as ppi
-import gt4py.next.program_processors.runners.roundtrip as roundtrip
+from gt4py.next import backend as next_backend
+from gt4py.next.program_processors.runners import roundtrip
 
 
 if TYPE_CHECKING:
@@ -30,7 +30,7 @@ def executor(program: itir.FencilDefinition, *args: Any, **kwargs: Any) -> None:
     roundtrip.execute_roundtrip(program, *args, dispatch_backend=roundtrip.executor, **kwargs)
 
 
-backend = otf_compile_executor.OTFBackend(
+backend = next_backend.Backend(
     executor=executor,
     allocator=roundtrip.backend.allocator,
 )
diff --git a/src/gt4py/next/program_processors/runners/gtfn.py b/src/gt4py/next/program_processors/runners/gtfn.py
index 13e3308a..53b3e817 100644
--- a/src/gt4py/next/program_processors/runners/gtfn.py
+++ b/src/gt4py/next/program_processors/runners/gtfn.py
@@ -22,14 +22,14 @@ import numpy.typing as npt
 import gt4py._core.definitions as core_defs
 import gt4py.next.allocators as next_allocators
 from gt4py.eve.utils import content_hash
-from gt4py.next import common, config
+from gt4py.next import backend, common, config
 from gt4py.next.iterator import transforms
 from gt4py.next.iterator.transforms import global_tmps
 from gt4py.next.otf import recipes, stages, workflow
 from gt4py.next.otf.binding import nanobind
 from gt4py.next.otf.compilation import compiler
 from gt4py.next.otf.compilation.build_systems import compiledb
-from gt4py.next.program_processors import otf_compile_executor
+from gt4py.next.program_processors import modular_executor
 from gt4py.next.program_processors.codegens.gtfn import gtfn_module
 from gt4py.next.type_system.type_translation import from_value
 
@@ -145,7 +145,7 @@ class GTFNCompileWorkflowFactory(factory.Factory):
 
 class GTFNBackendFactory(factory.Factory):
     class Meta:
-        model = otf_compile_executor.OTFBackend
+        model = backend.Backend
 
     class Params:
         name_device = "cpu"
@@ -159,7 +159,7 @@ class GTFNBackendFactory(factory.Factory):
         )
         cached = factory.Trait(
             executor=factory.LazyAttribute(
-                lambda o: otf_compile_executor.CachedOTFCompileExecutor(
+                lambda o: modular_executor.ModularExecutor(
                     otf_workflow=workflow.CachedStep(o.otf_workflow, hash_function=o.hash_function),
                     name=o.name,
                 )
@@ -181,7 +181,7 @@ class GTFNBackendFactory(factory.Factory):
         )
 
     executor = factory.LazyAttribute(
-        lambda o: otf_compile_executor.OTFCompileExecutor(otf_workflow=o.otf_workflow, name=o.name)
+        lambda o: modular_executor.ModularExecutor(otf_workflow=o.otf_workflow, name=o.name)
     )
     allocator = next_allocators.StandardCPUFieldBufferAllocator()
 
diff --git a/src/gt4py/next/program_processors/runners/roundtrip.py b/src/gt4py/next/program_processors/runners/roundtrip.py
index f81606ee..0f07b551 100644
--- a/src/gt4py/next/program_processors/runners/roundtrip.py
+++ b/src/gt4py/next/program_processors/runners/roundtrip.py
@@ -21,16 +21,16 @@ import textwrap
 from collections.abc import Callable, Iterable
 from typing import Any, Optional
 
-import gt4py.eve.codegen as codegen
 import gt4py.next.allocators as next_allocators
 import gt4py.next.common as common
 import gt4py.next.iterator.embedded as embedded
 import gt4py.next.iterator.ir as itir
 import gt4py.next.iterator.transforms as itir_transforms
 import gt4py.next.iterator.transforms.global_tmps as gtmps_transform
-import gt4py.next.program_processors.otf_compile_executor as otf_compile_executor
 import gt4py.next.program_processors.processor_interface as ppi
+from gt4py.eve import codegen
 from gt4py.eve.codegen import FormatTemplate as as_fmt, MakoTemplate as as_mako
+from gt4py.next import backend as next_backend
 
 
 def _create_tmp(axes, origin, shape, dtype):
@@ -229,6 +229,6 @@ def execute_roundtrip(
 
 executor = ppi.program_executor(execute_roundtrip)  # type: ignore[arg-type]
 
-backend = otf_compile_executor.OTFBackend(
+backend = next_backend.Backend(
     executor=executor, allocator=next_allocators.StandardCPUFieldBufferAllocator()
 )


Test patch:
diff --git a/docs/development/ADRs/0015-Test_Exclusion_Matrices.md b/docs/development/ADRs/0015-Test_Exclusion_Matrices.md
index c8687579..387170a6 100644
--- a/docs/development/ADRs/0015-Test_Exclusion_Matrices.md
+++ b/docs/development/ADRs/0015-Test_Exclusion_Matrices.md
@@ -49,9 +49,9 @@ The following backend processors are defined:
 ```python
 DACE_CPU = "dace_iterator.run_dace_cpu"
 DACE_GPU = "dace_iterator.run_dace_gpu"
-GTFN_CPU = "otf_compile_executor.run_gtfn"
-GTFN_CPU_IMPERATIVE = "otf_compile_executor.run_gtfn_imperative"
-GTFN_CPU_WITH_TEMPORARIES = "otf_compile_executor.run_gtfn_with_temporaries"
+GTFN_CPU = "gt4py.next.program_processors.runners.gtfn.run_gtfn"
+GTFN_CPU_IMPERATIVE = "gt4py.next.program_processors.runners.gtfn.run_gtfn_imperative"
+GTFN_CPU_WITH_TEMPORARIES = "gt4py.next.program_processors.runners.gtfn.run_gtfn_with_temporaries"
 GTFN_GPU = "gt4py.next.program_processors.runners.gtfn.run_gtfn_gpu"
 ```
 
diff --git a/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_temporaries_with_sizes.py b/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_temporaries_with_sizes.py
index a264843f..1bf640f9 100644
--- a/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_temporaries_with_sizes.py
+++ b/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_temporaries_with_sizes.py
@@ -16,9 +16,9 @@ import pytest
 from numpy import int32, int64
 
 from gt4py import next as gtx
-from gt4py.next import common
+from gt4py.next import backend, common
 from gt4py.next.iterator.transforms import LiftMode, apply_common_transforms
-from gt4py.next.program_processors import otf_compile_executor
+from gt4py.next.program_processors import modular_executor
 from gt4py.next.program_processors.runners.gtfn import run_gtfn_with_temporaries
 
 from next_tests.integration_tests import cases
@@ -38,8 +38,8 @@ from next_tests.toy_connectivity import Cell, Edge
 
 @pytest.fixture
 def run_gtfn_with_temporaries_and_symbolic_sizes():
-    return otf_compile_executor.OTFBackend(
-        executor=otf_compile_executor.OTFCompileExecutor(
+    return backend.Backend(
+        executor=modular_executor.ModularExecutor(
             name="run_gtfn_with_temporaries_and_sizes",
             otf_workflow=run_gtfn_with_temporaries.executor.otf_workflow.replace(
                 translation=run_gtfn_with_temporaries.executor.otf_workflow.translation.replace(