Repository: RDycore/RDycore
Pull request #269: Adds AMR
Author: login: bishtgautam
Labels: name: enhancement
description: New feature or request
name: AMR
description: related to adaptive mesh refinement
Created: 2025-01-24T04:45:42Z | Merged: 2025-04-12T03:00:36Z

This PR adds a new AMR drive `driver/amr.c` for RDycore. An RDycore simulation
gets configured similarly to the case without AMR. RDycore then advances to the
first coupling time while taking multiple time steps. Then, the mesh is refined, and
the model continues to the next coupling time step. The mesh is refined at each
coupling time until the end of the simulation.

Currently, after each refinement, the mesh is not redistributed across MPI ranks.



Diff:
diff --git a/config/set_petsc_settings.sh b/config/set_petsc_settings.sh
index 2ca4613a2..4600f3abd 100755
--- a/config/set_petsc_settings.sh
+++ b/config/set_petsc_settings.sh
@@ -62,11 +62,11 @@ if [ "$mach" = "pm-cpu" ]; then
     export PETSC_DIR=/global/cfs/projectdirs/m4267/petsc/petsc_v3.23.0/
 
     if [ "$config" -eq 1 ]; then
-        export PETSC_ARCH=pm-cpu-hdf5_1_14_3-opt-64bit-gcc-13-2-1-v3.22.4
+        export PETSC_ARCH=pm-cpu-hdf5_1_14_3-opt-64bit-gcc-13-2-1-95934b0d393
     elif [ "$config" -eq 2 ]; then
-        export PETSC_ARCH=pm-cpu-hdf5_1_14_3-debug-64bit-gcc-13-2-1-v3.22.4
+        export PETSC_ARCH=pm-cpu-hdf5_1_14_3-debug-64bit-gcc-13-2-1-95934b0d393
     elif [ "$config" -eq 3 ]; then
-        export PETSC_ARCH=pm-cpu-opt-32bit-gcc-13-2-1-v3.22.4
+        export PETSC_ARCH=pm-cpu-opt-32bit-gcc-13-2-1-95934b0d393
     fi
 
 elif [ "$mach" = "pm-gpu" ]; then
@@ -75,11 +75,11 @@ elif [ "$mach" = "pm-gpu" ]; then
     export PETSC_DIR=/global/cfs/projectdirs/m4267/petsc/petsc_v3.23.0/
 
     if [ "$config" -eq 1 ]; then
-      export PETSC_ARCH=pm-gpu-hdf5_1_14_3-opt-64bit-gcc-13-2-1-v3.22.4
+      export PETSC_ARCH=pm-gpu-hdf5_1_14_3-opt-64bit-gcc-13-2-1-95934b0d393
     elif [ "$config" -eq 2 ]; then
-      export PETSC_ARCH=pm-gpu-hdf5_1_14_3-debug-64bit-gcc-13-2-1-v3.22.4
+      export PETSC_ARCH=pm-gpu-hdf5_1_14_3-debug-64bit-gcc-13-2-1-95934b0d393
     elif [ "$config" -eq 3 ]; then
-      export PETSC_ARCH=pm-gpu-opt-32bit-gcc-13-2-1-v3.22.4
+      export PETSC_ARCH=pm-gpu-opt-32bit-gcc-13-2-1-95934b0d393
     else
       echo "Unsupported --config $config "
       display_help 
@@ -93,7 +93,6 @@ elif [ "$mach" = "frontier"  ]; then
   if [ "$config" -eq 1 ]; then
       export PETSC_ARCH=craygnu-mphipcc-opt-64bit-gcc-13-3-0-95934b0d393
   elif [ "$config" -eq 2 ]; then
-      export PETSC_ARCH=craygnu-mphipcc-hdf5_1_14_3-debug-64bit-gcc-13-3-0-v3.22.4
       export PETSC_ARCH=craygnu-mphipcc-debug-64bit-gcc-13-3-0-95934b0d393
   elif [ "$config" -eq 3 ]; then
       export PETSC_ARCH=craygnu-mphipcc-opt-32bit-gcc-13-3-0-95934b0d393
diff --git a/driver/CMakeLists.txt b/driver/CMakeLists.txt
index 29975438b..3bed74a5d 100644
--- a/driver/CMakeLists.txt
+++ b/driver/CMakeLists.txt
@@ -12,8 +12,15 @@ add_executable(rdycore_mms
 set_target_properties(rdycore_mms PROPERTIES OUTPUT_NAME rdycore_mms)
 target_link_libraries(rdycore_mms PRIVATE rdycore)
 
+# AMR C driver
+add_executable(rdycore_amr
+  amr.c
+)
+set_target_properties(rdycore_amr PROPERTIES OUTPUT_NAME rdycore_amr)
+target_link_libraries(rdycore_amr PRIVATE rdycore)
+
 install(
-  TARGETS rdycore_exe rdycore_mms
+  TARGETS rdycore_exe rdycore_mms rdycore_amr
   DESTINATION bin
 )
 
diff --git a/driver/amr.c b/driver/amr.c
new file mode 100644
index 000000000..2fa367b12
--- /dev/null
+++ b/driver/amr.c
@@ -0,0 +1,50 @@
+#include <petscsys.h>
+#include <rdycore.h>
+#include <time.h>
+
+static const char *help_str =
+    "rdycore_amr - a standalone RDycore driver for AMR problems\n"
+    "usage: rdycore_amr [options] <filename.yaml>\n";
+
+static void usage(const char *exe_name) {
+  fprintf(stderr, "%s: usage:\n", exe_name);
+  fprintf(stderr, "%s <input.yaml>\n\n", exe_name);
+}
+
+int main(int argc, char *argv[]) {
+  // print usage info if no arguments given
+  if (argc < 2) {
+    usage(argv[0]);
+    exit(-1);
+  }
+
+  // initialize subsystems
+  PetscCall(RDyInit(argc, argv, help_str));
+
+  PetscMPIInt myrank, commsize;
+  PetscCallMPI(MPI_Comm_rank(PETSC_COMM_WORLD, &myrank));
+  PetscCallMPI(MPI_Comm_size(PETSC_COMM_WORLD, &commsize));
+
+  if (strcmp(argv[1], "-help")) {  // if given a config file
+
+    // create rdycore and set it up with the given file
+    MPI_Comm comm = PETSC_COMM_WORLD;
+    RDy      rdy;
+
+    // set things up
+    PetscCall(RDyCreate(comm, argv[1], &rdy));
+    PetscCall(RDySetup(rdy));
+
+    PetscCall(RDyAdvance(rdy));
+
+    while (!RDyFinished(rdy)) {
+      PetscCall(RDyRefine(rdy));
+      PetscCall(RDyAdvance(rdy));
+    }
+
+    PetscCall(RDyDestroy(&rdy));
+  }
+
+  PetscCall(RDyFinalize());
+  return 0;
+}
diff --git a/driver/tests/CMakeLists.txt b/driver/tests/CMakeLists.txt
index 1ddb9e90b..df743034b 100644
--- a/driver/tests/CMakeLists.txt
+++ b/driver/tests/CMakeLists.txt
@@ -3,6 +3,8 @@
 # location of built driver executables
 set(rdycore_driver ${CMAKE_CURRENT_BINARY_DIR}/../rdycore)
 set(mms_driver ${CMAKE_CURRENT_BINARY_DIR}/../rdycore_mms)
+set(amr_driver ${CMAKE_CURRENT_BINARY_DIR}/../rdycore_amr)
+
 add_test(c_driver_usage ${MPIEXEC} ${MPIEXEC_OPTIONS} -n 1 ${rdycore_driver})
 set(supported_languages c)
 
@@ -14,6 +16,9 @@ if (ENABLE_FORTRAN)
   list(APPEND supported_languages f90)
 endif()
 
+# sediment dynamics
+add_subdirectory(amr)
+
 # shallow water equations with roe riemann solver
 add_subdirectory(swe_roe)
 
diff --git a/driver/tests/amr/CMakeLists.txt b/driver/tests/amr/CMakeLists.txt
new file mode 100644
index 000000000..1db1289b6
--- /dev/null
+++ b/driver/tests/amr/CMakeLists.txt
@@ -0,0 +1,20 @@
+find_program(H5DIFF h5diff REQUIRED
+             PATHS ${PETSC_DIR}/${PETSC_ARCH}/bin)
+set(amr_c_driver ${amr_driver})
+
+# arguments for specific configurations ("basic" config has no options)
+set(ceed_args -ceed /cpu/self)
+set(preload_args -preload)
+
+# AMR driver (C and Fortran)
+file(COPY
+     ${CMAKE_CURRENT_SOURCE_DIR}/amr_dx1.yaml
+     ${MESH_DIR}/mms_triangles_dx1.exo
+     DESTINATION ${CMAKE_CURRENT_BINARY_DIR})
+foreach(np 1)
+  foreach(config basic)
+    foreach(lang c)
+      add_test(amr_${lang}_np_${np}_${config} ${MPIEXEC} ${MPIEXEC_FLAGS} -n ${np} ${amr_${lang}_driver} amr_dx1.yaml -dm_plex_transform_type refine_sbr -dm_plex_transform_active adapt -dm_view -dm_fine_view ${${config}_args})
+    endforeach()
+  endforeach()
+endforeach()
diff --git a/driver/tests/amr/amr_dx1.yaml b/driver/tests/amr/amr_dx1.yaml
new file mode 100644
index 000000000..5df731b4f
--- /dev/null
+++ b/driver/tests/amr/amr_dx1.yaml
@@ -0,0 +1,50 @@
+physics:
+  flow:
+    mode: swe
+
+numerics:
+  spatial: fv
+  temporal: euler
+  riemann: roe
+
+logging:
+  level: debug
+
+time:
+  final_time: 0.05
+  unit: seconds
+  time_step : 0.005
+  coupling_interval: 0.01
+
+#output:
+#  format: xdmf
+#  step_interval: 100
+
+grid:
+  file: mms_triangles_dx1.exo
+
+# one region represents the whole domain
+regions:
+  - name: domain
+    grid_region_id: 1
+
+surface_composition:
+  - region: domain
+    material: smooth
+
+materials:
+  - name: smooth
+    properties:
+      manning:
+        value: 0.0
+
+initial_conditions:
+  - region: domain
+    flow: domain_flow_ic
+
+flow_conditions:
+  - name: domain_flow_ic
+    type: dirichlet
+    height: 10
+    x_momentum: 0
+    y_momentum: 0
diff --git a/include/private/rdycoreimpl.h b/include/private/rdycoreimpl.h
index b924a6c31..0024feadd 100644
--- a/include/private/rdycoreimpl.h
+++ b/include/private/rdycoreimpl.h
@@ -125,6 +125,16 @@ struct _p_RDy {
   // log file handle
   FILE *log;
 
+  //--------------------------
+  // Refinements
+  //--------------------------
+
+  // identifies if the mesh has been refined and the model has not taken a step
+  PetscBool mesh_was_refined;
+
+  // number of times the mesh has been refined
+  PetscInt num_refinements;
+
   //--------------------------
   // Solver and solution data
   //--------------------------
diff --git a/include/private/rdydmimpl.h b/include/private/rdydmimpl.h
index 17dd23211..9271d008a 100644
--- a/include/private/rdydmimpl.h
+++ b/include/private/rdydmimpl.h
@@ -13,7 +13,7 @@ typedef struct {
   char     field_component_names[MAX_NUM_FIELDS][MAX_NUM_FIELD_COMPONENTS][MAX_NAME_LEN + 1];
 } SectionFieldSpec;
 
-PETSC_INTERN PetscErrorCode CreateCellCenteredDMFromDM(DM, const SectionFieldSpec, DM *);
+PETSC_INTERN PetscErrorCode CreateCellCenteredDMFromDM(DM, PetscInt, const SectionFieldSpec, DM *);
 PETSC_INTERN PetscErrorCode CreateDM(RDy);
 PETSC_INTERN PetscErrorCode CreateAuxiliaryDM(RDy);
 PETSC_INTERN PetscErrorCode CreateFlowDM(RDy);
diff --git a/include/private/rdymeshimpl.h b/include/private/rdymeshimpl.h
index 979eafcee..9253abea4 100644
--- a/include/private/rdymeshimpl.h
+++ b/include/private/rdymeshimpl.h
@@ -201,7 +201,7 @@ typedef struct RDyMesh {
 
 } RDyMesh;
 
-PETSC_INTERN PetscErrorCode RDyMeshCreateFromDM(DM, RDyMesh *);
+PETSC_INTERN PetscErrorCode RDyMeshCreateFromDM(DM, PetscInt, RDyMesh *);
 PETSC_INTERN PetscErrorCode RDyMeshDestroy(RDyMesh);
 
 #endif
diff --git a/include/rdycore.h b/include/rdycore.h
index 70b5cdff1..5479d7ff7 100644
--- a/include/rdycore.h
+++ b/include/rdycore.h
@@ -37,7 +37,11 @@ PETSC_EXTERN PetscErrorCode RDyMMSComputeErrorNorms(RDy, PetscReal, PetscReal *,
 PETSC_EXTERN PetscErrorCode RDyMMSEstimateConvergenceRates(RDy, PetscReal *, PetscReal *, PetscReal *);
 PETSC_EXTERN PetscErrorCode RDyMMSRun(RDy);
 
+// RDycore support for AMR
+PETSC_EXTERN PetscErrorCode RDyRefine(RDy);
+
 // time integration
+PETSC_EXTERN PetscErrorCode DestroyOutputViewer(RDy);
 PETSC_EXTERN PetscErrorCode RDyAdvance(RDy);
 
 // Accessing data
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 2e5ff5de7..2f486b724 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -8,6 +8,7 @@ set(RDYCORE_C_SOURCE_FILES
   rdydm.c
   rdymesh.c
   rdymms.c
+  rdyrefine.c
   rdysetup.c
   operator.c
   operator_ceed.c
diff --git a/src/rdyadvance.c b/src/rdyadvance.c
index 3821c351e..2968cdce9 100644
--- a/src/rdyadvance.c
+++ b/src/rdyadvance.c
@@ -125,7 +125,7 @@ PetscErrorCode DetermineOutputFile(RDy rdy, PetscInt step, PetscReal time, const
   PetscFunctionReturn(PETSC_SUCCESS);
 }
 
-static PetscErrorCode DestroyOutputViewer(RDy rdy) {
+PetscErrorCode DestroyOutputViewer(RDy rdy) {
   PetscFunctionBegin;
   if (rdy->output_vf) PetscCall(PetscViewerAndFormatDestroy(&rdy->output_vf));
   if (rdy->output_viewer) PetscCall(PetscViewerDestroy(&rdy->output_viewer));
@@ -248,6 +248,13 @@ PetscErrorCode RDyAdvance(RDy rdy) {
     RDyLogDebug(rdy, "Running simulation...");
   }
 
+  if (rdy->mesh_was_refined) {
+    // create a viewer with the proper format for visualization output
+    PetscCall(CreateOutputViewer(rdy));
+
+    rdy->mesh_was_refined = PETSC_FALSE;
+  }
+
   PetscReal time;
   PetscCall(TSGetTime(rdy->ts, &time));
 
diff --git a/src/rdycore.c b/src/rdycore.c
index e2dfd009b..d3e9417c2 100644
--- a/src/rdycore.c
+++ b/src/rdycore.c
@@ -18,9 +18,6 @@ PetscErrorCode RDyInit(int argc, char *argv[], const char *help) {
       PetscCall(PetscInitialize(&argc, &argv, (char *)0, (char *)help));
     }
 
-    // some exit statuses don't get propagated without this vvv
-    PetscCall(PetscPushErrorHandler(PetscMPIAbortErrorHandler, NULL));
-
     // set up our logging class ID
     PetscCall(PetscClassIdRegister("RDycore", &RDY_CLASSID));
 
@@ -46,9 +43,6 @@ PetscErrorCode RDyInitFortran(void) {
       // called before this function in the rdycore Fortran module.
     }
 
-    // some exit statuses don't get propagated without this vvv
-    PetscCall(PetscPushErrorHandler(PetscMPIAbortErrorHandler, NULL));
-
     // set up our logging class ID
     PetscCall(PetscClassIdRegister("RDycore", &RDY_CLASSID));
 
@@ -179,6 +173,7 @@ PetscErrorCode RDyDestroyBoundaries(RDy *rdy) {
     }
   }
   if ((*rdy)->boundaries) PetscFree((*rdy)->boundaries);
+  (*rdy)->num_boundaries = 0;
 
   PetscFunctionReturn(PETSC_SUCCESS);
 }
diff --git a/src/rdydm.c b/src/rdydm.c
index e4b4d575d..43de37740 100644
--- a/src/rdydm.c
+++ b/src/rdydm.c
@@ -42,16 +42,14 @@ static PetscErrorCode CreateDMSection(DM dm, SectionFieldSpec fields) {
   PetscFunctionReturn(PETSC_SUCCESS);
 }
 
-PetscErrorCode CreateCellCenteredDMFromDM(DM dm, const SectionFieldSpec fields, DM *cc_dm) {
+PetscErrorCode CreateCellCenteredDMFromDM(DM dm, PetscInt refinement_level, const SectionFieldSpec fields, DM *cc_dm) {
   PetscFunctionBegin;
 
   PetscCall(DMClone(dm, cc_dm));
 
   PetscCall(CreateDMSection(*cc_dm, fields));
 
-  PetscInt refine_level;
-  PetscCall(DMGetRefineLevel(dm, &refine_level));
-  if (!refine_level) {
+  if (!refinement_level) {
     // copy adjacency info from the original DM
     PetscSF sf_migration, sf_natural;
     PetscCall(DMPlexGetMigrationSF(dm, &sf_migration));
@@ -59,6 +57,8 @@ PetscErrorCode CreateCellCenteredDMFromDM(DM dm, const SectionFieldSpec fields,
     PetscCall(DMSetNaturalSF(*cc_dm, sf_natural));
     PetscCall(DMSetUseNatural(*cc_dm, PETSC_TRUE));
     PetscCall(PetscSFDestroy(&sf_natural));
+  } else {
+    PetscCall(DMSetUseNatural(dm, PETSC_FALSE));
   }
   PetscSection section;
   PetscCall(DMGetLocalSection(*cc_dm, &section));
@@ -164,7 +164,7 @@ PetscErrorCode CreateDM(RDy rdy) {
 PetscErrorCode CreateAuxiliaryDM(RDy rdy) {
   PetscFunctionBegin;
 
-  PetscCall(CreateCellCenteredDMFromDM(rdy->dm, rdy->diag_fields, &rdy->aux_dm));
+  PetscCall(CreateCellCenteredDMFromDM(rdy->dm, rdy->num_refinements, rdy->diag_fields, &rdy->aux_dm));
   PetscFunctionReturn(PETSC_SUCCESS);
 }
 
@@ -183,7 +183,7 @@ PetscErrorCode CreateSedimentDM(RDy rdy) {
     snprintf(rdy->sediment_fields.field_component_names[0][i], MAX_NAME_LEN, "Class_%" PetscInt_FMT, i);
   }
 
-  PetscCall(CreateCellCenteredDMFromDM(rdy->dm, rdy->sediment_fields, &rdy->sediment_dm));
+  PetscCall(CreateCellCenteredDMFromDM(rdy->dm, rdy->num_refinements, rdy->sediment_fields, &rdy->sediment_dm));
 
   PetscFunctionReturn(PETSC_SUCCESS);
 }
@@ -200,7 +200,7 @@ PetscErrorCode CreateFlowDM(RDy rdy) {
   snprintf(rdy->flow_fields.field_component_names[0][1], MAX_NAME_LEN, "MomentumX");
   snprintf(rdy->flow_fields.field_component_names[0][2], MAX_NAME_LEN, "MomentumY");
 
-  PetscCall(CreateCellCenteredDMFromDM(rdy->dm, rdy->flow_fields, &rdy->flow_dm));
+  PetscCall(CreateCellCenteredDMFromDM(rdy->dm, rdy->num_refinements, rdy->flow_fields, &rdy->flow_dm));
 
   PetscFunctionReturn(PETSC_SUCCESS);
 }
diff --git a/src/rdymesh.c b/src/rdymesh.c
index 71efbde98..456f7cab6 100644
--- a/src/rdymesh.c
+++ b/src/rdymesh.c
@@ -81,10 +81,13 @@ static PetscErrorCode RDyCellsCreate(PetscInt num_cells, PetscInt nvertices_per_
 
 /// Creates a fully initialized RDyCells struct from a given DM.
 /// @param [in] dm A DM that provides cell data
+/// @param [in] nvertices_per_cell Maximum number of vertices per cell
+/// @param [in] nedges_per_cell Maximum number of edges per cell
 /// @param [out] cells A pointer to an RDyCells that stores allocated data.
+/// @param [out] num_cells A pointer to the number of cells
 ///
 /// @return 0 on success, or a non-zero error code on failure
-static PetscErrorCode RDyCellsCreateFromDM(DM dm, PetscInt nvertices_per_cell, PetscInt nedges_per_cell, RDyCells *cells) {
+static PetscErrorCode RDyCellsCreateFromDM(DM dm, PetscInt nvertices_per_cell, PetscInt nedges_per_cell, RDyCells *cells, PetscInt *num_cells) {
   PetscFunctionBegin;
 
   MPI_Comm comm;
@@ -101,8 +104,8 @@ static PetscErrorCode RDyCellsCreateFromDM(DM dm, PetscInt nvertices_per_cell, P
   DMPlexGetDepthStratum(dm, 0, &v_start, &v_end);
 
   // allocate cell storage
-  PetscInt num_cells = c_end - c_start;
-  PetscCall(RDyCellsCreate(num_cells, nvertices_per_cell, nedges_per_cell, cells));
+  *num_cells = c_end - c_start;
+  PetscCall(RDyCellsCreate(*num_cells, nvertices_per_cell, nedges_per_cell, cells));
 
   PetscInt num_owned_cells = 0;
   for (PetscInt c = c_start; c < c_end; c++) {
@@ -153,7 +156,7 @@ static PetscErrorCode RDyCellsCreateFromDM(DM dm, PetscInt nvertices_per_cell, P
   FILL(num_owned_cells, cells->owned_to_local, -1);
 
   PetscInt count = 0;
-  for (PetscInt icell = 0; icell < num_cells; icell++) {
+  for (PetscInt icell = 0; icell < *num_cells; icell++) {
     if (cells->is_owned[icell]) {
       cells->local_to_owned[icell] = count;
       cells->owned_to_local[count] = icell;
@@ -161,7 +164,7 @@ static PetscErrorCode RDyCellsCreateFromDM(DM dm, PetscInt nvertices_per_cell, P
     }
   }
 
-  for (PetscInt icell = 0; icell < num_cells; icell++) {
+  for (PetscInt icell = 0; icell < *num_cells; icell++) {
     if (!cells->is_owned[icell]) {
       cells->local_to_owned[icell] = count;
       count++;
@@ -171,7 +174,7 @@ static PetscErrorCode RDyCellsCreateFromDM(DM dm, PetscInt nvertices_per_cell, P
   // fetch global cell IDs.
   ISLocalToGlobalMapping map;
   PetscCall(DMGetLocalToGlobalMapping(dm, &map));
-  PetscCall(ISLocalToGlobalMappingApply(map, num_cells, cells->ids, cells->global_ids));
+  PetscCall(ISLocalToGlobalMappingApply(map, *num_cells, cells->ids, cells->global_ids));
 
   PetscFunctionReturn(PETSC_SUCCESS);
 }
@@ -187,6 +190,7 @@ static PetscErrorCode RDyCellsDestroy(RDyCells cells) {
   PetscCall(PetscFree(cells.global_ids));
   PetscCall(PetscFree(cells.natural_ids));
   PetscCall(PetscFree(cells.is_owned));
+  PetscCall(PetscFree(cells.local_to_owned));
   PetscCall(PetscFree(cells.num_vertices));
   PetscCall(PetscFree(cells.num_edges));
   PetscCall(PetscFree(cells.num_neighbors));
@@ -200,6 +204,8 @@ static PetscErrorCode RDyCellsDestroy(RDyCells cells) {
   PetscCall(PetscFree(cells.areas));
   PetscCall(PetscFree(cells.dz_dx));
   PetscCall(PetscFree(cells.dz_dy));
+  PetscCall(PetscFree(cells.local_to_owned));
+  PetscCall(PetscFree(cells.owned_to_local));
 
   PetscFunctionReturn(PETSC_SUCCESS);
 }
@@ -244,11 +250,15 @@ static PetscErrorCode RDyVerticesCreate(PetscInt num_vertices, PetscInt ncells_p
 
 /// Creates a fully initialized RDyVertices struct from a given DM.
 /// @param [in] dm A DM that provides vertex data
+/// @param [in] refinement_level Number of refinement levels for the mesh
+/// @param [in] ncells_per_vertex Maximum number of cells per vertex
+/// @param [in] nedges_per_vertex Maximum number of edges per vertex
 /// @param [out] vertices A pointer to an RDyVertices that stores allocated data.
-///
+/// @param [out] num_vertices A pointer to the number of vertices in the local mesh
+/// @param [out] num_vertices_global A pointer to the number of vertices in the global mesh
 /// @return 0 on success, or a non-zero error code on failure
-static PetscErrorCode RDyVerticesCreateFromDM(DM dm, PetscInt ncells_per_vertex, PetscInt nedges_per_vertex, RDyVertices *vertices,
-                                              PetscInt *num_vertices_global) {
+static PetscErrorCode RDyVerticesCreateFromDM(DM dm, PetscInt refinement_level, PetscInt ncells_per_vertex, PetscInt nedges_per_vertex,
+                                              RDyVertices *vertices, PetscInt *num_vertices, PetscInt *num_vertices_global) {
   PetscFunctionBegin;
 
   PetscInt dim;
@@ -262,8 +272,8 @@ static PetscErrorCode RDyVerticesCreateFromDM(DM dm, PetscInt ncells_per_vertex,
   DMPlexGetDepthStratum(dm, 0, &v_start, &v_end);
 
   // allocate vertex storage
-  PetscInt num_vertices = v_end - v_start;
-  PetscCall(RDyVerticesCreate(num_vertices, ncells_per_vertex, nedges_per_vertex, vertices));
+  *num_vertices = v_end - v_start;
+  PetscCall(RDyVerticesCreate(*num_vertices, ncells_per_vertex, nedges_per_vertex, vertices));
 
   PetscSection coordSection;
   Vec          coordinates;
@@ -311,9 +321,7 @@ static PetscErrorCode RDyVerticesCreateFromDM(DM dm, PetscInt ncells_per_vertex,
   VecRestoreArray(coordinates, &coords);
 
   // fetch global vertex IDs if mesh is not refined
-  PetscInt refine_level;
-  PetscCall(DMGetRefineLevel(dm, &refine_level));
-  if (!refine_level) {
+  if (!refinement_level) {
     PetscMPIInt commsize;
     MPI_Comm    comm;
     PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
@@ -361,6 +369,21 @@ static PetscErrorCode RDyVerticesCreateFromDM(DM dm, PetscInt ncells_per_vertex,
         vertices->global_ids[ivertex] -= min_vertex_idx;
       }
     }
+
+  } else {
+    IS globalVertexNumbers;
+    PetscCall(DMPlexGetVertexNumbering(dm, &globalVertexNumbers));
+
+    const PetscInt *ids;
+    PetscCall(ISGetIndices(globalVertexNumbers, &ids));
+
+    for (PetscInt v = v_start; v < v_end; v++) {
+      PetscInt ivertex = v - v_start;
+      if (ids[ivertex] < 0) vertices->global_ids[ivertex] = -ids[ivertex] - 1;
+      else vertices->global_ids[ivertex] = ids[ivertex];
+    }
+
+    PetscCall(ISRestoreIndices(globalVertexNumbers, &ids));
   }
 
   // compute total number of vertices
@@ -427,9 +450,10 @@ static PetscErrorCode RDyEdgesCreate(PetscInt num_edges, RDyEdges *edges) {
 /// Creates a fully initialized RDyEdges struct from a given DM.
 /// @param [in] dm A DM that provides edge data
 /// @param [out] edges A pointer to an RDyEdges that stores allocated data.
+/// @param [out] num_edges A pointer to the number of edges in the local mesh
 ///
 /// @return 0 on success, or a non-zero error code on failure
-static PetscErrorCode RDyEdgesCreateFromDM(DM dm, RDyEdges *edges) {
+static PetscErrorCode RDyEdgesCreateFromDM(DM dm, RDyEdges *edges, PetscInt *num_edges) {
   PetscFunctionBegin;
 
   MPI_Comm    comm;
@@ -448,8 +472,8 @@ static PetscErrorCode RDyEdgesCreateFromDM(DM dm, RDyEdges *edges) {
   DMPlexGetDepthStratum(dm, 0, &v_start, &v_end);
 
   // allocate edge storage
-  PetscInt num_edges = e_end - e_start;
-  PetscCall(RDyEdgesCreate(num_edges, edges));
+  *num_edges = e_end - e_start;
+  PetscCall(RDyEdgesCreate(*num_edges, edges));
 
   for (PetscInt e = e_start; e < e_end; e++) {
     PetscInt  iedge = e - e_start;
@@ -1007,13 +1031,19 @@ static PetscErrorCode CreateCellConnectionVector(DM dm, RDyMesh *mesh) {
             .num_field_components = {max_num_vertices},
             .field_names          = {"Cell Connections"},
   };
-  PetscCall(CreateCellCenteredDMFromDM(dm, field_spec, &local_dm));
+  PetscCall(CreateCellCenteredDMFromDM(dm, mesh->refine_level, field_spec, &local_dm));
 
   Vec          global_vec, natural_vec;
   PetscScalar *vec_ptr;
   PetscInt     n;
   PetscCall(DMCreateGlobalVector(local_dm, &global_vec));
-  PetscCall(DMPlexCreateNaturalVector(local_dm, &natural_vec));
+  PetscBool useNatural;
+  PetscCall(DMGetUseNatural(dm, &useNatural));
+  if (useNatural) {
+    PetscCall(DMPlexCreateNaturalVector(local_dm, &natural_vec));
+  } else {
+    PetscCall(VecDuplicate(global_vec, &natural_vec));
+  }
 
   PetscCall(VecGetLocalSize(global_vec, &n));
   PetscCheck(n == mesh->num_owned_cells * max_num_vertices, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG,
@@ -1035,8 +1065,12 @@ static PetscErrorCode CreateCellConnectionVector(DM dm, RDyMesh *mesh) {
 
   PetscCall(VecRestoreArray(global_vec, &vec_ptr));
 
-  PetscCall(DMPlexGlobalToNaturalBegin(local_dm, global_vec, natural_vec));
-  PetscCall(DMPlexGlobalToNaturalEnd(local_dm, global_vec, natural_vec));
+  if (useNatural) {
+    PetscCall(DMPlexGlobalToNaturalBegin(local_dm, global_vec, natural_vec));
+    PetscCall(DMPlexGlobalToNaturalEnd(local_dm, global_vec, natural_vec));
+  } else {
+    PetscCall(VecCopy(global_vec, natural_vec));
+  }
 
   if (0) {
     PetscCall(VecView(global_vec, PETSC_VIEWER_STDOUT_WORLD));
@@ -1129,16 +1163,26 @@ static PetscErrorCode CreateCellCentroidVectors(DM dm, RDyMesh *mesh) {
       .field_names          = {"Cell Coordinates"},
   };
 
-  PetscCall(CreateCellCenteredDMFromDM(dm, local_spec, &local_dm));
+  PetscCall(CreateCellCenteredDMFromDM(dm, mesh->refine_level, local_spec, &local_dm));
 
   Vec global_vec, natural_vec;
   PetscCall(DMCreateGlobalVector(local_dm, &global_vec));
-  PetscCall(DMPlexCreateNaturalVector(local_dm, &natural_vec));
 
-  // create the Vec for storing coordinates in nautral order
-  PetscCall(DMPlexCreateNaturalVector(local_dm, &mesh->output.xc));
-  PetscCall(DMPlexCreateNaturalVector(local_dm, &mesh->output.yc));
-  PetscCall(DMPlexCreateNaturalVector(local_dm, &mesh->output.zc));
+  PetscBool useNatural;
+  PetscCall(DMGetUseNatural(dm, &useNatural));
+  if (useNatural) {
+    PetscCall(DMPlexCreateNaturalVector(local_dm, &natural_vec));
+
+    // create the Vec for storing coordinates in nautral order
+    PetscCall(DMPlexCreateNaturalVector(local_dm, &mesh->output.xc));
+    PetscCall(DMPlexCreateNaturalVector(local_dm, &mesh->output.yc));
+    PetscCall(DMPlexCreateNaturalVector(local_dm, &mesh->output.zc));
+  } else {
+    PetscCall(VecDuplicate(global_vec, &natural_vec));
+    PetscCall(VecDuplicate(global_vec, &mesh->output.xc));
+    PetscCall(VecDuplicate(global_vec, &mesh->output.yc));
+    PetscCall(VecDuplicate(global_vec, &mesh->output.zc));
+  }
 
   // set names to the Vecs
   PetscCall((PetscObjectSetName((PetscObject)mesh->output.xc, "XC")));
@@ -1159,8 +1203,12 @@ static PetscErrorCode CreateCellCentroidVectors(DM dm, RDyMesh *mesh) {
     PetscCall(VecRestoreArray(global_vec, &vec_ptr));
 
     // scatter the data from global to natural order
-    PetscCall(DMPlexGlobalToNaturalBegin(local_dm, global_vec, natural_vec));
-    PetscCall(DMPlexGlobalToNaturalEnd(local_dm, global_vec, natural_vec));
+    if (useNatural) {
+      PetscCall(DMPlexGlobalToNaturalBegin(local_dm, global_vec, natural_vec));
+      PetscCall(DMPlexGlobalToNaturalEnd(local_dm, global_vec, natural_vec));
+    } else {
+      PetscCall(VecCopy(global_vec, natural_vec));
+    }
 
     // save the coordinate in appropriate Vec
     switch (idim) {
@@ -1257,38 +1305,25 @@ static PetscErrorCode DetermineMaxAttributesForCellsAndVertices(DM dm, RDyMesh *
 
 /// Creates an RDyMesh from a PETSc DM.
 /// @param [in] dm A PETSc DM
+/// @param [in] refinement_level The level of refinement for the mesh.
 /// @param [out] mesh A pointer to an RDyMesh that stores allocated data.
 /// @return 0 on success, or a non-zero error code on failure
-PetscErrorCode RDyMeshCreateFromDM(DM dm, RDyMesh *mesh) {
+PetscErrorCode RDyMeshCreateFromDM(DM dm, PetscInt refinement_level, RDyMesh *mesh) {
   PetscFunctionBegin;
 
   PetscCall(PetscMemzero(mesh, sizeof(RDyMesh)));
 
   // save the number of refinements
-  PetscCall(DMGetRefineLevel(dm, &mesh->refine_level));
-
-  // Determine the number of cells in the mesh
-  PetscInt c_start, c_end;
-  PetscCall(DMPlexGetHeightStratum(dm, 0, &c_start, &c_end));
-  mesh->num_cells = c_end - c_start;
-
-  // Determine the number of edges in the mesh
-  PetscInt e_start, e_end;
-  PetscCall(DMPlexGetDepthStratum(dm, 1, &e_start, &e_end));
-  mesh->num_edges = e_end - e_start;
-
-  // Determine the number of vertices in the mesh
-  PetscInt v_start, v_end;
-  PetscCall(DMPlexGetDepthStratum(dm, 0, &v_start, &v_end));
-  mesh->num_vertices = v_end - v_start;
+  mesh->refine_level = refinement_level;
 
   // Determine few max attributes per cell and per vertex
   PetscCall(DetermineMaxAttributesForCellsAndVertices(dm, mesh));
 
   // Create mesh elements from the DM
-  PetscCall(RDyCellsCreateFromDM(dm, mesh->max_nvertices_per_cell, mesh->max_nedges_per_cell, &mesh->cells));
-  PetscCall(RDyEdgesCreateFromDM(dm, &mesh->edges));
-  PetscCall(RDyVerticesCreateFromDM(dm, mesh->max_ncells_per_vertex, mesh->max_nedges_per_vertex, &mesh->vertices, &mesh->num_vertices_global));
+  PetscCall(RDyCellsCreateFromDM(dm, mesh->max_nvertices_per_cell, mesh->max_nedges_per_cell, &mesh->cells, &mesh->num_cells));
+  PetscCall(RDyEdgesCreateFromDM(dm, &mesh->edges, &mesh->num_edges));
+  PetscCall(RDyVerticesCreateFromDM(dm, refinement_level, mesh->max_ncells_per_vertex, mesh->max_nedges_per_vertex, &mesh->vertices,
+                                    &mesh->num_vertices, &mesh->num_vertices_global));
   PetscCall(ComputeAdditionalEdgeAttributes(dm, mesh));
   PetscCall(ComputeAdditionalCellAttributes(dm, mesh));
 
@@ -1307,11 +1342,9 @@ PetscErrorCode RDyMeshCreateFromDM(DM dm, RDyMesh *mesh) {
 
   PetscCall(MPI_Allreduce(&mesh->num_owned_cells, &mesh->num_cells_global, 1, MPI_INTEGER, MPI_SUM, comm));
 
-  if (!mesh->refine_level) {
-    PetscCall(CreateCoordinatesVectorInNaturalOrder(comm, mesh));
-    PetscCall(CreateCellConnectionVector(dm, mesh));
-    PetscCall(CreateCellCentroidVectors(dm, mesh));
-  }
+  PetscCall(CreateCoordinatesVectorInNaturalOrder(comm, mesh));
+  PetscCall(CreateCellConnectionVector(dm, mesh));
+  PetscCall(CreateCellCentroidVectors(dm, mesh));
 
   PetscFunctionReturn(PETSC_SUCCESS);
 }
@@ -1322,16 +1355,27 @@ PetscErrorCode RDyMeshCreateFromDM(DM dm, RDyMesh *mesh) {
 /// @return 0 on success, or a non-zero error code on failure
 PetscErrorCode RDyMeshDestroy(RDyMesh mesh) {
   PetscFunctionBegin;
-  PetscCall(RDyCellsDestroy(mesh.cells));
-  PetscCall(RDyEdgesDestroy(mesh.edges));
-  PetscCall(RDyVerticesDestroy(mesh.vertices));
-
-  if (!mesh.refine_level) {
-    PetscCall(VecDestroy(&mesh.output.vertices_xyz_norder));
-    PetscCall(VecDestroy(&mesh.output.cell_conns_norder));
-    PetscCall(VecDestroy(&mesh.output.xc));
-    PetscCall(VecDestroy(&mesh.output.yc));
-    PetscCall(VecDestroy(&mesh.output.zc));
+
+  if (mesh.num_cells) {
+    PetscCall(RDyCellsDestroy(mesh.cells));
+    mesh.num_cells = 0;
   }
+
+  if (mesh.num_edges) {
+    PetscCall(RDyEdgesDestroy(mesh.edges));
+    mesh.num_edges = 0;
+  }
+
+  if (mesh.num_vertices) {
+    PetscCall(RDyVerticesDestroy(mesh.vertices));
+    mesh.num_vertices = 0;
+  }
+
+  PetscCall(VecDestroy(&mesh.output.vertices_xyz_norder));
+  PetscCall(VecDestroy(&mesh.output.cell_conns_norder));
+  PetscCall(VecDestroy(&mesh.output.xc));
+  PetscCall(VecDestroy(&mesh.output.yc));
+  PetscCall(VecDestroy(&mesh.output.zc));
+
   PetscFunctionReturn(PETSC_SUCCESS);
 }
diff --git a/src/rdymms.c b/src/rdymms.c
index 7b1cde4fe..bbcdc2a43 100644
--- a/src/rdymms.c
+++ b/src/rdymms.c
@@ -229,7 +229,7 @@ PetscErrorCode RDyMMSSetup(RDy rdy) {
   // note: this must be done after global vectors are created so a global
   // note: section exists for the DM
   RDyLogDebug(rdy, "Creating FV mesh...");
-  PetscCall(RDyMeshCreateFromDM(rdy->dm, &rdy->mesh));
+  PetscCall(RDyMeshCreateFromDM(rdy->dm, 0, &rdy->mesh));
 
   RDyLogDebug(rdy, "Initializing regions...");
   PetscCall(InitRegions(rdy));
diff --git a/src/rdyrefine.c b/src/rdyrefine.c
new file mode 100644
index 000000000..95724c841
--- /dev/null
+++ b/src/rdyrefine.c
@@ -0,0 +1,577 @@
+#include <muParserDLL.h>
+#include <petsc/private/dmpleximpl.h> /*I      "petscdmplex.h"   I*/
+#include <petscdmceed.h>
+#include <petscdmplex.h>
+#include <petscsys.h>
+#include <private/rdycoreimpl.h>
+#include <private/rdydmimpl.h>
+#include <private/rdymathimpl.h>
+#include <private/rdysweimpl.h>
+typedef struct {
+  PetscInt  adapt;  /* Flag for adaptation of the surface mesh */
+  PetscBool metric; /* Flag to use metric adaptation, instead of tagging */
+} AppCtx;
+
+static PetscErrorCode ProcessOptions(MPI_Comm comm, AppCtx *options) {
+  PetscMPIInt size;
+
+  PetscFunctionBeginUser;
+  options->adapt  = 1;
+  options->metric = PETSC_FALSE;
+  PetscCallMPI(MPI_Comm_size(comm, &size));
+
+  PetscOptionsBegin(comm, "", "Meshing Interpolation Test Options", "DMPLEX");
+  PetscCall(PetscOptionsInt("-adapt", "Number of adaptation steps mesh", "ex10.c", options->adapt, &options->adapt, NULL));
+  PetscCall(PetscOptionsBool("-metric", "Flag for metric refinement", "ex41.c", options->metric, &options->metric, NULL));
+  PetscOptionsEnd();
+  PetscFunctionReturn(PETSC_SUCCESS);
+}
+
+static PetscErrorCode CreateAdaptLabel(DM dm, AppCtx *ctx, DMLabel *adaptLabel) {
+  /* PetscMPIInt rank; */
+  DMLabel  label;
+  PetscInt cStart, cEnd, c;
+
+  PetscFunctionBegin;
+  PetscCall(DMLabelCreate(PETSC_COMM_SELF, "Adaptation Label", adaptLabel));
+  label = *adaptLabel;
+  PetscCall(DMGetCoordinatesLocalSetUp(dm));
+  PetscCall(DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd));
+  for (c = cStart; c < cEnd; ++c) {
+    PetscReal centroid[3], volume, x, y;
+
+    PetscCall(DMPlexComputeCellGeometryFVM(dm, c, &volume, centroid, NULL));
+    x = centroid[0];
+    y = centroid[1];
+    /* Headwaters are (0.0,0.25)--(0.1,0.75) */
+    if ((x >= 0.0 && x < 1.) && (y >= 3. && y <= 4.)) {
+      PetscCall(DMLabelSetValue(label, c, DM_ADAPT_REFINE));
+      // PetscCall(PetscPrintf(PETSC_COMM_SELF, "refine: %" PetscInt_FMT "\n", c));
+    }
+  }
+  PetscFunctionReturn(PETSC_SUCCESS);
+}
+
+/// @brief Creates matrices for the interpolation between coarse and fine meshes.
+/// @param dm           The fine DM
+/// @param CoarseToFine Matrix for interpolating local Vec from coarse to fine grid
+/// @param FineToCoarse Matrix for interpolating local Vec from fine to coarse grid
+/// @return PETSC_SUCESS on success
+static PetscErrorCode ConstructRefineTree(DM dm, Mat CoarseToFine, Mat FineToCoarse) {
+  DMPlexTransform tr;
+  DM              odm;
+  PetscInt        cStart, cEnd, bs, Istart, Jstart;
+  PetscScalar     val = 1.0;
+
+  PetscFunctionBegin;
+  PetscCall(MatGetBlockSize(CoarseToFine, &bs));
+  PetscCall(DMPlexGetTransform(dm, &tr));
+  if (!tr) PetscFunctionReturn(PETSC_SUCCESS);
+  PetscCall(DMPlexTransformGetDM(tr, &odm));
+  PetscCall(DMPlexGetHeightStratum(odm, 0, &cStart, &cEnd));
+  PetscCall(MatGetOwnershipRange(CoarseToFine, &Istart, NULL));
+  PetscCall(MatGetOwnershipRangeColumn(CoarseToFine, &Jstart, NULL));
+  for (PetscInt c = cStart; c < cEnd; ++c) {
+    DMPolytopeType  ct;
+    DMPolytopeType *rct;
+    PetscInt       *rsize, *rcone, *rornt;
+    PetscInt        Nct, dim, pNew = 0;
+
+    // PetscCall(PetscPrintf(PETSC_COMM_SELF, "Cell %" PetscInt_FMT " produced new cells", c));
+    PetscCall(DMPlexGetCellType(odm, c, &ct));
+    dim = DMPolytopeTypeGetDim(ct);
+    PetscCall(DMPlexTransformCellTransform(tr, ct, c, NULL, &Nct, &rct, &rsize, &rcone, &rornt));
+    for (PetscInt n = 0; n < Nct; ++n) {
+      if (DMPolytopeTypeGetDim(rct[n]) != dim) continue;
+      for (PetscInt r = 0; r < rsize[n]; ++r) {
+        PetscCall(DMPlexTransformGetTargetPoint(tr, ct, rct[n], c, r, &pNew));
+        // PetscCall(PetscPrintf(PETSC_COMM_SELF, " %" PetscInt_FMT, pNew));
+        for (PetscInt i = 0; i < bs; i++) {
+          PetscCall(MatSetValue(CoarseToFine, Istart + bs * (pNew - 0) + i, Jstart + bs * (c - cStart) + i, val, INSERT_VALUES));
+          PetscCall(MatSetValue(FineToCoarse, Jstart + bs * (c - cStart) + i, Istart + bs * (pNew - 0) + i, 1.0 / rsize[n], INSERT_VALUES));
+        }
+      }
+    }
+    // PetscCall(PetscPrintf(PETSC_COMM_SELF, "\n"));
+  }
+
+  PetscCall(MatSetFromOptions(CoarseToFine));
+  PetscCall(MatAssemblyBegin(CoarseToFine, MAT_FINAL_ASSEMBLY));
+  PetscCall(MatAssemblyEnd(CoarseToFine, MAT_FINAL_ASSEMBLY));
+  PetscCall(MatViewFromOptions(CoarseToFine, NULL, "-adapt_c2f_mat_view"));
+
+  PetscCall(MatSetFromOptions(FineToCoarse));
+  PetscCall(MatAssemblyBegin(FineToCoarse, MAT_FINAL_ASSEMBLY));
+  PetscCall(MatAssemblyEnd(FineToCoarse, MAT_FINAL_ASSEMBLY));
+  PetscCall(MatViewFromOptions(FineToCoarse, NULL, "-adapt_f2c_mat_view"));
+
+  PetscFunctionReturn(PETSC_SUCCESS);
+}
+
+static PetscErrorCode AdaptMesh(DM dm, const PetscInt bs, DM *dm_fine, Mat *CoarseToFine, Mat *FineToCoarse, AppCtx *ctx) {
+  PetscFunctionBeginUser;
+  PetscCall(DMViewFromOptions(dm, NULL, "-adapt_pre_dm_view"));
+
+  DM       dmAdapt;
+  DMLabel  adaptLabel;
+  PetscInt d_nz = 4, ccStart, ccEnd, fcStart, fcEnd;
+  char     opt[128];
+
+  PetscCall(CreateAdaptLabel(dm, ctx, &adaptLabel));
+  PetscCall(DMPlexSetSaveTransform(dm, PETSC_TRUE));
+  PetscCall(DMAdaptLabel(dm, adaptLabel, &dmAdapt));  // DMRefine
+  PetscCall(DMLabelDestroy(&adaptLabel));
+
+  if (!dmAdapt) {
+    PetscCheck(PETSC_TRUE, PETSC_COMM_WORLD, PETSC_ERR_USER, "Refinement failed.");
+  }
+
+  PetscCall(PetscObjectSetName((PetscObject)dmAdapt, "Adapted Mesh"));
+  PetscCall(PetscSNPrintf(opt, 128, "-adapt_dm_view"));
+  PetscCall(DMViewFromOptions(dmAdapt, NULL, opt));
+
+  // make interpolation matrix
+  PetscCall(DMPlexGetHeightStratum(dm, 0, &ccStart, &ccEnd));
+  PetscCall(DMPlexGetHeightStratum(dmAdapt, 0, &fcStart, &fcEnd));
+  PetscCall(MatCreateSeqAIJ(PETSC_COMM_SELF, bs * (fcEnd - fcStart), bs * (ccEnd - ccStart), d_nz, NULL, CoarseToFine));
+  PetscCall(MatCreateSeqAIJ(PETSC_COMM_SELF, bs * (ccEnd - ccStart), bs * (fcEnd - fcStart), d_nz, NULL, FineToCoarse));
+  PetscCall(MatSetBlockSize(*CoarseToFine, bs));
+  PetscCall(MatSetBlockSize(*FineToCoarse, bs));
+  PetscCall(ConstructRefineTree(dmAdapt, *CoarseToFine, *FineToCoarse));
+
+  *dm_fine = dmAdapt;
+
+  PetscFunctionReturn(PETSC_SUCCESS);
+}
+
+/// @brief Given the coarse-to-fine mesh matrix, determine the mapping from coarse cells to fine cells.
+/// @param rdy_coarse       RDy struct corresponding to the coarse mesh
+/// @param CoarseToFine     Matrix to remap data from coarse to fine mesh
+/// @param X_coarse         A local Vec on the coarse mesh
+/// @param X_fine           A local Vec on the fine mesh
+/// @param fineIsOwned      For all fine cells, whether the cell is owned by this rank
+/// @param fineToCoarseMap  For all fine cells, the corresponding coarse cell ID
+/// @param coarseToFineMap  For all coarse cells, the corresponding fine cell IDs
+/// @param coarseNumFine    For all coarse cells, the number of fine cells
+/// @param coarseOffsetFine Offset for each coarse cell in the coarseToFineMap
+/// @return PETSC_SUCESS on success
+static PetscErrorCode DetermineCoarseToFineCellMapping(RDy rdy_coarse, Mat CoarseToFine, Vec X_coarse, Vec X_fine, PetscInt *fineIsOwned,
+                                                       PetscInt *fineToCoarseMap, PetscInt *coarseToFineMap, PetscInt *coarseNumFine,
+                                                       PetscInt *coarseOffsetFine) {
+  PetscFunctionBeginUser;
+
+  RDyMesh  *mesh  = &rdy_coarse->mesh;
+  RDyCells *cells = &mesh->cells;
+
+  PetscInt ndof;
+  PetscCall(VecGetBlockSize(X_coarse, &ndof));
+
+  // create Vec to mark owned cells
+  Vec X_coarse_owned, X_fine_owned;
+  PetscCall(VecDuplicate(X_coarse, &X_coarse_owned));
+  PetscCall(VecDuplicate(X_fine, &X_fine_owned));
+
+  PetscScalar *x_ptr_coarse, *x_ptr_coarse_owned, *x_ptr_fine, *x_ptr_fine_owned;
+
+  // - Put the local cell ID in the first position of X_coarse
+  //   and -1 in the rest of the positions
+  // - Put 1 or 0 in the first position of X_coarse_owned to indicate
+  //   if the cell is owned or not, and -1 in the rest of the positions
+  PetscCall(VecGetArray(X_coarse, &x_ptr_coarse));
+  PetscCall(VecGetArray(X_coarse_owned, &x_ptr_coarse_owned));
+
+  for (PetscInt c = 0; c < mesh->num_cells; ++c) {
+    x_ptr_coarse[c * ndof]       = c;
+    x_ptr_coarse_owned[c * ndof] = cells->is_owned[c];
+
+    // put -1 in the rest of the positions
+    for (PetscInt idof = 1; idof < ndof; idof++) {
+      x_ptr_coarse[c * ndof + idof]       = -1;
+      x_ptr_coarse_owned[c * ndof + idof] = -1;
+    }
+  }
+  PetscCall(VecRestoreArray(X_coarse, &x_ptr_coarse));
+  PetscCall(VecRestoreArray(X_coarse_owned, &x_ptr_coarse_owned));
+
+  // Multiply the coarse-to-fine matrix with the coarse vector
+  // to get the fine vector
+  PetscCall(MatMult(CoarseToFine, X_coarse, X_fine));
+  PetscCall(MatMult(CoarseToFine, X_coarse_owned, X_fine_owned));
+
+  PetscInt numFine, numCoarse;
+  PetscCall(VecGetLocalSize(X_fine, &numFine));
+  PetscCall(VecGetLocalSize(X_coarse, &numCoarse));
+  numFine   = (PetscInt)numFine / ndof;
+  numCoarse = (PetscInt)numCoarse / ndof;
+
+  // For each fine cell, determine the corresponding coarse cell
+  // and keep track of the number of fine cells corresponding to each coarse cell
+  PetscCall(VecGetArray(X_fine, &x_ptr_fine));
+  PetscCall(VecGetArray(X_fine_owned, &x_ptr_fine_owned));
+
+  for (PetscInt c = 0; c < numFine; c++) {
+    // get the local cell ID of the coarse cell
+    PetscInt coarse_cell_id = (PetscInt)x_ptr_fine[c * ndof];
+
+    fineToCoarseMap[c] = coarse_cell_id;
+    fineIsOwned[c]     = (PetscInt)x_ptr_fine_owned[c * ndof];
+    coarseNumFine[coarse_cell_id]++;
+  }
+
+  // Before creating the coarseToFineMap, we need to create the offsets
+  // for each coarse cell. The offsets are the number of fine cells
+  // corresponding to each coarse cell. The offsets are used to
+  // create the coarseToFineMap.
+  coarseOffsetFine[0] = 0;
+  for (PetscInt c = 1; c < numCoarse; c++) {
+    coarseOffsetFine[c]  = coarseOffsetFine[c - 1] + coarseNumFine[c - 1];
+    coarseNumFine[c - 1] = 0;
+  }
+  coarseNumFine[numCoarse - 1] = 0;
+
+  // Now traverse the fine vector again and save the fine cell ID
+  // for each coarse cell in the coarseToFineMap.
+  for (PetscInt c = 0; c < numFine; c++) {
+    // get the natural cell ID
+    PetscInt coarse_cell_id = (PetscInt)x_ptr_fine[c * ndof];
+
+    PetscInt offset   = coarseOffsetFine[coarse_cell_id];
+    PetscInt num_fine = coarseNumFine[coarse_cell_id];
+
+    coarseToFineMap[offset + num_fine] = c;
+
+    // increment the number of fine cells for this coarse cell
+    coarseNumFine[coarse_cell_id]++;
+  }
+  PetscCall(VecRestoreArray(X_fine, &x_ptr_fine));
+  PetscCall(VecRestoreArray(X_fine_owned, &x_ptr_fine_owned));
+
+  // clean up
+  PetscCall(VecDestroy(&X_coarse_owned));
+  PetscCall(VecDestroy(&X_fine_owned));
+
+  PetscFunctionReturn(PETSC_SUCCESS);
+}
+
+/// @brief For all the regions in the coarse mesh, create the corresponding regions in the fine mesh.
+/// @param rdy_coarse       RDy struct corresponding to the coarse mesh
+/// @param numFine          Number of local cells in the fine mesh
+/// @param fineIsOwned      For all fine cells, whether the cell is owned by this rank
+/// @param coarseToFineMap  For all coarse cells, the corresponding fine cell IDs
+/// @param coarseNumFine    For all coarse cells, the number of fine cells that the coarse cell was split into
+/// @param coarseOffsetFine For all coarse cells, the offset in the coarseToFineMap for the coarse cell
+/// @param num_regions      Number of regions for the fine mesh
+/// @param regions          Regions for the fine mesh
+/// @return PETSC_SUCESS on success
+static PetscErrorCode CreateRefinedRegionsFromCoarseRDy(RDy rdy_coarse, PetscInt numFine, PetscInt *fineIsOwned, PetscInt *coarseToFineMap,
+                                                        PetscInt *coarseNumFine, PetscInt *coarseOffsetFine, PetscInt *num_regions,
+                                                        RDyRegion **regions) {
+  PetscFunctionBeginUser;
+
+  RDyMesh  *mesh  = &rdy_coarse->mesh;
+  RDyCells *cells = &mesh->cells;
+
+  *num_regions = rdy_coarse->num_regions;
+
+  PetscCall(PetscCalloc1(*num_regions, regions));
+
+  // For fine cells, determine the global IDs of local cells, which depends on if a local cell
+  // is owned or not.
+  PetscInt *fine_LocaltoGlobalIDs, count = 0;
+  PetscCall(PetscCalloc1(numFine, &fine_LocaltoGlobalIDs));
+  for (PetscInt c = 0; c < numFine; c++) {
+    if (fineIsOwned[c]) {
+      fine_LocaltoGlobalIDs[c] = count;
+      count++;
+    } else {
+      fine_LocaltoGlobalIDs[c] = -1;
+    }
+  }
+
+  for (PetscInt r = 0; r < rdy_coarse->num_regions; ++r) {
+    RDyRegion *region_coarse = &rdy_coarse->regions[r];
+    RDyRegion *region_fine   = (*regions + r);
+
+    region_fine->id = region_coarse->id;
+    strcpy(region_fine->name, region_coarse->name);
+
+    if (region_coarse->num_owned_cells > 0) {
+      //
+      // Determine the global IDs of "owned" cells in the fine region
+      //
+
+      // 1. Determine the number of owned fine cells in the fine region by
+      //    traversing the coarse region and counting the number of fine cells
+      //    corresponding to each "owened" coarse cell.
+      PetscInt num_owned_fine_cells = 0;
+      for (PetscInt c = 0; c < region_coarse->num_owned_cells; ++c) {
+        PetscInt coarse_global_id = region_coarse->owned_cell_global_ids[c];
+        PetscInt coarse_local_id  = cells->owned_to_local[coarse_global_id];
+        num_owned_fine_cells += coarseNumFine[coarse_local_id];
+      }
+
+      // 2. Allocate memory
+      region_fine->num_owned_cells = num_owned_fine_cells;
+      PetscCall(PetscCalloc1(region_fine->num_owned_cells, &region_fine->owned_cell_global_ids));
+
+      // 3. Fill the global IDs of the owned cells in the fine region.
+      //    The coarseOffsetFine and coarseToFineMap use "local IDs".
+      //    So, the global ID of the corase cell must be converted to local ID
+      //    before using the coarseOffsetFine and coarseToFineMap.
+      count = 0;
+      for (PetscInt c = 0; c < region_coarse->num_owned_cells; ++c) {
+        PetscInt coarse_global_id = region_coarse->owned_cell_global_ids[c];
+        PetscInt coarse_local_id  = cells->owned_to_local[coarse_global_id];
+        PetscInt offset           = coarseOffsetFine[coarse_local_id];
+
+        for (PetscInt j = 0; j < coarseNumFine[coarse_local_id]; j++) {
+          PetscInt fine_local_id                    = coarseToFineMap[offset + j];
+          region_fine->owned_cell_global_ids[count] = fine_LocaltoGlobalIDs[fine_local_id];
+          count++;
+        }
+      }
+
+      //
+      // Determine the local IDs of "local" cells in the fine region
+      //
+
+      // 1. Determine the number of local fine cells in the fine region by
+      //    traversing the coarse region and counting the number of fine cells
+      //    corresponding to each coarse cell.
+      PetscInt num_local_fine_cells = 0;
+      for (PetscInt c = 0; c < region_coarse->num_local_cells; ++c) {
+        PetscInt coarse_local_id = region_coarse->cell_local_ids[c];
+        num_local_fine_cells += coarseNumFine[coarse_local_id];
+      }
+
+      // 2. Allocate memory
+      region_fine->num_local_cells = num_local_fine_cells;
+      PetscCall(PetscCalloc1(region_fine->num_local_cells, &region_fine->cell_local_ids));
+
+      // 3. Fill the local IDs of the fine cells in the fine region.
+      count = 0;
+      for (PetscInt c = 0; c < region_coarse->num_local_cells; ++c) {
+        PetscInt coarse_local_id = region_coarse->cell_local_ids[c];
+        PetscInt offset          = coarseOffsetFine[coarse_local_id];
+
+        for (PetscInt j = 0; j < coarseNumFine[coarse_local_id]; j++) {
+          region_fine->cell_local_ids[count] = coarseToFineMap[offset + j];
+          count++;
+        }
+      }
+
+    } else {
+      region_fine->num_local_cells = 0;
+      region_fine->num_owned_cells = 0;
+    }
+  }
+
+  PetscCall(PetscFree(fine_LocaltoGlobalIDs));
+
+  PetscFunctionReturn(PETSC_SUCCESS);
+}
+
+extern PetscErrorCode InitOperator(RDy rdy);
+extern PetscErrorCode InitMaterialProperties(RDy rdy);
+extern PetscErrorCode InitSolver(RDy rdy);
+extern PetscErrorCode InitDirichletBoundaryConditions(RDy rdy);
+extern PetscErrorCode InitSourceConditions(RDy rdy);
+
+PetscErrorCode RDyRefine(RDy rdy) {
+  AppCtx   user;
+  Mat      CoarseToFine, FineToCoarse;
+  DM       dm_fine;
+  Vec      U_coarse_local, U_fine_local;
+  PetscInt ndof_coarse;
+  PetscFunctionBeginUser;
+  PetscCall(VecGetBlockSize(rdy->u_global, &ndof_coarse));
+  PetscCall(ProcessOptions(PETSC_COMM_WORLD, &user));
+
+  /* Adapt */
+  PetscCall(AdaptMesh(rdy->dm, ndof_coarse, &dm_fine, &CoarseToFine, &FineToCoarse, &user));
+  PetscCall(DMLocalizeCoordinates(dm_fine));
+  PetscCall(DMViewFromOptions(dm_fine, NULL, "-dm_fine_view"));
+  PetscCall(DMSetCoarseDM(dm_fine, rdy->dm));
+  PetscCall(DMGetCoordinatesLocalSetUp(dm_fine));
+
+  if (0) {
+    PetscSection sec;
+    PetscCall(DMGetLocalSection(rdy->dm, &sec));
+    PetscInt nfields;
+    PetscSectionGetNumFields(sec, &nfields);
+    printf("++++++++++++++++++++++++++++++++++++\n");
+    printf("About coarse DM: \n");
+    printf("nfields = %d\n", (int)nfields);
+    for (PetscInt f = 0; f < nfields; f++) {
+      PetscInt ncomp;
+      PetscCall(PetscSectionGetFieldComponents(sec, f, &ncomp));
+      printf("  field = %d; num_component = %d\n", (int)f, (int)ncomp);
+      for (PetscInt c = 0; c < ncomp; c++) {
+        const char *comp_name;
+        PetscSectionGetComponentName(sec, f, c, &comp_name);
+        printf("    field = %d; component = %d; comp_name = %s\n", (int)f, (int)c, comp_name);
+      }
+    }
+  }
+
+  if (0) {
+    PetscSection sec;
+    PetscCall(DMGetLocalSection(dm_fine, &sec));
+    PetscInt nfields;
+    PetscSectionGetNumFields(sec, &nfields);
+    printf("++++++++++++++++++++++++++++++++++++\n");
+    printf("About refined DM: \n");
+    printf("nfields = %d\n", (int)nfields);
+    for (PetscInt f = 0; f < nfields; f++) {
+      PetscInt ncomp;
+      PetscCall(PetscSectionGetFieldComponents(sec, f, &ncomp));
+      printf("  field = %d; num_component = %d\n", (int)f, (int)ncomp);
+      for (PetscInt c = 0; c < ncomp; c++) {
+        const char *comp_name;
+        PetscSectionGetComponentName(sec, f, c, &comp_name);
+        printf("    field = %d; component = %d; comp_name = %s\n", (int)f, (int)c, comp_name);
+      }
+    }
+    printf("++++++++++++++++++++++++++++++++++++\n");
+  }
+
+  {
+    PetscSection coarse_sec, fine_sec;
+    PetscCall(DMGetLocalSection(rdy->dm, &coarse_sec));
+    PetscCall(DMGetLocalSection(dm_fine, &fine_sec));
+    PetscInt nfields;
+    PetscSectionGetNumFields(coarse_sec, &nfields);
+    for (PetscInt f = 0; f < nfields; f++) {
+      PetscInt ncomp;
+      PetscCall(PetscSectionGetFieldComponents(coarse_sec, f, &ncomp));
+      for (PetscInt c = 0; c < ncomp; c++) {
+        const char *comp_name;
+        PetscSectionGetComponentName(coarse_sec, f, c, &comp_name);
+        PetscSectionSetComponentName(fine_sec, f, c, comp_name);
+      }
+    }
+  }
+
+  // create a local vector for the coarse mesh
+  PetscCall(VecDuplicate(rdy->u_local, &U_coarse_local));
+
+  // determine the mapping of cells from fine to coarse mesh
+  PetscCall(DMCreateLocalVector(dm_fine, &U_fine_local));
+
+  PetscInt *fineToCoarseMap, *fineIsOwned, *coarseToFineMap, *coarseNumFine, *coarseOffsetFine;
+
+  PetscInt numFine, numCoarse;
+
+  PetscCall(VecGetLocalSize(U_coarse_local, &numCoarse));
+  PetscCall(PetscCalloc1(numCoarse, &coarseNumFine));
+  PetscCall(PetscCalloc1(numCoarse, &coarseOffsetFine));
+  for (PetscInt c = 0; c < numCoarse; c++) {
+    coarseNumFine[c] = 0;
+  }
+
+  PetscCall(VecGetLocalSize(U_fine_local, &numFine));
+  PetscCall(PetscCalloc1(numFine, &fineIsOwned));
+  PetscCall(PetscCalloc1(numFine, &fineToCoarseMap));
+  PetscCall(PetscCalloc1(numFine, &coarseToFineMap));
+
+  // determine the mapping of cells from coarse to fine mesh
+  PetscCall(DetermineCoarseToFineCellMapping(rdy, CoarseToFine, U_coarse_local, U_fine_local, fineIsOwned, fineToCoarseMap, coarseToFineMap,
+                                             coarseNumFine, coarseOffsetFine));
+
+  // create data structure for the refined regions from existing coarse regions
+  RDyRegion *refined_regions = NULL;
+  PetscInt   num_regions;
+
+  PetscMPIInt rank;
+  PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm_fine), &rank));
+  PetscCall(
+      CreateRefinedRegionsFromCoarseRDy(rdy, numFine, fineIsOwned, coarseToFineMap, coarseNumFine, coarseOffsetFine, &num_regions, &refined_regions));
+  PetscCall(PetscFree(fineIsOwned));
+  PetscCall(PetscFree(coarseNumFine));
+  PetscCall(PetscFree(coarseOffsetFine));
+  PetscCall(PetscFree(coarseToFineMap));
+  PetscCall(PetscFree(fineToCoarseMap));
+
+  // make a copy of the old solution
+  PetscCall(DMGlobalToLocal(rdy->dm, rdy->u_global, INSERT_VALUES, U_coarse_local));
+
+  // destroy the coarse vectors
+  PetscCall(RDyDestroyVectors(&rdy));
+
+  // destroy the coarse DMs
+  PetscCall(DMDestroy(&rdy->dm));
+  PetscCall(DMDestroy(&rdy->aux_dm));
+
+  // set the DM to be the refined DM
+  rdy->dm      = dm_fine;
+  rdy->flow_dm = rdy->dm;
+
+  // create the auxiliary DM
+  PetscCall(CreateAuxiliaryDM(rdy));
+
+  // create new refined vectors
+  PetscCall(CreateVectors(rdy));
+
+  // destroy and recreate regions
+  for (PetscInt i = 0; i < rdy->num_regions; ++i) {
+    PetscCall(DestroyRegion(&rdy->regions[i]));
+  }
+  if (rdy->regions) PetscFree(rdy->regions);
+
+  // set region as the new refined regions
+  rdy->num_regions = num_regions;
+  rdy->regions     = refined_regions;
+
+  // mark the mesh was refined
+  rdy->mesh_was_refined = PETSC_TRUE;
+  rdy->num_refinements++;
+
+  // destroy and recreate mesh
+  PetscCall(RDyMeshDestroy(rdy->mesh));
+  PetscCall(RDyMeshCreateFromDM(rdy->dm, rdy->num_refinements, &rdy->mesh));
+
+  // initialize the refined solution from existing previous solution
+  PetscCall(MatMult(CoarseToFine, U_coarse_local, rdy->u_local));
+  PetscCall(DMLocalToGlobal(rdy->dm, rdy->u_local, INSERT_VALUES, rdy->u_global));
+  PetscCall(MatDestroy(&CoarseToFine));
+  PetscCall(MatDestroy(&FineToCoarse));
+  PetscCall(VecDestroy(&U_coarse_local));
+  PetscCall(VecDestroy(&U_fine_local));
+
+  // destroy the operator
+  PetscCall(DestroyOperator(&rdy->operator));
+
+  // destroy the boundaries and reallocate memory
+  PetscCall(RDyDestroyBoundaries(&rdy));
+  InitBoundaries(rdy);
+
+  // reinitialize the operator
+  PetscCall(InitOperator(rdy));
+
+  // reinitialize material properties
+  PetscCall(InitMaterialProperties(rdy));
+
+  // save time and timestep from TS
+  PetscReal time, dt;
+  PetscInt  nstep;
+  PetscCall(TSGetTime(rdy->ts, &time));
+  PetscCall(TSGetTimeStep(rdy->ts, &dt));
+  PetscCall(TSGetStepNumber(rdy->ts, &nstep));
+
+  // destroy the TS and re-create it
+  PetscCall(TSDestroy(&rdy->ts));
+  PetscCall(InitSolver(rdy));
+
+  // set the time and timstep in TS
+  PetscCall(TSSetStepNumber(rdy->ts, nstep));
+  PetscCall(TSSetTime(rdy->ts, time));
+  PetscCall(TSSetTimeStep(rdy->ts, dt));
+
+  // make sure any Dirichlet boundary conditions are properly specified
+  PetscCall(InitDirichletBoundaryConditions(rdy));
+
+  // initialize the source terms
+  PetscCall(InitSourceConditions(rdy));
+
+  PetscFunctionReturn(PETSC_SUCCESS);
+}
diff --git a/src/rdysetup.c b/src/rdysetup.c
index 401bc9d9a..899dbd6bc 100644
--- a/src/rdysetup.c
+++ b/src/rdysetup.c
@@ -518,7 +518,7 @@ PetscErrorCode InitBoundaries(RDy rdy) {
 
 // sets up materials
 //   unsafe for refinement if file is given for surface composition
-static PetscErrorCode InitMaterialProperties(RDy rdy) {
+PetscErrorCode InitMaterialProperties(RDy rdy) {
   PetscFunctionBegin;
 
   // check that each region has a material assigned to it
@@ -1096,7 +1096,7 @@ PetscErrorCode InitSolver(RDy rdy) {
 }
 
 // initialize the data on the right hand side of the boundary edges
-static PetscErrorCode InitDirichletBoundaryConditions(RDy rdy) {
+PetscErrorCode InitDirichletBoundaryConditions(RDy rdy) {
   PetscFunctionBegin;
 
   for (PetscInt b = 0; b < rdy->num_boundaries; ++b) {
@@ -1134,7 +1134,7 @@ static PetscErrorCode InitDirichletBoundaryConditions(RDy rdy) {
 /// @brief Initializes a homogeneous water source term.
 /// @param rdy A RDy struct
 /// @return 0 on success, or a non-zero error code on failure
-static PetscErrorCode InitSourceConditions(RDy rdy) {
+PetscErrorCode InitSourceConditions(RDy rdy) {
   PetscFunctionBegin;
 
   if (rdy->config.num_sources > 0) {
@@ -1234,6 +1234,9 @@ PetscErrorCode RDySetup(RDy rdy) {
     rdy->log = stdout;
   }
 
+  rdy->mesh_was_refined = PETSC_FALSE;
+  rdy->num_refinements  = 0;
+
   // override parameters using command line arguments
   PetscCall(OverrideParameters(rdy));
 
@@ -1267,8 +1270,6 @@ PetscErrorCode RDySetup(RDy rdy) {
   };
   PetscCall(CreateAuxiliaryDM(rdy));
 
-  PetscCall(CreateFlowDM(rdy));
-
   if (rdy->config.physics.sediment.num_classes) {
     PetscCall(CreateFlowDM(rdy));
     PetscCall(CreateSedimentDM(rdy));
@@ -1281,7 +1282,7 @@ PetscErrorCode RDySetup(RDy rdy) {
   PetscCall(CreateVectors(rdy));
 
   RDyLogDebug(rdy, "Creating FV mesh...");
-  PetscCall(RDyMeshCreateFromDM(rdy->dm, &rdy->mesh));
+  PetscCall(RDyMeshCreateFromDM(rdy->dm, 0, &rdy->mesh));
 
   RDyLogDebug(rdy, "Initializing regions...");
   PetscCall(InitRegions(rdy));
diff --git a/src/tests/test_rdymesh.c b/src/tests/test_rdymesh.c
index 6eab085b2..f8c7b3874 100644
--- a/src/tests/test_rdymesh.c
+++ b/src/tests/test_rdymesh.c
@@ -92,8 +92,10 @@ static void TestRDyMeshCreateFromDM(void **state) {
   assert_int_equal(0, DMSetRefineLevel(dm, 1));
 
   // Now create a local mesh representation.
-  RDyMesh mesh;
-  assert_int_equal(0, RDyMeshCreateFromDM(dm, &mesh));
+  RDyMesh  mesh;
+  PetscInt refine_level;
+  assert_int_equal(0, DMGetRefineLevel(dm, &refine_level));
+  assert_int_equal(0, RDyMeshCreateFromDM(dm, refine_level, &mesh));
   // I expected the following statement to be true, but it's not (for nproc > 1)
   //  assert_int_equal(Nx * Ny, mesh.num_cells);
   assert_true(mesh.num_owned_cells <= Nx * Ny);  // (== iff nproc == 1)
diff --git a/src/xdmf_output.c b/src/xdmf_output.c
index 0f2513eab..edf3813f2 100644
--- a/src/xdmf_output.c
+++ b/src/xdmf_output.c
@@ -3,22 +3,22 @@
 #include <rdycore.h>
 #include <string.h>
 
-static PetscErrorCode WriteFieldData(DM dm, Vec global_vec, PetscViewer viewer) {
+static PetscErrorCode WriteFieldData(DM dm, Vec global_vec, PetscViewer viewer, PetscInt num_refinements) {
   PetscFunctionBegin;
 
   MPI_Comm comm;
   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
 
   // create and populate a multi-component vector
-  Vec       data_vec;
-  PetscBool use_natural;
-  PetscCall(DMGetUseNatural(dm, &use_natural));
-  if (use_natural) {
+  Vec data_vec;
+
+  if (!num_refinements) {
     PetscCall(DMPlexCreateNaturalVector(dm, &data_vec));
     PetscCall(DMPlexGlobalToNaturalBegin(dm, global_vec, data_vec));
     PetscCall(DMPlexGlobalToNaturalEnd(dm, global_vec, data_vec));
   } else {
-    data_vec = global_vec;
+    PetscCall(VecDuplicate(global_vec, &data_vec));
+    PetscCall(VecCopy(global_vec, data_vec));
     PetscCall(PetscObjectReference((PetscObject)data_vec));
   }
 
@@ -64,7 +64,7 @@ static PetscErrorCode WriteFieldData(DM dm, Vec global_vec, PetscViewer viewer)
     PetscCall(PetscFree(comp));
   }
 
-  if (use_natural) PetscCall(VecDestroy(&data_vec));  // FIXME: does this leave an extra ref in non-natural case?
+  PetscCall(VecDestroy(&data_vec));
 
   PetscFunctionReturn(PETSC_SUCCESS);
 }
@@ -129,8 +129,8 @@ static PetscErrorCode WriteXDMFHDF5Data(RDy rdy, PetscInt step, PetscReal time)
   char group_name[PETSC_MAX_PATH_LEN];
   snprintf(group_name, PETSC_MAX_PATH_LEN, "%" PetscInt_FMT " %E %s", step, time, units);
   PetscCall(PetscViewerHDF5PushGroup(viewer, group_name));
-  PetscCall(WriteFieldData(rdy->dm, rdy->u_global, viewer));
-  PetscCall(WriteFieldData(rdy->aux_dm, rdy->diags_vec, viewer));
+  PetscCall(WriteFieldData(rdy->dm, rdy->u_global, viewer, rdy->num_refinements));
+  PetscCall(WriteFieldData(rdy->aux_dm, rdy->diags_vec, viewer, rdy->num_refinements));
   PetscCall(PetscViewerHDF5PopGroup(viewer));
 
   // write the grid


Reviews:
user: login: jeff-cohere
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-11T17:10:48Z
user: login: jeff-cohere
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-11T17:15:52Z
user: login: bishtgautam
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-11T17:23:56Z
user: login: jeff-cohere
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-11T17:26:13Z
user: login: jeff-cohere
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-11T17:27:05Z
user: login: jeff-cohere
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-11T17:32:07Z
user: login: jeff-cohere
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-11T17:40:15Z
user: login: jeff-cohere
body: Lots of minor feedback, but overall this looks good. It's helpful to see the types of issues that AMR is introducing.
state: APPROVED
author_association: COLLABORATOR
submitted_at: 2025-04-11T17:41:18Z
user: login: jeff-cohere
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-11T17:42:16Z
user: login: bishtgautam
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-11T20:25:44Z

Review comments:
diff_hunk: @@ -18,9 +18,6 @@ PetscErrorCode RDyInit(int argc, char *argv[], const char *help) {
       PetscCall(PetscInitialize(&argc, &argv, (char *)0, (char *)help));
     }
 
-    // some exit statuses don't get propagated without this vvv
-    PetscCall(PetscPushErrorHandler(PetscMPIAbortErrorHandler, NULL));
path: src/rdycore.c
user: login: jeff-cohere
body: Do we not need this anymore? Does this error handler cause problems with AMR?
created_at: 2025-04-11T17:10:48Z
author_association: COLLABORATOR
diff_hunk: @@ -20,6 +20,7 @@ static PetscInt QUAD_ID_EXODUS = 5;
 /// @param [in] nvertices_per_cell Maximum number of vertices per cell
 /// @param [in] nedges_per_cell Maximum number of edges per cell
 /// @param [out] cells A pointer to an RDyCells that stores allocated data.
+/// @param [out] num_cells A pointer to the number of cells
path: src/rdymesh.c
user: login: jeff-cohere
body: Should this be associated with the next function down (`RDyCellsCreateFromDM`)?
created_at: 2025-04-11T17:15:51Z
author_association: COLLABORATOR
diff_hunk: @@ -18,9 +18,6 @@ PetscErrorCode RDyInit(int argc, char *argv[], const char *help) {
       PetscCall(PetscInitialize(&argc, &argv, (char *)0, (char *)help));
     }
 
-    // some exit statuses don't get propagated without this vvv
-    PetscCall(PetscPushErrorHandler(PetscMPIAbortErrorHandler, NULL));
path: src/rdycore.c
user: login: bishtgautam
body: We didn't have the corresponding `PetscPopErrorHandler` in the code. Thus, when running with `-malloc_debug`, we got errors. Removing this fixed the error reporting. I don't know if adding `PetscPopErrorHandler` is a better solution.
created_at: 2025-04-11T17:23:56Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,20 @@
+find_program(H5DIFF h5diff REQUIRED
+             PATHS ${PETSC_DIR}/${PETSC_ARCH}/bin)
+set(amr_c_driver ${amr_driver})
+
+# arguments for specific configurations ("basic" config has no options)
+set(ceed_args -ceed /cpu/self)
+set(preload_args -preload)
+
+# mms driver (C and Fortran)
path: driver/tests/amr/CMakeLists.txt
user: login: jeff-cohere
body: mms -> amr
created_at: 2025-04-11T17:26:12Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,50 @@
+physics:
+  flow:
+    mode: swe
+
+numerics:
+  spatial: fv
+  temporal: euler
+  riemann: roe
+
+logging:
+  level: debug
+
+time:
+  final_time: 0.05
+  unit: seconds
+  time_step : 0.005
+  coupling_interval: 0.01
+
+#output:
+#  format: xdmf
+#  step_interval: 100
+
+grid:
+  file: mms_triangles_dx1.exo
path: driver/tests/amr/amr_dx1.yaml
user: login: jeff-cohere
body: worth making an AMR-specific copy of this mesh to keep things separate?
created_at: 2025-04-11T17:27:04Z
author_association: COLLABORATOR
diff_hunk: @@ -42,23 +42,23 @@ static PetscErrorCode CreateDMSection(DM dm, SectionFieldSpec fields) {
   PetscFunctionReturn(PETSC_SUCCESS);
 }
 
-PetscErrorCode CreateCellCenteredDMFromDM(DM dm, const SectionFieldSpec fields, DM *cc_dm) {
+PetscErrorCode CreateCellCenteredDMFromDM(DM dm, PetscInt refinment_level, const SectionFieldSpec fields, DM *cc_dm) {
path: src/rdydm.c
user: login: jeff-cohere
body: refinment_level -> refinement_level
created_at: 2025-04-11T17:32:07Z
author_association: COLLABORATOR
diff_hunk: @@ -64,7 +64,7 @@ static PetscErrorCode WriteFieldData(DM dm, Vec global_vec, PetscViewer viewer)
     PetscCall(PetscFree(comp));
   }
 
-  if (use_natural) PetscCall(VecDestroy(&data_vec));  // FIXME: does this leave an extra ref in non-natural case?
+  PetscCall(VecDestroy(&data_vec));  // FIXME: does this leave an extra ref in non-natural case?
path: src/xdmf_output.c
user: login: jeff-cohere
body: If I understand this question correctly, the "extra ref" you're referring to is to `global_vec` in the case of a refined mesh. The lifetime of `global_vec` exceeds this function, so it should still have a live reference anyway. Does that sound right to you?
created_at: 2025-04-11T17:40:15Z
author_association: COLLABORATOR
diff_hunk: @@ -18,9 +18,6 @@ PetscErrorCode RDyInit(int argc, char *argv[], const char *help) {
       PetscCall(PetscInitialize(&argc, &argv, (char *)0, (char *)help));
     }
 
-    // some exit statuses don't get propagated without this vvv
-    PetscCall(PetscPushErrorHandler(PetscMPIAbortErrorHandler, NULL));
path: src/rdycore.c
user: login: jeff-cohere
body: Let's see how things work without it and revisit if needed.
created_at: 2025-04-11T17:42:16Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,50 @@
+physics:
+  flow:
+    mode: swe
+
+numerics:
+  spatial: fv
+  temporal: euler
+  riemann: roe
+
+logging:
+  level: debug
+
+time:
+  final_time: 0.05
+  unit: seconds
+  time_step : 0.005
+  coupling_interval: 0.01
+
+#output:
+#  format: xdmf
+#  step_interval: 100
+
+grid:
+  file: mms_triangles_dx1.exo
path: driver/tests/amr/amr_dx1.yaml
user: login: bishtgautam
body: For now, let's reuse the MMS mesh. In the next major AMR PR, we would add a realistic mesh for AMR.
created_at: 2025-04-11T20:25:44Z
author_association: CONTRIBUTOR

PR comments:
user: login: github-actions[bot]
created_at: 2025-01-24T04:46:16Z
body: [PR Preview Action](https://github.com/rossjrw/pr-preview-action) v1.6.1
:---:
| <p></p> :rocket: View preview at <br> https://RDycore.github.io/RDycore/pr-preview/pr-269/ <br><br>
| <h6>Built to branch [`gh-pages`](https://github.com/RDycore/RDycore/tree/gh-pages) at 2025-04-11 16:40 UTC. <br> Preview will be ready when the [GitHub Pages deployment](https://github.com/RDycore/RDycore/deployments) is complete. <br><br> </h6>
<!-- Sticky Pull Request Commentpr-preview -->
author_association: NONE
user: login: bishtgautam
created_at: 2025-01-24T04:58:40Z
body: @knepley, @markadams, Here is the AMR example

- In AMR driver (`driver/amr.c`)
   - First, things are set up for the coarse mesh [here L35-36](https://github.com/RDycore/RDycore/pull/269/files#diff-43a6648ee77fa9fa7631e124ea7032a8cc6681995822918cccb76ad0225dcdfdR35-R36)
   - Second, RDycore takes a step [L38](https://github.com/RDycore/RDycore/pull/269/files#diff-43a6648ee77fa9fa7631e124ea7032a8cc6681995822918cccb76ad0225dcdfdR38)
   - Third, RDycore is refined [L40](https://github.com/RDycore/RDycore/pull/269/files#diff-43a6648ee77fa9fa7631e124ea7032a8cc6681995822918cccb76ad0225dcdfdR40) via `RDyRefine`
   - Fourth, RDycore runs to the end of the simulation using the refined mesh [L42-44](https://github.com/RDycore/RDycore/pull/269/files#diff-43a6648ee77fa9fa7631e124ea7032a8cc6681995822918cccb76ad0225dcdfdR42-R44)

- The `RDyRefine()` is in `src/rdyrefine.c` [L90](https://github.com/RDycore/RDycore/pull/269/files#diff-28a99b822928ec6d38cc346df8cff2a64df522d921a3615158d0614d1653627eR90)
   - All cells are marked for refinement [L97-102](https://github.com/RDycore/RDycore/pull/269/files#diff-28a99b822928ec6d38cc346df8cff2a64df522d921a3615158d0614d1653627eR97-R102)
   - Next, `DMRefine()` is called.
   - Then, I have to do a lot of destruction and recreation of RDycore's data structures

- If `DMCreateInterpolation` is uncommented at [L112](https://github.com/RDycore/RDycore/pull/269/files#diff-28a99b822928ec6d38cc346df8cff2a64df522d921a3615158d0614d1653627eR112), the model crashes

- You can run the AMR example as follows

```
cd build/driver/tests/amr

$PETSC_DIR/$PETSC_ARCH/bin/mpiexec -n 1 \
../../rdycore_amr amr_dx1.yaml -dm_fine_view -dm_view  \
-dm_plex_transform_type refine_sbr -dm_plex_transform_active adapt
```
author_association: CONTRIBUTOR
user: login: codecov[bot]
created_at: 2025-01-24T05:00:16Z
body: ## [Codecov](https://app.codecov.io/gh/RDycore/RDycore/pull/269?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=RDycore) Report
Attention: Patch coverage is `97.54300%` with `10 lines` in your changes missing coverage. Please review.
> Project coverage is 94.28%. Comparing base [(`f4a6cc4`)](https://app.codecov.io/gh/RDycore/RDycore/commit/f4a6cc4be672a7bbdccfdd6462f574ec8d421335?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=RDycore) to head [(`65eff84`)](https://app.codecov.io/gh/RDycore/RDycore/commit/65eff845ca3f720769fb485fb00d649b67b58358?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=RDycore).
> Report is 49 commits behind head on main.

| [Files with missing lines](https://app.codecov.io/gh/RDycore/RDycore/pull/269?dropdown=coverage&src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=RDycore) | Patch % | Lines |
|---|---|---|
| [driver/amr.c](https://app.codecov.io/gh/RDycore/RDycore/pull/269?src=pr&el=tree&filepath=driver%2Famr.c&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=RDycore#diff-ZHJpdmVyL2Ftci5j) | 75.00% | [6 Missing :warning: ](https://app.codecov.io/gh/RDycore/RDycore/pull/269?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=RDycore) |
| [src/rdyrefine.c](https://app.codecov.io/gh/RDycore/RDycore/pull/269?src=pr&el=tree&filepath=src%2Frdyrefine.c&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=RDycore#diff-c3JjL3JkeXJlZmluZS5j) | 99.28% | [2 Missing :warning: ](https://app.codecov.io/gh/RDycore/RDycore/pull/269?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=RDycore) |
| [src/xdmf\_output.c](https://app.codecov.io/gh/RDycore/RDycore/pull/269?src=pr&el=tree&filepath=src%2Fxdmf_output.c&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=RDycore#diff-c3JjL3hkbWZfb3V0cHV0LmM=) | 75.00% | [2 Missing :warning: ](https://app.codecov.io/gh/RDycore/RDycore/pull/269?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=RDycore) |

<details><summary>Additional details and impacted files</summary>


```diff
@@            Coverage Diff             @@
##             main     #269      +/-   ##
==========================================
+ Coverage   94.18%   94.28%   +0.09%     
==========================================
  Files          35       37       +2     
  Lines        7774     8101     +327     
==========================================
+ Hits         7322     7638     +316     
- Misses        452      463      +11     
```

</details>

[:umbrella: View full report in Codecov by Sentry](https://app.codecov.io/gh/RDycore/RDycore/pull/269?dropdown=coverage&src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=RDycore).   
:loudspeaker: Have feedback on the report? [Share it here](https://about.codecov.io/codecov-pr-comment-feedback/?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=RDycore).

<details><summary> :rocket: New features to boost your workflow: </summary>

- :snowflake: [Test Analytics](https://docs.codecov.com/docs/test-analytics): Detect flaky tests, report on failures, and find test suite problems.
</details>
author_association: NONE
user: login: bishtgautam
created_at: 2025-01-27T22:25:59Z
body: @knepley, The issue regarding the names of components getting lost can be reproduced via the following code:

```
$PETSC_DIR/$PETSC_ARCH/bin/mpiexec -n 1 ../../rdycore_amr amr_dx1.yaml -dm_fine_view -dm_view  -dm_plex_transform_type refine_sbr -dm_plex_transform_active adapt -dm_view

DETAIL: ==========================================================
DETAIL: RDycore (input read from amr_dx1.yaml)
DETAIL: ==========================================================
DETAIL: Physics:
DETAIL:   Flow:
DETAIL:   Sediment model: disabled
DETAIL:   Salinity model: disabled
DETAIL: Numerics:
DETAIL:   Spatial discretization: finite volume (FV)
DETAIL:   Temporal discretization: forward euler
DETAIL:   Riemann solver: roe
DETAIL: Time:
DETAIL:   Final time: 0.05 minutes
DETAIL: Logging:
DETAIL:   Primary log file: <stdout>
DETAIL: Checkpoint:
DETAIL:   (disabled)
DETAIL: Restart:
DETAIL:   (disabled)
DEBUG: Creating DMs...
DM Object: grid 1 MPI process
  type: plex
grid in 2 dimensions:
  Number of 0-cells per rank: 61
  Number of 1-cells per rank: 160
  Number of 2-cells per rank: 100
Labels:
  celltype: 3 strata with value/size (0 (61), 1 (160), 3 (100))
  depth: 3 strata with value/size (0 (61), 1 (160), 2 (100))
  Cell Sets: 1 strata with value/size (1 (100))
  Face Sets: 1 strata with value/size (1 (20))
  boundary_edges: 1 strata with value/size (1 (20))
Field Solution:
  adjacency FVM++
DEBUG: Creating FV mesh...
DEBUG: Initializing regions...
DEBUG:   Found region 1 (100 cells)
DEBUG: Initializing initial conditions and sources...
DEBUG: Initializing boundaries and boundary conditions...
DEBUG:   Found boundary 1 (20 edges)
DEBUG: Initializing solution data...
DEBUG: Initializing operator...
DEBUG: Initializing material properties...
DEBUG: Initializing solver...
DEBUG: Initializing checkpoints...
DEBUG: Creating output directory output...
DEBUG: Running simulation...
DETAIL: Advancing from t = 0. to 0.01...
DEBUG: [0] Time = 0.000000 [sec] Max courant number 0.19805
DEBUG: [1] Time = 0.005000 [sec] Max courant number 0.198052
DM Object: 1 MPI process
  type: plex
DM_0x117e051c0_1 in 2 dimensions:
  Number of 0-cells per rank: 221
  Number of 1-cells per rank: 620
  Number of 2-cells per rank: 400
Labels:
  celltype: 3 strata with value/size (0 (221), 1 (620), 3 (400))
  depth: 3 strata with value/size (0 (221), 1 (620), 2 (400))
  Cell Sets: 1 strata with value/size (1 (700))
  Face Sets: 1 strata with value/size (1 (60))
  boundary_edges: 1 strata with value/size (1 (60))
Field Solution:
  adjacency FVM++
++++++++++++++++++++++++++++++++++++
About coarse DM:
nfields = 1
  field = 0; num_component = 3
    field = 0; component = 0; comp_name = Height
    field = 0; component = 1; comp_name = MomentumX
    field = 0; component = 2; comp_name = MomentumY
++++++++++++++++++++++++++++++++++++
About refined DM:
nfields = 1
  field = 0; num_component = 3
    field = 0; component = 0; comp_name = 0
    field = 0; component = 1; comp_name = 1
    field = 0; component = 2; comp_name = 2
++++++++++++++++++++++++++++++++++++
DETAIL: Advancing from t = 0.01 to 0.02...
DEBUG: [2] Time = 0.010000 [sec] Max courant number 0.396106
DEBUG: [3] Time = 0.015000 [sec] Max courant number 0.396109
DETAIL: Advancing from t = 0.02 to 0.03...
DEBUG: [4] Time = 0.020000 [sec] Max courant number 0.397
DEBUG: [5] Time = 0.025000 [sec] Max courant number 0.399448
DETAIL: Advancing from t = 0.03 to 0.04...
DEBUG: [6] Time = 0.030000 [sec] Max courant number 0.40308
DEBUG: [7] Time = 0.035000 [sec] Max courant number 0.407406
DETAIL: Advancing from t = 0.04 to 0.05...
DEBUG: [8] Time = 0.040000 [sec] Max courant number 0.411822
DEBUG: [9] Time = 0.045000 [sec] Max courant number 0.415884
```

I used a very PETSc hash:

```
commit 1acb5306efbd6b897a796cf04de7b7f73041d5b3 (HEAD -> main, origin/main, origin/HEAD)
Merge: 5fcaa7196c3 2b78b3bbb62
Author: Satish Balay <balay@mcs.anl.gov>
Date:   Thu Jan 23 19:15:33 2025 +0000

    Merge branch 'knepley/feature-optionsused-python' into 'main'

    Options+Python: Binding for PetscOptionsUsed()

    See merge request petsc/petsc!8090
```
author_association: CONTRIBUTOR
user: login: bishtgautam
created_at: 2025-04-02T19:45:21Z
body: ![amr](https://github.com/user-attachments/assets/625b2da1-42c7-4e7f-8f3e-92ff4ef19099)

Things seem to be working as expected.
author_association: CONTRIBUTOR
user: login: bishtgautam
created_at: 2025-04-04T03:46:08Z
body: This PR needs:
1. https://github.com/RDycore/RDycore/issues/299 needs to be fixed
2. Docker image to be updated.
author_association: CONTRIBUTOR