Repository: AMReX-Codes/amrex — pull request #3332 (cpp)
Title: Implement Checkpoint, Restart, and Plotfile for pure SOA

Follow-up to #2878.

The proposed changes:
- [x] fix a bug or incorrect behavior in AMReX
- [x] add new capabilities to AMReX
- [ ] changes answers in the test suite to more than roundoff level
- [ ] are likely to significantly affect the results of downstream AMReX users
- [ ] include documentation in the code and/or rst files, if appropriate


Resolved issues:
title: Support SoA-Only Particles
body: ## Summary

Further updates of  #2853 where @atmyers and @ax3l can also push to.

## Upgrade Guide for C++ API Changes

These PRs show changes to keep using the *old layout*:
- example in WarpX: https://github.com/ECP-WarpX/WarpX/pull/3848
- example in HiPACE++: https://github.com/Hi-PACE/hipace/pull/921
- example in ImpactX: https://github.com/ECP-WarpX/impactx/pull/342
- example in pyAMReX: https://github.com/AMReX-Codes/pyamrex/pull/119

These PRs show changes to *use the new layout* (pure SoA):
- example in WarpX: https://github.com/ECP-WarpX/WarpX/pull/3850
- example in ImpactX: https://github.com/ECP-WarpX/impactx/pull/348
- example in pyAMReX: https://github.com/AMReX-Codes/pyamrex/pull/124

## Additional background

## Checklist

The proposed changes:
- [ ] fix a bug or incorrect behavior in AMReX
- [x] add new capabilities to AMReX
- [ ] changes answers in the test suite to more than roundoff level
- [ ] are likely to significantly affect the results of downstream AMReX users
- [ ] include documentation in the code and/or rst files, if appropriate


Hints:

Fix patch:
diff --git a/Src/Particle/AMReX_ParticleContainer.H b/Src/Particle/AMReX_ParticleContainer.H
index abffbefc860..3e6a6113fc8 100644
--- a/Src/Particle/AMReX_ParticleContainer.H
+++ b/Src/Particle/AMReX_ParticleContainer.H
@@ -1346,7 +1346,8 @@ protected:
     * \param lev_max
     * \param local_grid
     */
-    bool EnforcePeriodicWhere (ParticleType& prt, ParticleLocData& pld,
+    template <typename P>
+    bool EnforcePeriodicWhere (P& prt, ParticleLocData& pld,
                                int lev_min = 0, int lev_max = -1, int local_grid=-1) const;
 
 public:
@@ -1378,7 +1379,8 @@ private:
     void RedistributeMPI (std::map<int, Vector<char> >& not_ours,
                           int lev_min = 0, int lev_max = 0, int nGrow = 0, int local=0);
 
-    void locateParticle (ParticleType& p, ParticleLocData& pld,
+    template <typename P>
+    void locateParticle (P& p, ParticleLocData& pld,
                          int lev_min, int lev_max, int nGrow, int local_grid=-1) const;
 
     void Initialize ();
diff --git a/Src/Particle/AMReX_ParticleContainerI.H b/Src/Particle/AMReX_ParticleContainerI.H
index 9a5007f86ec..034bc50cd6f 100644
--- a/Src/Particle/AMReX_ParticleContainerI.H
+++ b/Src/Particle/AMReX_ParticleContainerI.H
@@ -164,9 +164,10 @@ ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt, Allocator>
 
 template <typename ParticleType, int NArrayReal, int NArrayInt,
           template<class> class Allocator>
+template <typename P>
 bool
 ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt, Allocator>
-::EnforcePeriodicWhere (ParticleType&    p,
+::EnforcePeriodicWhere (P&               p,
                         ParticleLocData& pld,
                         int              lev_min,
                         int              lev_max,
@@ -312,8 +313,9 @@ ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt, Allocator>::resizeDa
 
 template <typename ParticleType, int NArrayReal, int NArrayInt,
           template<class> class Allocator>
+template <typename P>
 void
-ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt, Allocator>::locateParticle (ParticleType& p, ParticleLocData& pld,
+ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt, Allocator>::locateParticle (P& p, ParticleLocData& pld,
                                                                                         int lev_min, int lev_max, int nGrow, int local_grid) const
 {
     bool success;
diff --git a/Src/Particle/AMReX_ParticleIO.H b/Src/Particle/AMReX_ParticleIO.H
index a9849ca8a3c..cfc55dfd550 100644
--- a/Src/Particle/AMReX_ParticleIO.H
+++ b/Src/Particle/AMReX_ParticleIO.H
@@ -919,7 +919,7 @@ ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt, Allocator>
     int*   iptr = istuff.dataPtr();
     RTYPE* rptr = rstuff.dataPtr();
 
-    ParticleType p;
+    Particle<NStructReal, NStructInt> ptemp;
     ParticleLocData pld;
 
     Vector<std::map<std::pair<int, int>, Gpu::HostVector<ParticleType> > > host_particles;
@@ -937,41 +937,43 @@ ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt, Allocator>
     host_int_attribs.resize(finest_level_in_file+1);
 
     for (int i = 0; i < cnt; i++) {
-        if (convert_ids) {
+        // note: for pure SoA particle layouts, we do write the id, cpu and positions as a struct
+        //       for backwards compatibility with readers
+        if (!ParticleType::is_soa_particle && convert_ids) {
             std::int32_t  xi, yi;
             std::uint32_t xu, yu;
             xi = iptr[0];
             yi = iptr[1];
             std::memcpy(&xu, &xi, sizeof(xi));
             std::memcpy(&yu, &yi, sizeof(yi));
-            p.m_idcpu = ((std::uint64_t)xu) << 32 | yu;
+            ptemp.m_idcpu = ((std::uint64_t)xu) << 32 | yu;
         } else {
-            p.id()   = iptr[0];
-            p.cpu()  = iptr[1];
+            ptemp.id()   = iptr[0];
+            ptemp.cpu()  = iptr[1];
         }
         iptr += 2;
 
         for (int j = 0; j < NStructInt; j++)
         {
-            p.idata(j) = *iptr;
+            ptemp.idata(j) = *iptr;
             ++iptr;
         }
 
-        AMREX_ASSERT(p.id() > 0);
+        AMREX_ASSERT(ptemp.id() > 0);
 
-        AMREX_D_TERM(p.pos(0) = ParticleReal(rptr[0]);,
-                     p.pos(1) = ParticleReal(rptr[1]);,
-                     p.pos(2) = ParticleReal(rptr[2]););
+        AMREX_D_TERM(ptemp.pos(0) = ParticleReal(rptr[0]);,
+                     ptemp.pos(1) = ParticleReal(rptr[1]);,
+                     ptemp.pos(2) = ParticleReal(rptr[2]););
 
         rptr += AMREX_SPACEDIM;
 
         for (int j = 0; j < NStructReal; j++)
         {
-            p.rdata(j) = ParticleReal(*rptr);
+            ptemp.rdata(j) = ParticleReal(*rptr);
             ++rptr;
         }
 
-        locateParticle(p, pld, 0, finestLevel(), 0);
+        locateParticle(ptemp, pld, 0, finestLevel(), 0);
 
         std::pair<int, int> ind(grd, pld.m_tile);
 
@@ -979,35 +981,69 @@ ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt, Allocator>
         host_int_attribs[lev][ind].resize(NumIntComps());
 
         // add the struct
-        host_particles[lev][ind].push_back(p);
+        if constexpr(!ParticleType::is_soa_particle)
+        {
+            host_particles[lev][ind].push_back(ptemp);
 
-        // add the real...
-        for (int icomp = 0; icomp < NumRealComps(); icomp++) {
-            host_real_attribs[lev][ind][icomp].push_back(*rptr);
-            ++rptr;
-        }
+            // add the real...
+            for (int icomp = 0; icomp < NumRealComps(); icomp++) {
+                host_real_attribs[lev][ind][icomp].push_back(*rptr);
+                ++rptr;
+            }
 
-        // ... and int array data
-        for (int icomp = 0; icomp < NumIntComps(); icomp++) {
-            host_int_attribs[lev][ind][icomp].push_back(*iptr);
-            ++iptr;
+            // ... and int array data
+            for (int icomp = 0; icomp < NumIntComps(); icomp++) {
+                host_int_attribs[lev][ind][icomp].push_back(*iptr);
+                ++iptr;
+            }
+        } else {
+            host_particles[lev][ind];
+
+            for (int j = 0; j < AMREX_SPACEDIM; j++) {
+                host_real_attribs[pld.m_lev][ind][j].push_back(ptemp.pos(j));
+            }
+
+            host_int_attribs[pld.m_lev][ind][0].push_back(ptemp.id());
+            host_int_attribs[pld.m_lev][ind][1].push_back(ptemp.cpu());
+
+            // read all other SoA
+            // add the real...
+            for (int icomp = AMREX_SPACEDIM; icomp < NumRealComps(); icomp++) {
+                host_real_attribs[lev][ind][icomp].push_back(*rptr);
+                ++rptr;
+            }
+
+            // ... and int array data
+            for (int icomp = 2; icomp < NumIntComps(); icomp++) {
+                host_int_attribs[lev][ind][icomp].push_back(*iptr);
+                ++iptr;
+            }
         }
     }
 
     for (int host_lev = 0; host_lev < static_cast<int>(host_particles.size()); ++host_lev)
-      {
+    {
         for (auto& kv : host_particles[host_lev]) {
           auto grid = kv.first.first;
           auto tile = kv.first.second;
           const auto& src_tile = kv.second;
 
           auto& dst_tile = DefineAndReturnParticleTile(host_lev, grid, tile);
-          auto old_size = dst_tile.GetArrayOfStructs().size();
-          auto new_size = old_size + src_tile.size();
+          auto old_size = dst_tile.size();
+          auto new_size = old_size;
+          if constexpr(!ParticleType::is_soa_particle)
+          {
+              new_size += src_tile.size();
+          } else {
+              new_size += host_real_attribs[host_lev][std::make_pair(grid,tile)][0].size();
+          }
           dst_tile.resize(new_size);
 
-          Gpu::copyAsync(Gpu::hostToDevice, src_tile.begin(), src_tile.end(),
-                         dst_tile.GetArrayOfStructs().begin() + old_size);
+          if constexpr(!ParticleType::is_soa_particle)
+          {
+              Gpu::copyAsync(Gpu::hostToDevice, src_tile.begin(), src_tile.end(),
+                             dst_tile.GetArrayOfStructs().begin() + old_size);
+          }
 
           for (int i = 0; i < NumRealComps(); ++i) {
               Gpu::copyAsync(Gpu::hostToDevice,
diff --git a/Src/Particle/AMReX_ParticleTile.H b/Src/Particle/AMReX_ParticleTile.H
index 9f1fe712c64..3b5ee15fdb0 100644
--- a/Src/Particle/AMReX_ParticleTile.H
+++ b/Src/Particle/AMReX_ParticleTile.H
@@ -226,6 +226,9 @@ struct ParticleTileData
     {
         AMREX_ASSERT(index < m_size);
         SuperParticleType sp;
+        for (int i = 0; i < AMREX_SPACEDIM; ++i) {sp.pos(i) = m_rdata[i][index];}
+        sp.id()  = m_idata[0][index];
+        sp.cpu() = m_idata[1][index];
         for (int i = 0; i < NAR; ++i)
             sp.rdata(i) = m_rdata[i][index];
         for (int i = 0; i < NAI; ++i)
@@ -595,6 +598,9 @@ struct ConstParticleTileData
     {
         AMREX_ASSERT(index < m_size);
         SuperParticleType sp;
+        for (int i = 0; i < AMREX_SPACEDIM; ++i) {sp.pos(i) = m_rdata[i][index];}
+        sp.id()  = m_idata[0][index];
+        sp.cpu() = m_idata[1][index];
         for (int i = 0; i < NAR; ++i)
             sp.rdata(i) = m_rdata[i][index];
         for (int i = 0; i < NAI; ++i)
diff --git a/Src/Particle/AMReX_WriteBinaryParticleData.H b/Src/Particle/AMReX_WriteBinaryParticleData.H
index 7767972998d..f182eb2b3e8 100644
--- a/Src/Particle/AMReX_WriteBinaryParticleData.H
+++ b/Src/Particle/AMReX_WriteBinaryParticleData.H
@@ -13,7 +13,7 @@ struct KeepValidFilter
     AMREX_GPU_HOST_DEVICE
     int operator() (const SrcData& src, int i) const noexcept
     {
-        return (src.m_aos[i].id() > 0);
+        return (src.id(i) > 0);
     }
 };
 
@@ -197,7 +197,7 @@ packIOData (Vector<int>& idata, Vector<ParticleReal>& rdata, const PC& pc, int l
     for (int i = 0; i < tiles.size(); i++) {
         const auto& ptile = pc.ParticlesAt(lev, grid, tiles[i]);
         const auto& pflags = particle_io_flags[lev].at(std::make_pair(grid, tiles[i]));
-        int np_tile = ptile.GetArrayOfStructs().numParticles();
+        int np_tile = ptile.numParticles();
         typename PC::IntVector offsets(np_tile);
         int num_copies = Scan::ExclusiveSum(np_tile, pflags.begin(), offsets.begin(), Scan::retSum);
 
@@ -283,44 +283,70 @@ packIOData (Vector<int>& idata, Vector<ParticleReal>& rdata, const PC& pc, int l
     const Long rChunkSize = AMREX_SPACEDIM + num_output_real;
     rdata.resize(np*rChunkSize);
 
-    int* iptr = idata.dataPtr();
+        int* iptr = idata.dataPtr();
     ParticleReal* rptr = rdata.dataPtr();
     for (unsigned i = 0; i < tiles.size(); i++) {
         const auto& ptile = pc.ParticlesAt(lev, grid, tiles[i]);
         const auto& pflags = particle_io_flags[lev].at(std::make_pair(grid, tiles[i]));
-        for (int pindex = 0; pindex < ptile.GetArrayOfStructs().numParticles(); ++pindex) {
-            const auto& aos = ptile.GetArrayOfStructs();
-            const auto& p = aos[pindex];
+        for (int pindex = 0; pindex < ptile.numParticles(); ++pindex) {
             if (pflags[pindex]) {
-                packParticleIDs(iptr, p, is_checkpoint);
-                iptr += 2;
-
-                for (int j = 0; j < AMREX_SPACEDIM; j++) rptr[j] = p.pos(j);
-                rptr += AMREX_SPACEDIM;
+                const auto& soa  = ptile.GetStructOfArrays();
 
-                for (int j = 0; j < PC::NStructInt; j++) {
-                    if (write_int_comp[j]) {
-                        *iptr = p.idata(j);
-                        ++iptr;
+                // note: for pure SoA particle layouts, we do write the id, cpu and positions as a struct
+                //       for backwards compatibility with readers
+                if constexpr(!PC::ParticleType::is_soa_particle)
+                {
+                    const auto& aos = ptile.GetArrayOfStructs();
+                    const auto& p = aos[pindex];
+
+                    // Int: id, cpu
+                    packParticleIDs(iptr, p, is_checkpoint);
+                    iptr += 2;
+
+                    // Real: positions
+                    for (int j = 0; j < AMREX_SPACEDIM; j++) rptr[j] = p.pos(j);
+                    rptr += AMREX_SPACEDIM;
+
+                    // extra AoS Int components
+                    for (int j = 0; j < PC::NStructInt; j++) {
+                        if (write_int_comp[j]) {
+                            *iptr = p.idata(j);
+                            ++iptr;
+                        }
+                    }
+                    // extra AoS Real components
+                    for (int j = 0; j < PC::NStructReal; j++) {
+                        if (write_real_comp[j]) {
+                            *rptr = p.rdata(j);
+                            ++rptr;
+                        }
                     }
                 }
+                else {
+                    amrex::ignore_unused(is_checkpoint);
+                    // Int: id, cpu
+                    *iptr = soa.GetIntData(0)[pindex];
+                    iptr += 1;
+                    *iptr = soa.GetIntData(1)[pindex];
+                    iptr += 1;
+
+                    // Real: position
+                    for (int j = 0; j < AMREX_SPACEDIM; j++) rptr[j] = soa.GetRealData(j)[pindex];
+                    rptr += AMREX_SPACEDIM;
+                }
 
-                const auto& soa  = ptile.GetStructOfArrays();
-                for (int j = 0; j < pc.NumIntComps(); j++) {
+                // extra SoA int data
+                const int int_start_offset = PC::ParticleType::is_soa_particle ? 2 : 0; // pure SoA: skip id, cpu
+                for (int j = int_start_offset; j < pc.NumIntComps(); j++) {
                     if (write_int_comp[PC::NStructInt+j]) {
                         *iptr = soa.GetIntData(j)[pindex];
                         ++iptr;
                     }
                 }
 
-                for (int j = 0; j < PC::NStructReal; j++) {
-                    if (write_real_comp[j]) {
-                        *rptr = p.rdata(j);
-                        ++rptr;
-                    }
-                }
-
-                for (int j = 0; j < pc.NumRealComps(); j++) {
+                // extra SoA Real components
+                const int real_start_offset = PC::ParticleType::is_soa_particle ? AMREX_SPACEDIM : 0; // pure SoA: skip positions
+                for (int j = real_start_offset; j < pc.NumRealComps(); j++) {
                     if (write_real_comp[PC::NStructReal+j]) {
                         *rptr = (ParticleReal) soa.GetRealData(j)[pindex];
                         ++rptr;
@@ -647,8 +673,7 @@ void WriteBinaryParticleDataAsync (PC const& pc,
         {
             int gid = pti.index();
             const auto& ptile = pc.ParticlesAt(lev, pti);
-            const auto& aos = ptile.GetArrayOfStructs();
-            const auto pstruct = aos().dataPtr();
+            const auto& ptd = ptile.getConstParticleTileData();
             const int np = ptile.numParticles();
 
             ReduceOps<ReduceOpSum> reduce_op;
@@ -658,7 +683,7 @@ void WriteBinaryParticleDataAsync (PC const& pc,
             reduce_op.eval(np, reduce_data,
             [=] AMREX_GPU_DEVICE (int i) -> ReduceTuple
             {
-                return (pstruct[i].id() > 0) ? 1 : 0;
+                return (ptd.id(i) > 0) ? 1 : 0;
             });
 
             int np_valid = amrex::get<0>(reduce_data.value(reduce_op));
@@ -938,30 +963,43 @@ void WriteBinaryParticleDataAsync (PC const& pc,
                 for (unsigned i = 0; i < tile_map[grid].size(); i++) {
                     auto ptile_index = std::make_pair(grid, tile_map[grid][i]);
                     const auto& pbox = (*myptiles)[lev][ptile_index];
-                    for (int pindex = 0;
-                         pindex < pbox.GetArrayOfStructs().numParticles(); ++pindex)
+                    const auto ptd = pbox.getConstParticleTileData();
+                    for (int pindex = 0; pindex < pbox.numParticles(); ++pindex)
                     {
-                        const auto& aos = pbox.GetArrayOfStructs();
-                        const auto& p = aos[pindex];
+                        const auto& soa  = pbox.GetStructOfArrays();
 
+                        const auto& p = make_particle<typename PC::ConstParticleType>{}(ptd, pindex);
                         if (p.id() <= 0) continue;
 
-                        // always write these
-                        particle_detail::packParticleIDs(iptr, p, is_checkpoint);
-                        iptr += 2;
-
-                        // optionally write these
-                        for (int j = 0; j < NStructInt; j++)
+                        // note: for pure SoA particle layouts, we do write the id, cpu and positions as a struct
+                        //       for backwards compatibility with readers
+                        if constexpr(!PC::ParticleType::is_soa_particle)
                         {
-                            if (write_int_comp[j])
+                            // Ints: id, cpu
+                            particle_detail::packParticleIDs(iptr, p, is_checkpoint);
+                            iptr += 2;
+
+                            // extra AoS Int components
+                            for (int j = 0; j < NStructInt; j++)
                             {
-                                *iptr = p.idata(j);
-                                ++iptr;
+                                if (write_int_comp[j])
+                                {
+                                    *iptr = p.idata(j);
+                                    ++iptr;
+                                }
                             }
                         }
+                        else {
+                            // Ints: id, cpu
+                            *iptr = soa.GetIntData(0)[pindex];
+                            iptr += 1;
+                            *iptr = soa.GetIntData(1)[pindex];
+                            iptr += 1;
+                        }
 
-                        const auto& soa  = pbox.GetStructOfArrays();
-                        for (int j = 0; j < nic; j++)
+                        // extra SoA Ints
+                        const int int_start_offset = PC::ParticleType::is_soa_particle ? 2 : 0; // pure SoA: skip id, cpu
+                        for (int j = int_start_offset; j < nic; j++)
                         {
                             if (write_int_comp[NStructInt+j])
                             {
@@ -987,30 +1025,40 @@ void WriteBinaryParticleDataAsync (PC const& pc,
                 for (unsigned i = 0; i < tile_map[grid].size(); i++) {
                     auto ptile_index = std::make_pair(grid, tile_map[grid][i]);
                     const auto& pbox = (*myptiles)[lev][ptile_index];
+                    const auto ptd = pbox.getConstParticleTileData();
                     for (int pindex = 0;
-                         pindex < pbox.GetArrayOfStructs().numParticles(); ++pindex)
+                         pindex < pbox.numParticles(); ++pindex)
                     {
-                        const auto& aos = pbox.GetArrayOfStructs();
-                        const auto& p = aos[pindex];
+                        const auto& soa  = pbox.GetStructOfArrays();
+                        const auto& p = make_particle<typename PC::ConstParticleType>{}(ptd, pindex);
 
                         if (p.id() <= 0) continue;
 
-                        // always write these
-                        for (int j = 0; j < AMREX_SPACEDIM; j++) rptr[j] = p.pos(j);
-                        rptr += AMREX_SPACEDIM;
-
-                        // optionally write these
-                        for (int j = 0; j < NStructReal; j++)
+                        if constexpr(!PC::ParticleType::is_soa_particle)
                         {
-                            if (write_real_comp[j])
+                            // Real: position
+                            for (int j = 0; j < AMREX_SPACEDIM; j++) rptr[j] = p.pos(j);
+                            rptr += AMREX_SPACEDIM;
+
+                            // extra AoS real
+                            for (int j = 0; j < NStructReal; j++)
                             {
-                                *rptr = p.rdata(j);
-                                ++rptr;
+                                if (write_real_comp[j])
+                                {
+                                    *rptr = p.rdata(j);
+                                    ++rptr;
+                                }
                             }
                         }
+                        else {
+                            // Real: position
+                            for (int j = 0; j < AMREX_SPACEDIM; j++) rptr[j] = soa.GetRealData(j)[pindex];
+                            rptr += AMREX_SPACEDIM;
+                        }
 
-                        const auto& soa  = pbox.GetStructOfArrays();
-                        for (int j = 0; j < nrc; j++)
+                        // extra SoA real
+                        const int real_start_offset = PC::ParticleType::is_soa_particle ? AMREX_SPACEDIM : 0; // pure SoA: positions
+                        for (int j = real_start_offset; j < nrc; j++)
                         {
                             if (write_real_comp[NStructReal+j])
                             {


Test patch:
diff --git a/Tests/Particles/CheckpointRestartSOA/CMakeLists.txt b/Tests/Particles/CheckpointRestartSOA/CMakeLists.txt
new file mode 100644
index 00000000000..087d3ad4998
--- /dev/null
+++ b/Tests/Particles/CheckpointRestartSOA/CMakeLists.txt
@@ -0,0 +1,14 @@
+# This tests requires particle support
+if (NOT AMReX_PARTICLES)
+   return()
+endif ()
+
+foreach(D IN LISTS AMReX_SPACEDIM)
+    set(_sources     main.cpp)
+    set(_input_files inputs)
+
+    setup_test(${D} _sources _input_files)
+
+    unset(_sources)
+    unset(_input_files)
+endforeach()
diff --git a/Tests/Particles/CheckpointRestartSOA/GNUmakefile b/Tests/Particles/CheckpointRestartSOA/GNUmakefile
new file mode 100644
index 00000000000..b3510b88d2e
--- /dev/null
+++ b/Tests/Particles/CheckpointRestartSOA/GNUmakefile
@@ -0,0 +1,31 @@
+AMREX_HOME = ../../../
+
+# DEBUG	= TRUE
+DEBUG	= FALSE
+
+DIM	= 3
+
+COMP    = gnu
+
+PRECISION = DOUBLE
+
+USE_MPI   = TRUE
+MPI_THREAD_MULTIPLE = FALSE
+
+USE_OMP   = FALSE
+
+TINY_PROFILE = TRUE
+
+USE_PARTICLES = TRUE
+
+###################################################
+
+EBASE     = main
+
+include $(AMREX_HOME)/Tools/GNUMake/Make.defs
+
+include ./Make.package
+include $(AMREX_HOME)/Src/Base/Make.package
+include $(AMREX_HOME)/Src/Particle/Make.package
+
+include $(AMREX_HOME)/Tools/GNUMake/Make.rules
diff --git a/Tests/Particles/CheckpointRestartSOA/Make.package b/Tests/Particles/CheckpointRestartSOA/Make.package
new file mode 100644
index 00000000000..6b4b865e8fc
--- /dev/null
+++ b/Tests/Particles/CheckpointRestartSOA/Make.package
@@ -0,0 +1,1 @@
+CEXE_sources += main.cpp
diff --git a/Tests/Particles/CheckpointRestartSOA/inputs b/Tests/Particles/CheckpointRestartSOA/inputs
new file mode 100644
index 00000000000..81855c5947d
--- /dev/null
+++ b/Tests/Particles/CheckpointRestartSOA/inputs
@@ -0,0 +1,26 @@
+# Domain size
+ncells = 64
+
+# Maximum allowable size of each subdomain in the problem domain;
+# this is used to decompose the domain for parallel calculations.
+max_grid_size = 8
+
+# Number of levels
+nlevs = 1
+
+# Number of components in the multifabs
+ncomp = 6
+
+# Number of particles per cell
+nppc = 2
+
+# Number of plot files to write
+nplotfile = 1
+
+# Number of plot files to write
+nparticlefile = 1
+
+# Whether to check the correctness of Checkpoint / Restart
+restart_check = 1
+
+directory=.
diff --git a/Tests/Particles/CheckpointRestartSOA/main.cpp b/Tests/Particles/CheckpointRestartSOA/main.cpp
new file mode 100644
index 00000000000..b6ccc365964
--- /dev/null
+++ b/Tests/Particles/CheckpointRestartSOA/main.cpp
@@ -0,0 +1,245 @@
+#include <AMReX.H>
+#include <AMReX_ParmParse.H>
+#include <AMReX_PlotFileUtil.H>
+#include <AMReX_Particles.H>
+
+#include <cstdio>
+
+using namespace amrex;
+
+void set_grids_nested (Vector<Box>& domains,
+                       Vector<BoxArray>& grids,
+                       Vector<IntVect>& ref_ratio);
+void test ();
+
+int main(int argc, char* argv[])
+{
+    amrex::Initialize(argc,argv);
+    test();
+    amrex::Finalize();
+}
+
+void test ()
+{
+    const int nghost = 0;
+    int ncells, max_grid_size, ncomp, nlevs, nppc;
+    int restart_check = 0, nplotfile = 1, nparticlefile = 1;
+    std::string directory;
+
+    ParmParse pp;
+    pp.get("ncells", ncells);
+    pp.get("max_grid_size", max_grid_size);
+    pp.get("ncomp", ncomp);
+    pp.get("nlevs", nlevs);
+    pp.get("nppc", nppc);
+    pp.query("nplotfile", nplotfile);
+    pp.query("nparticlefile", nparticlefile);
+    pp.query("restart_check", restart_check);
+    pp.query("directory", directory);
+
+    if (!directory.empty() && directory.back() != '/') {
+        // Include separator if one was not provided
+        directory += "/";
+    }
+
+    Vector<Box> domains;
+    Vector<BoxArray> ba;
+    Vector<IntVect> ref_ratio;
+
+    set_grids_nested(domains, ba, ref_ratio);
+
+    RealBox real_box;
+    for (int n = 0; n < AMREX_SPACEDIM; n++) {
+        real_box.setLo(n, 0.0);
+        real_box.setHi(n, 1.0);
+    }
+
+    // This sets the boundary conditions to be doubly or triply periodic
+    int is_per[] = {AMREX_D_DECL(1,1,1)};
+
+    // This defines a Geometry object for each level
+    Vector<Geometry> geom(nlevs);
+    geom[0].define(domains[0], &real_box, CoordSys::cartesian, is_per);
+    for (int lev = 1; lev < nlevs; lev++) {
+        geom[lev].define(domains[lev], &real_box, CoordSys::cartesian, is_per);
+    }
+
+    Vector<DistributionMapping> dmap(nlevs);
+
+    // write some mesh data too, because tools like yt expect them to be there
+    Vector<std::unique_ptr<MultiFab> > mf(nlevs);
+    for (int lev = 0; lev < nlevs; lev++) {
+        dmap[lev] = DistributionMapping{ba[lev]};
+        mf[lev] = std::make_unique<MultiFab>(ba[lev], dmap[lev], ncomp, nghost);
+        mf[lev]->setVal(lev);
+    }
+
+    // these don't really matter, make something up
+    const Real time = 0.0;
+
+    Vector<std::string> varnames;
+    for (int i = 0; i < ncomp; ++i)
+    {
+        varnames.push_back("component_" + std::to_string(i));
+    }
+
+    Vector<int> level_steps(nlevs, 0);
+
+    char fname[512];
+    for (int ts = 0; ts < nplotfile; ts++) {
+        std::snprintf(fname, sizeof fname, "%splt%05d", directory.c_str(), ts);
+
+        amrex::Print() << "Writing plot file [" << fname << "] ..." << std::endl;
+
+        WriteMultiLevelPlotfile(fname, nlevs, amrex::GetVecOfConstPtrs(mf),
+                                varnames, geom, time, level_steps, ref_ratio);
+
+        amrex::Print() << " done \n";
+    }
+
+    // Add some particles
+    constexpr int NReal  = 12;
+    constexpr int NInt   = 4;
+
+    typedef ParticleContainerPureSoA<NReal, NInt> MyPC;
+    MyPC myPC(geom, dmap, ba, ref_ratio);
+    myPC.SetVerbose(false);
+
+    int num_particles = nppc * AMREX_D_TERM(ncells, * ncells, * ncells);
+    bool serialize = false;
+    int iseed = 451;
+    MyPC::ParticleInitData pdata = {{}, {},
+                      {1.0, 2.0, 3.0, 4.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0},
+                      {5, 14, 15, 16}};
+
+    if (nparticlefile > 0) {
+        amrex::Print() << "Init particles ..." << std::endl;
+
+        myPC.InitRandom(num_particles, iseed, pdata, serialize);
+
+        amrex::Print() << " done \n";
+
+        Vector<std::string> particle_realnames;
+        for (int i = 0; i < NReal; ++i)
+            particle_realnames.push_back("particle_real_component_" + std::to_string(i));
+
+        Vector<std::string> particle_intnames;
+        for (int i = 0; i < NInt; ++i)
+            particle_intnames.push_back("particle_int_component_" + std::to_string(i));
+
+        for (int ts = 0; ts < nparticlefile; ts++) {
+            std::snprintf(fname, sizeof fname, "%splt%05d", directory.c_str(), ts);
+
+            amrex::Print() << "Writing particle file [" << fname << "] ..." << std::endl;
+
+            myPC.Checkpoint(fname, "particle0", false, particle_realnames, particle_intnames);
+
+            amrex::Print() << " done \n";
+        }
+    }
+
+    AsyncOut::Finish();
+    ParallelDescriptor::Barrier();
+
+    char directory_path[512];
+    if (restart_check && nparticlefile > 0)
+    {
+        MyPC newPC(geom, dmap, ba, ref_ratio);
+        std::snprintf(directory_path, sizeof directory_path, "%s%s", directory.c_str(), "plt00000");
+        newPC.Restart(directory_path, "particle0");
+
+        using PType = typename MyPC::SuperParticleType;
+
+        for (int icomp=0; icomp<NReal; ++icomp)
+        {
+            amrex::Print() << "working on comp " << icomp << "\n";
+            auto sm_new = amrex::ReduceSum(newPC,
+                [=] AMREX_GPU_HOST_DEVICE (const PType& p) -> Real
+                {
+                    return p.rdata(icomp);
+                });
+
+            auto sm_old = amrex::ReduceSum(myPC,
+                [=] AMREX_GPU_HOST_DEVICE (const PType& p) -> Real
+                {
+                    return p.rdata(icomp);
+                });
+
+            ParallelDescriptor::ReduceRealSum(sm_new);
+            ParallelDescriptor::ReduceRealSum(sm_old);
+
+            amrex::Print() << sm_old << " " << sm_new << "\n";
+            AMREX_ALWAYS_ASSERT(sm_old == sm_new);
+        }
+
+        for (int icomp=0; icomp<NInt; ++icomp)
+        {
+            amrex::Print() << "working on comp " << icomp << "\n";
+            auto sm_new = amrex::ReduceSum(newPC,
+                [=] AMREX_GPU_HOST_DEVICE (const PType& p) -> Real
+                {
+                    return p.idata(icomp);
+                });
+
+            auto sm_old = amrex::ReduceSum(myPC,
+                [=] AMREX_GPU_HOST_DEVICE (const PType& p) -> Real
+                {
+                    return p.idata(icomp);
+                });
+
+            ParallelDescriptor::ReduceRealSum(sm_new);
+            ParallelDescriptor::ReduceRealSum(sm_old);
+
+            amrex::Print() << sm_old << " " << sm_new << "\n";
+            AMREX_ALWAYS_ASSERT(sm_old == sm_new);
+        }
+    }
+}
+
+void set_grids_nested (Vector<Box>& domains,
+                       Vector<BoxArray>& grids,
+                       Vector<IntVect>& ref_ratio)
+{
+    int ncells, max_grid_size, nlevs;
+
+    ParmParse pp;
+    pp.get("ncells", ncells);
+    pp.get("max_grid_size", max_grid_size);
+    pp.get("nlevs", nlevs);
+
+    AMREX_ALWAYS_ASSERT(nlevs < 2); // relax this later
+
+    IntVect domain_lo(AMREX_D_DECL(0, 0, 0));
+    IntVect domain_hi(AMREX_D_DECL(ncells-1, ncells-1, ncells-1));
+
+    domains.resize(nlevs);
+    domains[0].setSmall(domain_lo);
+    domains[0].setBig(domain_hi);
+
+    ref_ratio.resize(nlevs-1);
+    for (int lev = 1; lev < nlevs; lev++)
+        ref_ratio[lev-1] = IntVect(AMREX_D_DECL(2, 2, 2));
+
+    grids.resize(nlevs);
+    grids[0].define(domains[0]);
+
+    // Now we make the refined level be the center eighth of the domain
+    if (nlevs > 1) {
+        int n_fine = ncells*ref_ratio[0][0];
+        IntVect refined_lo(AMREX_D_DECL(n_fine/4,n_fine/4,n_fine/4));
+        IntVect refined_hi(AMREX_D_DECL(3*n_fine/4-1,3*n_fine/4-1,3*n_fine/4-1));
+
+        // Build a box for the level 1 domain
+        Box refined_patch(refined_lo, refined_hi);
+        grids[1].define(refined_patch);
+    }
+
+    // break the BoxArrays at both levels into max_grid_size^3 boxes
+    for (int lev = 0; lev < nlevs; lev++) {
+        grids[lev].maxSize(max_grid_size);
+    }
+
+    for (int lev = 1; lev < nlevs; lev++) {
+        domains[lev] = amrex::refine(domains[lev-1], ref_ratio[lev-1]);
+    }
+}
diff --git a/Tests/Particles/CheckpointRestartSOA_AsyncIO/CMakeLists.txt b/Tests/Particles/CheckpointRestartSOA_AsyncIO/CMakeLists.txt
new file mode 100644
index 00000000000..087d3ad4998
--- /dev/null
+++ b/Tests/Particles/CheckpointRestartSOA_AsyncIO/CMakeLists.txt
@@ -0,0 +1,14 @@
+# This tests requires particle support
+if (NOT AMReX_PARTICLES)
+   return()
+endif ()
+
+foreach(D IN LISTS AMReX_SPACEDIM)
+    set(_sources     main.cpp)
+    set(_input_files inputs)
+
+    setup_test(${D} _sources _input_files)
+
+    unset(_sources)
+    unset(_input_files)
+endforeach()
diff --git a/Tests/Particles/CheckpointRestartSOA_AsyncIO/GNUmakefile b/Tests/Particles/CheckpointRestartSOA_AsyncIO/GNUmakefile
new file mode 120000
index 00000000000..402f7774f22
--- /dev/null
+++ b/Tests/Particles/CheckpointRestartSOA_AsyncIO/GNUmakefile
@@ -0,0 +1,1 @@
+../CheckpointRestartSOA/GNUmakefile
\ No newline at end of file
diff --git a/Tests/Particles/CheckpointRestartSOA_AsyncIO/Make.package b/Tests/Particles/CheckpointRestartSOA_AsyncIO/Make.package
new file mode 120000
index 00000000000..6fb6dc981b7
--- /dev/null
+++ b/Tests/Particles/CheckpointRestartSOA_AsyncIO/Make.package
@@ -0,0 +1,1 @@
+../CheckpointRestartSOA/Make.package
\ No newline at end of file
diff --git a/Tests/Particles/CheckpointRestartSOA_AsyncIO/inputs b/Tests/Particles/CheckpointRestartSOA_AsyncIO/inputs
new file mode 100644
index 00000000000..8efbea33ad4
--- /dev/null
+++ b/Tests/Particles/CheckpointRestartSOA_AsyncIO/inputs
@@ -0,0 +1,28 @@
+# Domain size
+ncells = 64
+
+# Maximum allowable size of each subdomain in the problem domain;
+# this is used to decompose the domain for parallel calculations.
+max_grid_size = 8
+
+# Number of levels
+nlevs = 1
+
+# Number of components in the multifabs
+ncomp = 6
+
+# Number of particles per cell
+nppc = 2
+
+# Number of plot files to write
+nplotfile = 1
+
+# Number of plot files to write
+nparticlefile = 1
+
+# Whether to check the correctness of Checkpoint / Restart
+restart_check = 1
+
+directory=.
+
+amrex.async_out=1
\ No newline at end of file
diff --git a/Tests/Particles/CheckpointRestartSOA_AsyncIO/main.cpp b/Tests/Particles/CheckpointRestartSOA_AsyncIO/main.cpp
new file mode 120000
index 00000000000..36e9ca62484
--- /dev/null
+++ b/Tests/Particles/CheckpointRestartSOA_AsyncIO/main.cpp
@@ -0,0 +1,1 @@
+../CheckpointRestartSOA/main.cpp
\ No newline at end of file