Task: Ascent: SoA Particle Support (AMReX-Codes/amrex#3350)

org: AMReX-Codes
repo: amrex
pr_number: 3350
issue_title: Ascent: SoA Particle Support
issue_body: ## Summary

Add support for pure SoA layouted particle containers for Ascent.

## Additional background

Follow-up to #2878.

## Checklist

The proposed changes:
- [ ] fix a bug or incorrect behavior in AMReX
- [x] add new capabilities to AMReX
- [ ] changes answers in the test suite to more than roundoff level
- [ ] are likely to significantly affect the results of downstream AMReX users
- [ ] include documentation in the code and/or rst files, if appropriate

resolved_issues: number: 2878
title: Support SoA-Only Particles
body: ## Summary

Further updates of  #2853 where @atmyers and @ax3l can also push to.

## Upgrade Guide for C++ API Changes

These PRs show changes to keep using the *old layout*:
- example in WarpX: https://github.com/ECP-WarpX/WarpX/pull/3848
- example in HiPACE++: https://github.com/Hi-PACE/hipace/pull/921
- example in ImpactX: https://github.com/ECP-WarpX/impactx/pull/342
- example in pyAMReX: https://github.com/AMReX-Codes/pyamrex/pull/119

These PRs show changes to *use the new layout* (pure SoA):
- example in WarpX: https://github.com/ECP-WarpX/WarpX/pull/3850
- example in ImpactX: https://github.com/ECP-WarpX/impactx/pull/348
- example in pyAMReX: https://github.com/AMReX-Codes/pyamrex/pull/124

## Additional background

## Checklist

The proposed changes:
- [ ] fix a bug or incorrect behavior in AMReX
- [x] add new capabilities to AMReX
- [ ] changes answers in the test suite to more than roundoff level
- [ ] are likely to significantly affect the results of downstream AMReX users
- [ ] include documentation in the code and/or rst files, if appropriate


Reference solution:
fix_patch: diff --git a/Src/Extern/Conduit/AMReX_Conduit_Blueprint.H b/Src/Extern/Conduit/AMReX_Conduit_Blueprint.H
index 6d23bcf07ed..9ac8eb31fd0 100644
--- a/Src/Extern/Conduit/AMReX_Conduit_Blueprint.H
+++ b/Src/Extern/Conduit/AMReX_Conduit_Blueprint.H
@@ -96,9 +96,8 @@ namespace amrex
     //  coordset and fields used to represent the passed particle container.
     //  This allows you to use unique names to wrap multiple particle containers
     //  into a single blueprint tree.
-    template <int NStructReal, int NStructInt, int NArrayReal, int NArrayInt>
-    void ParticleContainerToBlueprint (const ParticleContainer<NStructReal,
-                                                               NStructInt,
+    template <typename ParticleType, int NArrayReal, int NArrayInt>
+    void ParticleContainerToBlueprint (const ParticleContainer_impl<ParticleType,
                                                                NArrayReal,
                                                                NArrayInt> &pc,
                                        const Vector<std::string> &real_comp_names,
diff --git a/Src/Extern/Conduit/AMReX_Conduit_Blueprint_ParticlesI.H b/Src/Extern/Conduit/AMReX_Conduit_Blueprint_ParticlesI.H
index f2d7d1ed2da..e4186ba247b 100644
--- a/Src/Extern/Conduit/AMReX_Conduit_Blueprint_ParticlesI.H
+++ b/Src/Extern/Conduit/AMReX_Conduit_Blueprint_ParticlesI.H
@@ -20,10 +20,9 @@ namespace amrex
 // Note:
 // This is a helper function, it's not part of the AMReX Blueprint Interface.
 //---------------------------------------------------------------------------//
-template <int NStructReal, int NStructInt, int NArrayReal, int NArrayInt>
+template <typename ParticleType, int NArrayReal, int NArrayInt>
 void
-ParticleTileToBlueprint(const ParticleTile<amrex::Particle<NStructReal,
-                                                           NStructInt>,
+ParticleTileToBlueprint(const ParticleTile<ParticleType,
                                            NArrayReal,
                                            NArrayInt> &ptile,
                         const Vector<std::string> &real_comp_names,
@@ -31,15 +30,11 @@ ParticleTileToBlueprint(const ParticleTile<amrex::Particle<NStructReal,
                         conduit::Node &res,
                         const std::string &topology_name)
 {
-    int num_particles = ptile.GetArrayOfStructs().size();
-    int struct_size   = sizeof(Particle<NStructReal, NStructInt>);
+    int num_particles = ptile.size();
 
     // knowing the above, we can zero copy the x,y,z positions + id, cpu
     // and any user fields in the AOS
 
-    // get the first particle's struct
-    const auto &pstruct = ptile.GetArrayOfStructs();
-
     // setup a blueprint description for the particle mesh
     // create a coordinate set
     std::string coordset_name = topology_name + "_coords";
@@ -63,29 +58,56 @@ ParticleTileToBlueprint(const ParticleTile<amrex::Particle<NStructReal,
     //----------------------------------//
     // point locations from from aos
     //----------------------------------//
+    char* pbuf = nullptr;
 
-    const char* pbuf_const = reinterpret_cast<const char*>(pstruct.data());
-    char* pbuf = const_cast<char*>(pbuf_const);
+    if constexpr(ParticleType::is_soa_particle)
+    {
+        amrex::ignore_unused(pbuf);
 
-    ParticleReal* xp = reinterpret_cast<ParticleReal*>(pbuf); pbuf += sizeof(ParticleReal);
-    n_coords["values/x"].set_external(xp,
-                                      num_particles,
-                                      0,
-                                      struct_size);
+        const auto &soa = ptile.GetStructOfArrays();
+
+        // for soa entries, we can use standard strides,
+        // since these are contiguous arrays
+
+        n_coords["values/x"].set_external(const_cast<ParticleReal*>(&soa.GetRealData(0)[0]),
+                                          num_particles);
 #if AMREX_SPACEDIM > 1
-    ParticleReal* yp = reinterpret_cast<ParticleReal*>(pbuf); pbuf += sizeof(ParticleReal);
-    n_coords["values/y"].set_external(yp,
-                                      num_particles,
-                                      0,
-                                      struct_size);
+        n_coords["values/y"].set_external(const_cast<ParticleReal*>(&soa.GetRealData(1)[0]),
+                                          num_particles);
 #endif
 #if AMREX_SPACEDIM > 2
-    ParticleReal* zp = reinterpret_cast<ParticleReal*>(pbuf); pbuf += sizeof(ParticleReal);
-    n_coords["values/z"].set_external(zp,
-                                      num_particles,
-                                      0,
-                                      struct_size);
+        n_coords["values/z"].set_external(const_cast<ParticleReal*>(&soa.GetRealData(2)[0]),
+                                          num_particles);
+#endif
+    } else
+    {
+        // get the first particle's struct
+        const auto &pstruct = ptile.GetArrayOfStructs();
+        const int struct_size = sizeof(ParticleType);
+
+        const char* pbuf_const = reinterpret_cast<const char*>(pstruct.data());
+        pbuf = const_cast<char*>(pbuf_const);
+
+        ParticleReal* xp = reinterpret_cast<ParticleReal*>(pbuf); pbuf += sizeof(ParticleReal);
+        n_coords["values/x"].set_external(xp,
+                                          num_particles,
+                                          0,
+                                          struct_size);
+#if AMREX_SPACEDIM > 1
+        ParticleReal* yp = reinterpret_cast<ParticleReal*>(pbuf); pbuf += sizeof(ParticleReal);
+        n_coords["values/y"].set_external(yp,
+                                          num_particles,
+                                          0,
+                                          struct_size);
+#endif
+#if AMREX_SPACEDIM > 2
+        ParticleReal* zp = reinterpret_cast<ParticleReal*>(pbuf); pbuf += sizeof(ParticleReal);
+        n_coords["values/z"].set_external(zp,
+                                          num_particles,
+                                          0,
+                                          struct_size);
 #endif
+    }
 
     // fields
     conduit::Node &n_fields = res["fields"];
@@ -95,20 +117,26 @@ ParticleTileToBlueprint(const ParticleTile<amrex::Particle<NStructReal,
     // -----------------------------
 
     int vname_real_idx = 0;
-    // struct real fields, the first set are always the particle positions
-    // which we wrap above
-    for (int i = 0; i < NStructReal; i++)
+    if constexpr(!ParticleType::is_soa_particle)
     {
-        ParticleReal* val = reinterpret_cast<ParticleReal*>(pbuf); pbuf += sizeof(ParticleReal);
-        conduit::Node &n_f = n_fields[real_comp_names.at(vname_real_idx)];
-        n_f["topology"] = topology_name;
-        n_f["association"] = "element";
-        n_f["values"].set_external(val,
-                                   num_particles,
-                                   0,
-                                   struct_size);
+        constexpr int struct_size = sizeof(ParticleType);
+        constexpr int NStructReal = ParticleType::NReal;
 
-        vname_real_idx++;
+        // struct real fields, the first set are always the particle positions
+        // which we wrap above
+        for (int i = 0; i < NStructReal; i++)
+        {
+            ParticleReal* val = reinterpret_cast<ParticleReal*>(pbuf); pbuf += sizeof(ParticleReal);
+            conduit::Node &n_f = n_fields[real_comp_names.at(vname_real_idx)];
+            n_f["topology"] = topology_name;
+            n_f["association"] = "element";
+            n_f["values"].set_external(val,
+                                       num_particles,
+                                       0,
+                                       struct_size);
+
+            vname_real_idx++;
+        }
     }
 
     //----------------------------------//
@@ -116,44 +144,77 @@ ParticleTileToBlueprint(const ParticleTile<amrex::Particle<NStructReal,
     // (id, cpu)
     //----------------------------------//
 
-    // id is the first int entry
-    int* id = reinterpret_cast<int*>(pbuf); pbuf += sizeof(int);
-    conduit::Node &n_f_id = n_fields[topology_name + "_id"];
+    if constexpr(!ParticleType::is_soa_particle)
+    {
+        const int struct_size = sizeof(ParticleType);
+
+        // id is the first int entry
+        int* id = reinterpret_cast<int*>(pbuf); pbuf += sizeof(int);
+        conduit::Node &n_f_id = n_fields[topology_name + "_id"];
+
+        n_f_id["topology"] = topology_name;
+        n_f_id["association"] = "element";
+        n_f_id["values"].set_external(id,
+                                      num_particles,
+                                      0,
+                                      struct_size);
+
+        // cpu is the second int entry
+        int* cpu = reinterpret_cast<int*>(pbuf); pbuf += sizeof(int);
+        conduit::Node &n_f_cpu = n_fields[topology_name + "_cpu"];
+
+        n_f_cpu["topology"] = topology_name;
+        n_f_cpu["association"] = "element";
+        n_f_cpu["values"].set_external(cpu,
+                                       num_particles,
+                                       0,
+                                       struct_size);
+    } else {
+        const auto &soa = ptile.GetStructOfArrays();
+
+        // for soa entries, we can use standard strides,
+        // since these are contiguous arrays
 
-    n_f_id["topology"] = topology_name;
-    n_f_id["association"] = "element";
-    n_f_id["values"].set_external(id,
-                                  num_particles,
-                                  0,
-                                  struct_size);
+        // id is the first int entry
+        conduit::Node &n_f_id = n_fields[topology_name + "_id"];
 
-    // cpu is the second int entry
-    int* cpu = reinterpret_cast<int*>(pbuf); pbuf += sizeof(int);
-    conduit::Node &n_f_cpu = n_fields[topology_name + "_cpu"];
+        n_f_id["topology"] = topology_name;
+        n_f_id["association"] = "element";
+        n_f_id["values"].set_external(const_cast<int*>(&soa.GetIntData(0)[0]),
+                                      num_particles);
 
-    n_f_cpu["topology"] = topology_name;
-    n_f_cpu["association"] = "element";
-    n_f_cpu["values"].set_external(cpu,
-                                   num_particles,
-                                   0,
-                                   struct_size);
+        // cpu is the second int entry
+        conduit::Node &n_f_cpu = n_fields[topology_name + "_cpu"];
+
+        n_f_cpu["topology"] = topology_name;
+        n_f_cpu["association"] = "element";
+        n_f_cpu["values"].set_external(const_cast<int*>(&soa.GetIntData(0)[0]),
+                                       num_particles);
+
+    }
 
     // --------------------------------
     // user defined, integer aos fields
     // --------------------------------
 
     int vname_int_idx = 0;
-    for (int i = 0; i < NStructInt; i++)
+    if constexpr(!ParticleType::is_soa_particle)
     {
-        int* val = reinterpret_cast<int*>(pbuf); pbuf += sizeof(int);
-        conduit::Node &n_f = n_fields[int_comp_names.at(vname_int_idx)];
-        n_f["topology"] = topology_name;
-        n_f["association"] = "element";
-        n_f["values"].set_external(val,
-                                   num_particles,
-                                   0,
-                                   struct_size);
-        vname_int_idx++;
+        constexpr int struct_size = sizeof(ParticleType);
+        constexpr int NStructInt = ParticleType::NInt;
+
+        for (int i = 0; i < NStructInt; i++)
+        {
+            int* val = reinterpret_cast<int*>(pbuf); pbuf += sizeof(int);
+            conduit::Node &n_f = n_fields[int_comp_names.at(vname_int_idx)];
+            n_f["topology"] = topology_name;
+            n_f["association"] = "element";
+            n_f["values"].set_external(val,
+                                       num_particles,
+                                       0,
+                                       struct_size);
+            vname_int_idx++;
+        }
     }
 
     // -------------------------
@@ -193,10 +254,9 @@ ParticleTileToBlueprint(const ParticleTile<amrex::Particle<NStructReal,
 //---------------------------------------------------------------------------//
 // Converts a AMReX Particle Container into a Conduit Mesh Blueprint Hierarchy.
 //---------------------------------------------------------------------------//
-template <int NStructReal, int NStructInt, int NArrayReal, int NArrayInt>
+template <typename ParticleType, int NArrayReal, int NArrayInt>
 void
-ParticleContainerToBlueprint(const ParticleContainer<NStructReal,
-                                                     NStructInt,
+ParticleContainerToBlueprint(const ParticleContainer_impl<ParticleType,
                                                      NArrayReal,
                                                      NArrayInt> &pc,
                              const Vector<std::string> &real_comp_names,
@@ -209,8 +269,13 @@ ParticleContainerToBlueprint(const ParticleContainer<NStructReal,
     // validate varnames, which are used to provide field names
     // for user defined aos and soa values.
 
-    BL_ASSERT(real_comp_names.size() == (NStructReal + NArrayReal) );
-    BL_ASSERT(int_comp_names.size()  == (NStructInt + NArrayInt) );
+    if constexpr(ParticleType::is_soa_particle) {
+        BL_ASSERT(real_comp_names.size() == NArrayReal);
+        BL_ASSERT(int_comp_names.size()  == NArrayInt);
+    } else {
+        BL_ASSERT(real_comp_names.size() == (ParticleType::NReal + NArrayReal) );
+        BL_ASSERT(int_comp_names.size()  == (ParticleType::NInt + NArrayInt) );
+    }
 
     int num_levels = pc.maxLevel() + 1;
     int num_domains = 0;
@@ -224,7 +289,7 @@ ParticleContainerToBlueprint(const ParticleContainer<NStructReal,
     int rank   = ParallelDescriptor::MyProc();
     int nprocs = ParallelDescriptor::NProcs();
 
-    using MyParConstIter = ParConstIter<NStructReal, NStructInt, NArrayReal, NArrayInt>;
+    using MyParConstIter = ParConstIter_impl<ParticleType, NArrayReal, NArrayInt>;
 
     //
     // blueprint expects unique ids for each domain published