Repository: yt-project/yt

Issue:
No way to do ProjectionPlot on non-SPH demeshed particle data
### Bug report

**Bug summary**

Asking for a projection plot on non-SPH particle data in yt-4.0 (i.e. demeshed yt) suggests using a ParticlePlot (which does not provide the same visual or scientific appeal), or fails if you try to push it too hard.

**Code for reproduction**

Once loading particle data as `ds`, here accomplished with `yt.load_particles`:

```python

prj = yt.ProjectionPlot(ds, 0, ('star', 'particle_mass'), center=center)
   ...:
---------------------------------------------------------------------------
YTInvalidFieldType                        Traceback (most recent call last)
<ipython-input-8-f0cebe42ef5b> in <module>
----> 1 prj = yt.ProjectionPlot(ds, 0, ('star', 'particle_mass'), center=center)

~/yt4/src/yt/yt/visualization/plot_window.py in __init__(self, ds, axis, fields, center, width, axes_unit, weight_field, max_level, origin, right_handed, fontsize, field_parameters, data_source, method, proj_style, window_size, aspect)
   1462         # plotting classes to avoid an exception
   1463         test_data_source = ds.all_data()
-> 1464         validate_mesh_fields(test_data_source, fields)
   1465
   1466         if isinstance(ds, YTSpatialPlotDataset):

~/yt4/src/yt/yt/visualization/plot_window.py in validate_mesh_fields(data_source, fields)
    147
    148     if len(invalid_fields) > 0:
--> 149         raise YTInvalidFieldType(invalid_fields)
    150
    151

YTInvalidFieldType:
SlicePlot, ProjectionPlot, and OffAxisProjectionPlot can only plot fields that
are defined on a mesh or for SPH particles, but received the following N-body
particle fields:

    [('star', 'particle_mass')]

Did you mean to use ParticlePlot or plot a deposited particle field instead?
```

or, alternatively (and probably what _should_ work in the future):

```python

prj = yt.ProjectionPlot(ds, 0, ('deposit', 'star_density'), center=center)
   ...:
---------------------------------------------------------------------------
YTNonIndexedDataContainer                 Traceback (most recent call last)
<ipython-input-9-81448988a4b4> in <module>
----> 1 prj = yt.ProjectionPlot(ds, 0, ('deposit', 'star_density'), center=center)

~/yt4/src/yt/yt/visualization/plot_window.py in __init__(self, ds, axis, fields, center, width, axes_unit, weight_field, max_level, origin, right_handed, fontsize, field_parameters, data_source, method, proj_style, window_size, aspect)
   1478                            center=center, data_source=data_source,
   1479                            field_parameters=field_parameters, method=method,
-> 1480                            max_level=max_level)
   1481         PWViewerMPL.__init__(self, proj, bounds, fields=fields, origin=origin,
   1482                              right_handed=right_handed, fontsize=fontsize, window_size=window_size,

~/yt4/src/yt/yt/data_objects/construction_data_containers.py in __init__(self, field, axis, weight_field, center, ds, data_source, style, method, field_parameters, max_level)
    446
    447         if not self.deserialize(field):
--> 448             self.get_data(field)
    449             self.serialize()
    450

~/yt4/src/yt/yt/data_objects/construction_data_containers.py in get_data(self, fields)
    248                     self._initialize_projected_units(fields, chunk)
    249                     _units_initialized = True
--> 250                 self._handle_chunk(chunk, fields, tree)
    251         # if there's less than nprocs chunks, units won't be initialized
    252         # on all processors, so sync with _projected_units on rank 0

~/yt4/src/yt/yt/data_objects/construction_data_containers.py in _handle_chunk(self, chunk, fields, tree)
    503     def _handle_chunk(self, chunk, fields, tree):
    504         mylog.debug("Adding chunk (%s) to tree (%0.3e GB RAM)",
--> 505                     chunk.ires.size, get_memory_usage()/1024.)
    506         if self.method == "mip" or self._sum_only:
    507             dl = self.ds.quan(1.0, "")

~/yt4/src/yt/yt/data_objects/data_containers.py in ires(self)
   1575         if self._current_chunk is None:
   1576             self.index._identify_base_chunk(self)
-> 1577         return self._current_chunk.ires
   1578
   1579     @property

~/yt4/src/yt/yt/geometry/geometry_handler.py in cached_func(self)
    267             return getattr(self, n)
    268         if self.data_size is None:
--> 269             tr = self._accumulate_values(n[1:])
    270         else:
    271             tr = func(self)

~/yt4/src/yt/yt/geometry/geometry_handler.py in _accumulate_values(self, method)
    295         for obj in self._fast_index or self.objs:
    296             f = getattr(obj, mname)
--> 297             arrs.append(f(self.dobj))
    298         if method == "dtcoords":
    299             arrs = [arr[0] for arr in arrs]

~/yt4/src/yt/yt/data_objects/particle_container.py in _func_non_indexed(self, *args, **kwargs)
     28 def _non_indexed(name):
     29     def _func_non_indexed(self, *args, **kwargs):
---> 30         raise YTNonIndexedDataContainer(self)
     31     return _func_non_indexed
     32

YTNonIndexedDataContainer: The data container (<class 'yt.data_objects.particle_container.ParticleContainer'>) is an unindexed type.  Operations such as ires, icoords, fcoords and fwidth will not work on it.

```

**Expected outcome**

Should be able to visualize at least the density fields of non-SPH particle data.

**Version Information**
<!--Please specify your platform and versions of the relevant libraries you are
using:-->
  * Operating System:
  * Python Version:
  * yt version:
  * Other Libraries (if applicable): 

Using yt-4.0dev0, merge-rev 8154cbffd 751b2239e




Discussion:
So there are two issues here. 

One is that in the demeshening we don't have a 3D structure to deposit onto by "default". We used to use the octree. The grid frontends use the AMR mesh. We should probably come up with a way to handle this since it's also a backward compatibility issue.

The other issue is that we don't have a way to generate smoothing lengths for N-body particle data and visualize those data with the SPH pixelizers.

The solution to both problems are probably related, for example we could just deposit onto images, for example, and add the SPH smoothed visualuzation as a new deposit type.
I'm not sure I understand the subtleties of the first issue you mention (specifically, are SPH particles not put onto a 3D structure?), but with regard to the second -- per my discussion on the Slack -- is there a problem with using the distance to the Nth nearest neighbor (where N is an option passed in) to define smoothing length?  You could also allow the user to pass in an array/single value to hard-code the smoothing length too. 
There’s no problem with either option. We already have a utility function that does that (calculate smoothing lengths using the distance to the nth nearest neighbor, it’s used in the tipsy frontend because that data format doesn’t have smoothing lengths on dism). We just need to figure out an appropriate API and wire everything up.
Perhaps, although I think spelling it "from yt.utilities.sph_utilties import foo" vs "from sph_utilities import foo" doesn't really make much difference. Is there some other advantage you see from factoring this out?

Note also that this package would need to depend on cykdtree at a binary level and would have cython code, so you wouldn't get out of any packaging headaches doing that.

I think it might be a better use of time right now to focus on the API for exposing this functionality. Once we agree on an API we can think about moving code into new packages.

Also having a lower-level "nice" API for SPH smoothing things and generating smoothing lengths doesn't really fix the original issue with ProjectionPlot not knowing how to handle non-SPH particle data, so there would still be work that needs to happen in the high-level yt interface.
Sorry I deleted my original comment thinking it's immature after posting, which is pasted below:
> This might be a bad idea, but I'm wondering if the SPH-related functionalities, like pixelization, interpolation and smoothing length calculation could be extracted into a separate package, just like unyt? I feel they are not tightly coupled to any frontend, or the yt data representation in general. The low-level Cython functions operate on arrays as far as I know. If extracted into a separate package, it would probably be more flexible to use and easier to maintain. This might be a significant amount of work. But as the yt dev workshop is coming, could it be something worth considering?

Thanks for your kind reply. Yes, I'm mostly thinking about the lower-level API for SPH smoothing on arrays directly. Then a workaround for handling non-SPH particles would be pixelizing with a calculated smoothing length array using this lower-level API. It's indeed not solving the high-level interface issue. But having a convinient workarount could be a nice thing.

As I said, I feel my comment is immature because I haven't explored the relevant codebase enough. I'll take a deeper look to have a better idea.
I don't think your comment was immature! I'm sorry my tone made you feel that way, that was not my intent. It's definitely a fair question. I was just trying to explain why I think we need to worry about other things before it makes sense to discuss factoring the low-level SPH utilities into a new package.
Here's an example of what I'm thinking about: https://gist.github.com/qobilidop/a8c3b4777c94f160442af0446d191ace

For the original issue, I'm proposing using this low-level API to do SPH visualization with arbitrary arrays as a workaround.

Resolving patch:
diff --git a/yt/frontends/stream/data_structures.py b/yt/frontends/stream/data_structures.py
--- a/yt/frontends/stream/data_structures.py
+++ b/yt/frontends/stream/data_structures.py
@@ -26,6 +26,7 @@
 
 from numbers import Number as numeric_type
 
+from yt.utilities.lib.cykdtree import PyKDTree
 from yt.funcs import \
     iterable, \
     ensure_list, \
@@ -58,6 +59,9 @@
 from yt.utilities.logger import ytLogger as mylog
 from yt.utilities.lib.misc_utilities import \
     get_box_grids_level
+from yt.utilities.lib.particle_kdtree_tools import \
+    generate_smoothing_length, \
+    estimate_density
 from yt.geometry.grid_container import \
     GridTree, \
     MatchPointsToGrids
@@ -1069,6 +1073,49 @@ def _setup_data_io(self):
         else:
             self.io = io_registry[self.dataset_type](self.ds)
 
+    def update_data(self, data):
+        """
+        Update the stream data with a new data dict. If fields already exist,
+        they will be replaced, but if they do not, they will be added. Fields
+        already in the stream but not part of the data dict will be left
+        alone.
+        """
+        # Alias
+        ds = self.ds
+        handler = ds.stream_handler
+
+        # Preprocess
+        field_units, data, _ = process_data(data)
+        pdata = {}
+        for key in data.keys():
+            if not isinstance(key, tuple):
+                field = ("io", key)
+                mylog.debug("Reassigning '%s' to '%s'", key, field)
+            else:
+                field = key
+            pdata[field] = data[key]
+        data = pdata # Drop reference count
+        particle_types = set_particle_types(data)
+
+        # Update particle types
+        handler.particle_types.update(particle_types)
+        ds._find_particle_types()
+
+        # Update fields
+        handler.field_units.update(field_units)
+        fields = handler.fields
+        for field in data.keys():
+            if field not in fields._additional_fields:
+                fields._additional_fields += (field,)
+        fields["stream_file"].update(data)
+
+        # Update field list
+        for field in self.ds.field_list:
+            if field[0] in ["all", "nbody"]:
+                self.ds.field_list.remove(field)
+        self._detect_output_fields()
+        self.ds.create_field_info()
+
 class StreamParticleFile(ParticleFile):
     pass
 
@@ -1092,11 +1139,90 @@ def __init__(self, stream_handler, storage_filename=None,
         if ('io', 'density') in fields and ('io', 'smoothing_length') in fields:
             self._sph_ptype = 'io'
 
-def load_particles(data, length_unit = None, bbox=None,
-                   sim_time=0.0, mass_unit = None, time_unit = None,
+    def add_sph_fields(self, n_neighbors=32, kernel="cubic", sph_ptype="io"):
+        """Add SPH fields for the specified particle type.
+
+        For a particle type with "particle_position" and "particle_mass" already
+        defined, this method adds the "smoothing_length" and "density" fields.
+        "smoothing_length" is computed as the distance to the nth nearest
+        neighbor. "density" is computed as the SPH (gather) smoothed mass. The
+        SPH fields are added only if they don't already exist.
+
+        Parameters
+        ----------
+        n_neighbors : int
+            The number of neighbors to use in smoothing length computation.
+        kernel : str
+            The kernel function to use in density estimation.
+        sph_ptype : str
+            The SPH particle type. Each dataset has one sph_ptype only. This
+            method will overwrite existing sph_ptype of the dataset.
+
+        """
+        mylog.info("Generating SPH fields")
+
+        # Unify units
+        l_unit = "code_length"
+        m_unit = "code_mass"
+        d_unit = "code_mass / code_length**3"
+
+        # Read basic fields
+        ad = self.all_data()
+        pos = ad[sph_ptype, "particle_position"].to(l_unit).d
+        mass = ad[sph_ptype, "particle_mass"].to(m_unit).d
+
+        # Construct k-d tree
+        kdtree = PyKDTree(
+            pos.astype("float64"),
+            left_edge=self.domain_left_edge.to_value(l_unit),
+            right_edge=self.domain_right_edge.to_value(l_unit),
+            periodic=self.periodicity,
+            leafsize=2*int(n_neighbors),
+        )
+        order = np.argsort(kdtree.idx)
+
+        def exists(fname):
+            if (sph_ptype, fname) in self.derived_field_list:
+                mylog.info("Field ('%s','%s') already exists. Skipping",
+                            sph_ptype, fname)
+                return True
+            else:
+                mylog.info("Generating field ('%s','%s')",
+                            sph_ptype, fname)
+                return False
+        data = {}
+
+        # Add smoothing length field
+        fname = "smoothing_length"
+        if not exists(fname):
+            hsml = generate_smoothing_length(
+                pos[kdtree.idx], kdtree, n_neighbors
+            )
+            hsml = hsml[order]
+            data[(sph_ptype, "smoothing_length")] = (hsml, l_unit)
+        else:
+            hsml = ad[sph_ptype, fname].to(l_unit).d
+
+        # Add density field
+        fname = "density"
+        if not exists(fname):
+            dens = estimate_density(
+                pos[kdtree.idx], mass[kdtree.idx], hsml[kdtree.idx],
+                kdtree, kernel_name=kernel,
+            )
+            dens = dens[order]
+            data[(sph_ptype, "density")] = (dens, d_unit)
+
+        # Add fields
+        self._sph_ptype = sph_ptype
+        self.index.update_data(data)
+
+def load_particles(data, length_unit=None, bbox=None,
+                   sim_time=None, mass_unit=None, time_unit=None,
                    velocity_unit=None, magnetic_unit=None,
                    periodicity=(True, True, True),
-                   geometry = "cartesian", unit_system="cgs"):
+                   geometry="cartesian", unit_system="cgs",
+                   data_source=None):
     r"""Load a set of particles into yt as a
     :class:`~yt.frontends.stream.data_structures.StreamParticleHandler`.
 
@@ -1116,8 +1242,8 @@ def load_particles(data, length_unit = None, bbox=None,
     Parameters
     ----------
     data : dict
-        This is a dict of numpy arrays or (numpy array, unit name) tuples, 
-        where the keys are the field names. Particles positions must be named 
+        This is a dict of numpy arrays or (numpy array, unit name) tuples,
+        where the keys are the field names. Particles positions must be named
         "particle_position_x", "particle_position_y", and "particle_position_z".
     length_unit : float
         Conversion factor from simulation length units to centimeters
@@ -1136,6 +1262,9 @@ def load_particles(data, length_unit = None, bbox=None,
     periodicity : tuple of booleans
         Determines whether the data will be treated as periodic along
         each axis
+    data_source : YTSelectionContainer, optional
+        If set, parameters like `bbox`, `sim_time`, and code units are derived
+        from it.
 
     Examples
     --------
@@ -1151,6 +1280,13 @@ def load_particles(data, length_unit = None, bbox=None,
 
     domain_dimensions = np.ones(3, "int32")
     nprocs = 1
+
+    # Parse bounding box
+    if data_source is not None:
+        le, re = data_source.get_bbox()
+        le = le.to_value("code_length")
+        re = re.to_value("code_length")
+        bbox = list(zip(le, re))
     if bbox is None:
         bbox = np.array([[0.0, 1.0], [0.0, 1.0], [0.0, 1.0]], 'float64')
     else:
@@ -1159,6 +1295,29 @@ def load_particles(data, length_unit = None, bbox=None,
     domain_right_edge = np.array(bbox[:, 1], 'float64')
     grid_levels = np.zeros(nprocs, dtype='int32').reshape((nprocs,1))
 
+    # Parse simulation time
+    if data_source is not None:
+        sim_time = data_source.ds.current_time
+    if sim_time is None:
+        sim_time = 0.0
+    else:
+        sim_time = float(sim_time)
+
+    # Parse units
+    def parse_unit(unit, dimension):
+        if unit is None:
+            unit = "code_" + dimension
+            if data_source is not None:
+                unit = data_source.ds.quan(1, unit).in_cgs()
+        return unit
+
+    length_unit = parse_unit(length_unit, "length")
+    mass_unit = parse_unit(mass_unit, "mass")
+    time_unit = parse_unit(time_unit, "time")
+    velocity_unit = parse_unit(velocity_unit, "velocity")
+    magnetic_unit = parse_unit(magnetic_unit, "magnetic")
+
+    # Preprocess data
     field_units, data, _ = process_data(data)
     sfh = StreamDictFieldHandler()
 
@@ -1178,17 +1337,6 @@ def load_particles(data, length_unit = None, bbox=None,
     grid_right_edges = domain_right_edge
     grid_dimensions = domain_dimensions.reshape(nprocs,3).astype("int32")
 
-    if length_unit is None:
-        length_unit = 'code_length'
-    if mass_unit is None:
-        mass_unit = 'code_mass'
-    if time_unit is None:
-        time_unit = 'code_time'
-    if velocity_unit is None:
-        velocity_unit = 'code_velocity'
-    if magnetic_unit is None:
-        magnetic_unit = 'code_magnetic'
-
     # I'm not sure we need any of this.
     handler = StreamHandler(
         grid_left_edges,


Test patch:
diff --git a/yt/frontends/stream/tests/test_stream_particles.py b/yt/frontends/stream/tests/test_stream_particles.py
--- a/yt/frontends/stream/tests/test_stream_particles.py
+++ b/yt/frontends/stream/tests/test_stream_particles.py
@@ -2,6 +2,7 @@
 
 from yt.testing import \
     assert_equal, \
+    fake_particle_ds, \
     fake_sph_orientation_ds
 from yt.frontends.stream.api import load_uniform_grid, \
     refine_amr, \
@@ -307,6 +308,45 @@ def test_load_particles_types():
         assert npart == num_tot_particles
         assert dd["all", "particle_position_%s" % ax].size == num_tot_particles
 
+def test_load_particles_with_data_source():
+    ds1 = fake_particle_ds()
+
+    # Load from dataset
+    ad = ds1.all_data()
+    fields = ['particle_mass']
+    fields += ['particle_position_{}'.format(ax) for ax in 'xyz']
+    data = {field: ad[field] for field in fields}
+    ds2 = load_particles(data, data_source=ad)
+
+    def in_cgs(quan):
+        return quan.in_cgs().v
+
+    # Test bbox is parsed correctly
+    for attr in ['domain_left_edge', 'domain_right_edge']:
+        assert np.allclose(
+            in_cgs(getattr(ds1, attr)),
+            in_cgs(getattr(ds2, attr))
+        )
+
+    # Test sim_time is parsed correctly
+    assert in_cgs(ds1.current_time) == in_cgs(ds2.current_time)
+
+    # Test code units are parsed correctly
+    def get_cu(ds, dim):
+        return ds.quan(1, 'code_' + dim)
+    for dim in ['length', 'mass', 'time', 'velocity', 'magnetic']:
+        assert in_cgs(get_cu(ds1, dim)) == in_cgs(get_cu(ds2, dim))
+
+def test_add_sph_fields():
+    ds = fake_particle_ds()
+    ds.index
+    assert set(ds.particle_types) == {'io', 'all', 'nbody'}
+
+    ds.add_sph_fields()
+    assert set(ds.particle_types) == {'io', 'all'}
+    assert ('io', 'smoothing_length') in ds.field_list
+    assert ('io', 'density') in ds.field_list
+
 def test_particles_outside_domain():
     np.random.seed(0x4d3d3d3)
     posx_arr = np.random.uniform(low=-1.6, high=1.5, size=1000)
@@ -332,4 +372,3 @@ def test_stream_sph_projection():
     image = frb['gas', 'density']
     assert image.max() > 0
     assert image.shape == (256, 256)
-    
diff --git a/yt/frontends/stream/tests/test_update_data.py b/yt/frontends/stream/tests/test_update_data.py
--- a/yt/frontends/stream/tests/test_update_data.py
+++ b/yt/frontends/stream/tests/test_update_data.py
@@ -1,12 +1,14 @@
-from yt.testing import fake_random_ds
+import numpy as np
+from yt.testing import \
+    fake_particle_ds, \
+    fake_random_ds
 from yt.data_objects.profiles import create_profile
-from numpy.random import uniform
 
-def test_update_data():
+def test_update_data_grid():
     ds = fake_random_ds(64, nprocs=8)
     ds.index
     dims = (32,32,32)
-    grid_data = [{"temperature": uniform(size=dims)}
+    grid_data = [{"temperature": np.random.uniform(size=dims)}
                  for i in range(ds.index.num_grids)]
     ds.index.update_data(grid_data)
     prj = ds.proj("temperature", 2)
@@ -14,3 +16,12 @@ def test_update_data():
     dd = ds.all_data()
     profile = create_profile(dd, "density", "temperature", 10)
     profile["temperature"]
+
+def test_update_data_particle():
+    npart = 100
+    ds = fake_particle_ds(npart=npart)
+    part_data = {"temperature": np.random.rand(npart)}
+    ds.index.update_data(part_data)
+    assert ("io", "temperature") in ds.field_list
+    dd = ds.all_data()
+    dd["temperature"]