Repository: ARM-DOE/ACT

Issue:
HistogramDisplay, plot_stairstep_graph encounters ValueError for range
I'm attempting to generate histograms for some Aerosol Optical Depth data. It is rather sparse, and looks like this:

[nan, nan, 0.04041248, nan, nan, nan, nan, nan, nan, 0.08403611, nan, nan, nan, nan, nan, 0.08443181, 0.05368716, nan, 0.07251927, 0.05726737, nan, 0.04878159, nan, 0.06544831, nan, nan, nan, nan, 0.04621224, nan, nan]

Plotting like so:
```
    var_name = 'aod_be_polyfit_415'
    nbins = 12
    try:
        ts_display = act.plotting.HistogramDisplay(
            nc_data,
            figsize=(19,5.5),
            subplot_shape=(1,),
        )
    except Exception as e:
        print(f"{type(e)} error generating histogram of netCDF data for {var_name}:\n{e}")
    else:
        ts_display.plot_stairstep_graph(var_name, bins=nbins, subplot_index=(0,))
```

It encounters this error:
Traceback (most recent call last):
  File "plot_aodbe_daily_avgs.py", line 335, in <module>
    sys.exit(main())
  File "plot_aodbe_daily_avgs.py", line 320, in main
    ts_display.plot_stairstep_graph(var_name, bins=nbins, subplot_index=(0,))
  File "/home/<username>/.local/lib/python3.9/site-packages/act/plotting/histogramdisplay.py", line 386, in plot_stairstep_graph
    my_hist, bins = np.histogram(
  File "<__array_function__ internals>", line 180, in histogram
  File "/home/<username>/.local/lib/python3.9/site-packages/numpy/lib/histograms.py", line 793, in histogram
    bin_edges, uniform_bins = _get_bin_edges(a, bins, range, weights)
  File "/home/<username>/.local/lib/python3.9/site-packages/numpy/lib/histograms.py", line 426, in _get_bin_edges
    first_edge, last_edge = _get_outer_edges(a, range)
  File "/home/<username>/.local/lib/python3.9/site-packages/numpy/lib/histograms.py", line 323, in _get_outer_edges
    raise ValueError(
ValueError: autodetected range of [nan, nan] is not finite


Since np.histogram accepts a `range` parameter, I assume this error is because it attempts to auto-calculate the range, but the data contains nans and throws that off. I experimented and found if I replace nan with e.g. 1.0, the plotting works. I have to replace all nans.

I imagine there might be various ways to address this.
* Can this be corrected within ACT, so either A) ACT determines the range better than numpy does, ignoring the nans, or B) accepts a range parameter that it will pass to np.histogram?
* Is there a way you might recommend handling this in the data before attempting to plot a histogram? As I noted, I could replace all nans with other values (-9999), but then there'd need to be a way to mask out that value... and might as well just mask the nans.



Discussion:
@gg2 We might need to add a check for it, for now can you np.ma.masked_invalid your range? should just mask the nans then ignore them hopefully
If not then I might have to think of something else
Wow! That was fast. I'll try it.

Also, `bins` is not allowed to be `None`, or not provided as a parameter to plot_stairstep_graph() (because it then defaults to `None`.
I don't know how to apply the MaskedArray to the underlying Xarray.

```
var_data = nc_data[var_name]
print(type(var_data))
print(type(var_data.values)) # Is the ndarray type that MaskedArray is a variant of.
new_data = np.ma.masked_invalid(var_data.values, copy=True)
print(f"{type(new_data)}:\n{new_data}\n")
# Testing -- creates the same MaskedArray
#new_data = np.ma.masked_invalid(var_data, copy=True)
#print(f"{type(new_data)}:\n{new_data}\n")

data_mask = np.ones(len(var_data.values), dtype=np.int8) # Just all ones, to make an index array, since Xarray is silly about not allowing updates to single values, but also because I wasn't seeing a change with:
#nc_data[var_name].values = new_data # doesn't make a difference
print(f"{data_mask}")
nc_data[var_name].values[data_mask] = new_data

print(f"{var_name} ({type(nc_data[var_name].values)}):\n{nc_data[var_name].values}\n")

# The following successfully updated specific values in previous tests.
#data_nans = np.where(np.isnan(var_data.values))
#print(data_nans)
#var_data[data_nans] = 1.0
```
@gg2 do you have a sample file we can test with?
@gg2 one solution would be to drop the missing the values using Xarray's dropna() functionality. It would like something like this:

```python
ds_with_missing_values = act.io.read_netcdf("your_file.nc")

ds_with_missing_values = ds_with_missing_values.dropna("time")

ts_display = act.plotting.HistogramDisplay(
            ds_with_missing_values,
            figsize=(19,5.5),
            subplot_shape=(1,),
        )

ts_display.plot_stairstep_graph("aod_field", bins=12)
```

Resolving patch:
diff --git a/act/plotting/histogramdisplay.py b/act/plotting/histogramdisplay.py
--- a/act/plotting/histogramdisplay.py
+++ b/act/plotting/histogramdisplay.py
@@ -83,6 +83,11 @@ def set_yrng(self, yrng, subplot_index=(0,)):
         self.axes[subplot_index].set_ylim(yrng)
         self.yrng[subplot_index, :] = yrng
 
+    def _get_data(self, dsname, fields):
+        if isinstance(fields, str):
+            fields = [fields]
+        return self._obj[dsname][fields].dropna('time')
+
     def plot_stacked_bar_graph(
         self,
         field,
@@ -139,16 +144,17 @@ def plot_stacked_bar_graph(
         elif dsname is None:
             dsname = list(self._obj.keys())[0]
 
-        xdata = self._obj[dsname][field]
+        if sortby_field is not None:
+            ds = self._get_data(dsname, [field, sortby_field])
+            xdata, ydata = ds[field], ds[sortby_field]
+        else:
+            xdata = self._get_data(dsname, field)[field]
 
         if 'units' in xdata.attrs:
             xtitle = ''.join(['(', xdata.attrs['units'], ')'])
         else:
             xtitle = field
 
-        if sortby_field is not None:
-            ydata = self._obj[dsname][sortby_field]
-
         if bins is not None and sortby_bins is None and sortby_field is not None:
             # We will defaut the y direction to have the same # of bins as x
             sortby_bins = np.linspace(ydata.values.min(), ydata.values.max(), len(bins))
@@ -268,7 +274,7 @@ def plot_size_distribution(
         elif dsname is None:
             dsname = list(self._obj.keys())[0]
 
-        xdata = self._obj[dsname][field]
+        xdata = self._get_data(dsname, field)[field]
 
         if isinstance(bins, str):
             bins = self._obj[dsname][bins]
@@ -380,7 +386,7 @@ def plot_stairstep_graph(
         elif dsname is None:
             dsname = list(self._obj.keys())[0]
 
-        xdata = self._obj[dsname][field]
+        xdata = self._get_data(dsname, field)[field]
 
         if 'units' in xdata.attrs:
             xtitle = ''.join(['(', xdata.attrs['units'], ')'])
@@ -522,13 +528,13 @@ def plot_heatmap(
         elif dsname is None:
             dsname = list(self._obj.keys())[0]
 
-        xdata = self._obj[dsname][x_field]
+        ds = self._get_data(dsname, [x_field, y_field])
+        xdata, ydata = ds[x_field], ds[y_field]
 
         if 'units' in xdata.attrs:
             xtitle = ''.join(['(', xdata.attrs['units'], ')'])
         else:
             xtitle = x_field
-        ydata = self._obj[dsname][y_field]
 
         if x_bins is not None and y_bins is None:
             # We will defaut the y direction to have the same # of bins as x


Test patch:
diff --git a/act/tests/test_plotting.py b/act/tests/test_plotting.py
--- a/act/tests/test_plotting.py
+++ b/act/tests/test_plotting.py
@@ -171,13 +171,13 @@ def test_histogram_errors():
     mu = 50
     bins = np.linspace(0, 100, 50)
     ydata = 1 / (sigma * np.sqrt(2 * np.pi)) * np.exp(-((bins - mu) ** 2) / (2 * sigma**2))
-    y_array = xr.DataArray(ydata, dims={'bins': bins})
-    bins = xr.DataArray(bins, dims={'bins': bins})
-    my_fake_ds = xr.Dataset({'bins': bins, 'ydata': y_array})
+    y_array = xr.DataArray(ydata, dims={'time': bins})
+    bins = xr.DataArray(bins, dims={'time': bins})
+    my_fake_ds = xr.Dataset({'time': bins, 'ydata': y_array})
     histdisplay = HistogramDisplay(my_fake_ds)
     histdisplay.axes = None
     histdisplay.fig = None
-    histdisplay.plot_size_distribution('ydata', 'bins', set_title='Fake distribution.')
+    histdisplay.plot_size_distribution('ydata', 'time', set_title='Fake distribution.')
     assert histdisplay.fig is not None
     assert histdisplay.axes is not None
 
@@ -549,11 +549,11 @@ def test_size_distribution():
     mu = 50
     bins = np.linspace(0, 100, 50)
     ydata = 1 / (sigma * np.sqrt(2 * np.pi)) * np.exp(-((bins - mu) ** 2) / (2 * sigma**2))
-    y_array = xr.DataArray(ydata, dims={'bins': bins})
-    bins = xr.DataArray(bins, dims={'bins': bins})
-    my_fake_ds = xr.Dataset({'bins': bins, 'ydata': y_array})
+    y_array = xr.DataArray(ydata, dims={'time': bins})
+    bins = xr.DataArray(bins, dims={'time': bins})
+    my_fake_ds = xr.Dataset({'time': bins, 'ydata': y_array})
     histdisplay = HistogramDisplay(my_fake_ds)
-    histdisplay.plot_size_distribution('ydata', 'bins', set_title='Fake distribution.')
+    histdisplay.plot_size_distribution('ydata', 'time', set_title='Fake distribution.')
     try:
         return histdisplay.fig
     finally:
@@ -948,12 +948,13 @@ def test_time_plot2():
 @pytest.mark.mpl_image_compare(tolerance=30)
 def test_y_axis_flag_meanings():
     variable = 'detection_status'
-    obj = arm.read_netcdf(sample_files.EXAMPLE_CEIL1,
-                          keep_variables=[variable, 'lat', 'lon', 'alt'])
+    obj = arm.read_netcdf(
+        sample_files.EXAMPLE_CEIL1, keep_variables=[variable, 'lat', 'lon', 'alt']
+    )
     obj.clean.clean_arm_state_variables(variable, override_cf_flag=True)
 
     display = TimeSeriesDisplay(obj, figsize=(12, 8), subplot_shape=(1,))
-    display.plot(variable, subplot_index=(0, ), day_night_background=True, y_axis_flag_meanings=18)
+    display.plot(variable, subplot_index=(0,), day_night_background=True, y_axis_flag_meanings=18)
     display.fig.subplots_adjust(left=0.15, right=0.95, bottom=0.1, top=0.94)
 
     return display.fig
@@ -969,13 +970,12 @@ def test_colorbar_labels():
 
     y_axis_labels = {}
     flag_colors = ['white', 'green', 'blue', 'red', 'cyan', 'orange', 'yellow', 'black', 'gray']
-    for value, meaning, color in zip(obj[variable].attrs['flag_values'],
-                                     obj[variable].attrs['flag_meanings'],
-                                     flag_colors):
+    for value, meaning, color in zip(
+        obj[variable].attrs['flag_values'], obj[variable].attrs['flag_meanings'], flag_colors
+    ):
         y_axis_labels[value] = {'text': meaning, 'color': color}
 
-    display.plot(variable, subplot_index=(0, ), colorbar_labels=y_axis_labels,
-                 cbar_h_adjust=0)
+    display.plot(variable, subplot_index=(0,), colorbar_labels=y_axis_labels, cbar_h_adjust=0)
     display.fig.subplots_adjust(left=0.08, right=0.88, bottom=0.1, top=0.94)
 
     return display.fig