Repository: openradar/xradar

Issue:
TypeError: only integer scalar arrays can be converted to a scalar index
I'm currently working on a project where I'm working with raw LiDAR data that comes in .hpl format and I'm struggling to get xradar to read in the file. 

I used the following code adapted from the example here: https://docs.openradarscience.org/projects/xradar/en/latest/notebooks/HaloPhotonics.html

Code:
`hpl = 'User1_100_20240714_122137.hpl'
rhi = xd.io.open_hpl_datatree(hpl, backend_kwargs = dict(latitude = 42.7452, longitude = -86.0961, altitude = 213))`

Data file format:

Filename: User1_100_20240714_122137.hpl
System ID: 100
Number of gates: 833
Range gate length (m): 12.0
Gate length (pts): 4
Pulses/ray: 3000
No. of waypoints in file: 1
Scan type: User file 1 - csm
Focus range: 65535
Start time: 20240714 12:21:45.80
Resolution (m/s): 0.0382
Range of measurement (center of gate) = (range gate + 0.5) * Gate length
Data line 1: Decimal time (hours) Azimuth (degrees) Elevation (degrees) Pitch (degrees) Roll (degrees)
f9.6,1x,f6.2,1x,f6.2
Data line 2: Range Gate Doppler (m/s) Intensity (SNR + 1) Beta (m-1 sr-1) Spectral Width
i3,1x,f6.4,1x,f8.6,1x,e12.6,1x,f6.4 - repeat for no. gates
**** Instrument spectral width = 0.305763
12.36262778 90.00 -0.00 -0.46 -1.77
0 0.1529 1.339292 1.910213E-5 0.0382
1 -0.6115 1.239853 1.350899E-5 0.0382
2 -3.0576 1.074272 4.185253E-6 0.0382
3 -14.1033 1.056070 3.161522E-6 0.0382
4 2.4079 1.034895 1.969022E-6 0.0382
5 2.7519 1.015216 8.592881E-7 16.3583
6 3.0959 1.038732 2.189486E-6 18.2311
7 1.4524 1.035328 1.999168E-6 29.4297
8 2.5990 1.022900 1.297460E-6 13.1478
9 6.8797 1.036261 2.057073E-6 20.6773
10 1.6053 1.027125 1.540954E-6 19.6453

However, I keep getting this error and I'm not sure how to address it.  Any guidance on how to get beyond this hurdle? I'd sure appreciate it. 

---------------------------------------------------------------------------
KeyError                                  Traceback (most recent call last)
File /opt/anaconda3/envs/devel/lib/python3.10/site-packages/xarray/backends/file_manager.py:211, in CachingFileManager._acquire_with_cache_info(self, needs_lock)
    210 try:
--> 211     file = self._cache[self._key]
    212 except KeyError:

File /opt/anaconda3/envs/devel/lib/python3.10/site-packages/xarray/backends/lru_cache.py:56, in LRUCache.__getitem__(self, key)
     55 with self._lock:
---> 56     value = self._cache[key]
     57     self._cache.move_to_end(key)

KeyError: [<class 'xradar.io.backends.hpl.HplFile'>, ('/Volumes/LaCie/University of IL/ATMS 590 Individual Study/MITTEN-CI/Obs/July 14/071424_IOP9/20240714/User1_100_20240714_122137.hpl',), 'r', (), 'e02fb1ca-fe8a-47a2-a1d1-dfc8aeed83c5']

During handling of the above exception, another exception occurred:

TypeError                                 Traceback (most recent call last)
Cell In[48], line 2
      1 hpl = '/Volumes/LaCie/University of IL/ATMS 590 Individual Study/MITTEN-CI/Obs/July 14/071424_IOP9/20240714/User1_100_20240714_122137.hpl'
----> 2 rhi = xd.io.open_hpl_datatree(hpl, backend_kwargs = dict(latitude = 42.7452, longitude = -86.0961, altitude = 213))

File /opt/anaconda3/envs/devel/lib/python3.10/site-packages/xradar/io/backends/hpl.py:638, in open_hpl_datatree(filename_or_obj, **kwargs)
    636         sweeps.extend(sweep)
    637 else:
--> 638     sweeps = _get_h5group_names(filename_or_obj)
    640 ls_ds: list[xr.Dataset] = [
    641     xr.open_dataset(filename_or_obj, group=swp, engine="hpl", **kwargs)
    642     for swp in sweeps
    643 ]
    645 dtree: dict = {
    646     "/": _get_required_root_dataset(ls_ds, optional=optional).rename(
    647         {"sweep_fixed_angle": "fixed_angle"}
   (...)
    653     "/radar_calibration": _get_radar_calibration(ls_ds, radar_calibration_subgroup),
    654 }

File /opt/anaconda3/envs/devel/lib/python3.10/site-packages/xradar/io/backends/hpl.py:584, in _get_h5group_names(filename_or_obj)
    583 def _get_h5group_names(filename_or_obj):
--> 584     store = HplStore.open(filename_or_obj)
    585     return [f"sweep_{i}" for i in store.root.data["sweep_number"]]

File /opt/anaconda3/envs/devel/lib/python3.10/site-packages/xradar/io/backends/hpl.py:440, in HplStore.open(cls, filename, mode, group, **kwargs)
    437 @classmethod
    438 def open(cls, filename, mode="r", group=None, **kwargs):
    439     manager = CachingFileManager(HplFile, filename, mode=mode, kwargs=kwargs)
--> 440     return cls(manager, group=group)

File /opt/anaconda3/envs/devel/lib/python3.10/site-packages/xradar/io/backends/hpl.py:435, in HplStore.__init__(self, manager, group)
    433 self._manager = manager
    434 self._group = group
--> 435 self._filename = self.filename

File /opt/anaconda3/envs/devel/lib/python3.10/site-packages/xradar/io/backends/hpl.py:444, in HplStore.filename(self)
    442 @property
    443 def filename(self):
--> 444     with self._manager.acquire_context(False) as root:
    445         return root.filename

File /opt/anaconda3/envs/devel/lib/python3.10/contextlib.py:135, in _GeneratorContextManager.__enter__(self)
    133 del self.args, self.kwds, self.func
    134 try:
--> 135     return next(self.gen)
    136 except StopIteration:
    137     raise RuntimeError("generator didn't yield") from None

File /opt/anaconda3/envs/devel/lib/python3.10/site-packages/xarray/backends/file_manager.py:199, in CachingFileManager.acquire_context(self, needs_lock)
    196 @contextlib.contextmanager
    197 def acquire_context(self, needs_lock=True):
    198     """Context manager for acquiring a file."""
--> 199     file, cached = self._acquire_with_cache_info(needs_lock)
    200     try:
    201         yield file

File /opt/anaconda3/envs/devel/lib/python3.10/site-packages/xarray/backends/file_manager.py:217, in CachingFileManager._acquire_with_cache_info(self, needs_lock)
    215     kwargs = kwargs.copy()
    216     kwargs["mode"] = self._mode
--> 217 file = self._opener(*self._args, **kwargs)
    218 if self._mode == "w":
    219     # ensure file doesn't get overridden when opened again
    220     self._mode = "a"

File /opt/anaconda3/envs/devel/lib/python3.10/site-packages/xradar/io/backends/hpl.py:326, in HplFile.__init__(self, filename, **kwargs)
    319 data_unsorted["azimuth"] = np.where(
    320     data_unsorted["azimuth"] < 360.0,
    321     data_unsorted["azimuth"],
    322     data_unsorted["azimuth"] - 360.0,
    323 )
    324 if data_unsorted["sweep_mode"] == "rhi":
    325     data_unsorted["fixed_angle"] = np.unique(
--> 326         data_unsorted["azimuth"].data[
    327             np.argwhere(np.abs(diff_azimuth) <= transition_threshold_azi) + 1
    328         ]
    329     )
    330 elif (
    331     data_unsorted["sweep_mode"] == "azimuth_surveillance"
    332     or data_unsorted["sweep_mode"] == "vertical_pointing"
    333 ):
    334     data_unsorted["fixed_angle"] = np.array(unique_elevations)

TypeError: only integer scalar arrays can be converted to a scalar index

Discussion:
review-notebook-app[bot]: Check out this pull request on&nbsp; <a href="https://app.reviewnb.com/openradar/xradar/pull/250"><img align="absmiddle"  alt="ReviewNB" height="28" class="BotMessageButtonImage" src="https://raw.githubusercontent.com/ReviewNB/support/master/images/button_reviewnb.png"/></a> 

 See visual diffs & provide feedback on Jupyter Notebooks. 

---

 <i>Powered by <a href='https://www.reviewnb.com/?utm_source=gh'>ReviewNB</a></i>
codecov[bot]: ## [Codecov](https://app.codecov.io/gh/openradar/xradar/pull/250?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=openradar) Report
Attention: Patch coverage is `55.55556%` with `4 lines` in your changes missing coverage. Please review.
> Project coverage is 93.44%. Comparing base [(`b4684fa`)](https://app.codecov.io/gh/openradar/xradar/commit/b4684faca3f9b91717571e230f47f38edcde12d9?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=openradar) to head [(`6c028ae`)](https://app.codecov.io/gh/openradar/xradar/commit/6c028ae207600ddcfca47b5bc5f56eab0c885648?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=openradar).

| [Files with missing lines](https://app.codecov.io/gh/openradar/xradar/pull/250?dropdown=coverage&src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=openradar) | Patch % | Lines |
|---|---|---|
| [xradar/io/backends/hpl.py](https://app.codecov.io/gh/openradar/xradar/pull/250?src=pr&el=tree&filepath=xradar%2Fio%2Fbackends%2Fhpl.py&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=openradar#diff-eHJhZGFyL2lvL2JhY2tlbmRzL2hwbC5weQ==) | 55.55% | [4 Missing :warning: ](https://app.codecov.io/gh/openradar/xradar/pull/250?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=openradar) |

<details><summary>Additional details and impacted files</summary>


```diff
@@            Coverage Diff             @@
##             main     #250      +/-   ##
==========================================
- Coverage   93.49%   93.44%   -0.06%     
==========================================
  Files          26       26              
  Lines        5028     5032       +4     
==========================================
+ Hits         4701     4702       +1     
- Misses        327      330       +3     
```

| [Flag](https://app.codecov.io/gh/openradar/xradar/pull/250/flags?src=pr&el=flags&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=openradar) | Coverage Δ | |
|---|---|---|
| [notebooktests](https://app.codecov.io/gh/openradar/xradar/pull/250/flags?src=pr&el=flag&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=openradar) | `78.97% <55.55%> (-0.05%)` | :arrow_down: |
| [unittests](https://app.codecov.io/gh/openradar/xradar/pull/250/flags?src=pr&el=flag&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=openradar) | `93.02% <55.55%> (-0.06%)` | :arrow_down: |

Flags with carried forward coverage won't be shown. [Click here](https://docs.codecov.io/docs/carryforward-flags?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=openradar#carryforward-flags-in-the-pull-request-comment) to find out more.

</details>

[:umbrella: View full report in Codecov by Sentry](https://app.codecov.io/gh/openradar/xradar/pull/250?dropdown=coverage&src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=openradar).   
:loudspeaker: Have feedback on the report? [Share it here](https://about.codecov.io/codecov-pr-comment-feedback/?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=openradar).

rcjackson: > One slight suggestion - can you add a quick test here to hit those thresholds?

I've added a PR to open-radar-data for a Halo Photonics RHI file to make the test. Once that PR is approved and open-radar-data is updated, I can add a test for the RHI file.
zssherman: @rcjackson I approved the PR and got a release going so the open-radar-data repo once it releases on conda-forge should be good to go.
rcjackson: > @rcjackson I approved the PR and got a release going so the open-radar-data repo once it releases on conda-forge should be good to go.

Ok it looks like we're just waiting for the release to conda-forge to get the tests passing. I added 2 tests for reading from an IOBase and from an RHI file, so that should improve the metrics.

Patch:
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 6223eb4..26d306a 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -157,7 +157,7 @@ jobs:
             h5py
             lat_lon_parser
             netCDF4
-            open-radar-data>=0.3.0
+            open-radar-data>=0.4.2
             packaging
             pandas
             pip
diff --git a/docs/history.md b/docs/history.md
index a6e1761..802eda2 100644
--- a/docs/history.md
+++ b/docs/history.md
@@ -5,6 +5,7 @@
 * ENH: Adding test to `open_datatree` function for all backends. Adding "scan_name" to nexradlevel2 datatree attributes ({pull}`238`) by [@aladinor](https://github.com/aladinor)
 * FIX: Improving performance of `open_nexradlevel2_datatree` function and adding tests for `sweep` parameter. ({issue}`239`) ({pull}`240`) by [@aladinor](https://github.com/aladinor)
 * FIX: Keeping attributes at each variable when using `open_nexradlevel2_datatree`. ({issue}`241`) ({pull}`242`) by [@aladinor](https://github.com/aladinor)
+* FIX: Correctly read transition rays in RHI scans ({issue}`247`) ({pull}`250`) by [@rcjackson](https://github.com/rcjackson)
 
 ## 0.8.0 (2024-11-04)
 
diff --git a/examples/notebooks/HaloPhotonics.ipynb b/examples/notebooks/HaloPhotonics.ipynb
index c508f19..f7d9c2b 100644
--- a/examples/notebooks/HaloPhotonics.ipynb
+++ b/examples/notebooks/HaloPhotonics.ipynb
@@ -67,7 +67,7 @@
    "source": [
     "fig, ax = plt.subplots(3, 3, figsize=(12, 10))\n",
     "for sweep in range(9):\n",
-    "    sweep_ds = xd.georeference.get_x_y_z(ds[\"sweep_%d\" % sweep].ds)\n",
+    "    sweep_ds = xd.georeference.get_x_y_z(ds[f\"sweep_{sweep:d}\"].ds)\n",
     "    sweep_ds = sweep_ds.set_coords([\"x\", \"y\", \"z\", \"time\", \"range\"])\n",
     "    sweep_ds[\"mean_doppler_velocity\"].plot(\n",
     "        x=\"x\", y=\"y\", ax=ax[int(sweep / 3), sweep % 3]\n",
diff --git a/xradar/io/backends/hpl.py b/xradar/io/backends/hpl.py
index 4197a00..958958a 100644
--- a/xradar/io/backends/hpl.py
+++ b/xradar/io/backends/hpl.py
@@ -322,11 +322,13 @@ class HplFile:
             data_unsorted["azimuth"] - 360.0,
         )
         if data_unsorted["sweep_mode"] == "rhi":
-            data_unsorted["fixed_angle"] = np.unique(
-                data_unsorted["azimuth"].data[
-                    np.argwhere(np.abs(diff_azimuth) <= transition_threshold_azi) + 1
-                ]
+            non_transitions = (
+                np.argwhere(np.abs(diff_azimuth) <= transition_threshold_azi) + 1
             )
+            unique_elevations = np.unique(
+                data_unsorted["azimuth"][np.squeeze(non_transitions)]
+            )
+            data_unsorted["fixed_angle"] = unique_elevations
         elif (
             data_unsorted["sweep_mode"] == "azimuth_surveillance"
             or data_unsorted["sweep_mode"] == "vertical_pointing"
@@ -343,8 +345,11 @@ class HplFile:
         )
         start_indicies = []
         end_indicies = []
-        for i, t in enumerate(unique_elevations):
-            where_in_sweep = np.argwhere(data_unsorted["elevation"] == t)
+        for i, t in enumerate(data_unsorted["fixed_angle"]):
+            if data_unsorted["sweep_mode"] == b"rhi":
+                where_in_sweep = np.argwhere(data_unsorted["azimuth"] == t)
+            else:
+                where_in_sweep = np.argwhere(data_unsorted["elevation"] == t)
             start_indicies.append(int(where_in_sweep.min()))
             end_indicies.append(int(where_in_sweep.max()))
         end_indicies = np.array(end_indicies)
@@ -353,7 +358,7 @@ class HplFile:
         data_unsorted["sweep_end_ray_index"] = np.array(end_indicies)
         data_unsorted["antenna_transition"] = transitions
         self._data = OrderedDict()
-        for i in range(len(unique_elevations)):
+        for i in range(len(data_unsorted["fixed_angle"])):
             sweep_dict = OrderedDict()
             time_inds = slice(
                 data_unsorted["sweep_start_ray_index"][i],
@@ -373,7 +378,7 @@ class HplFile:
                     sweep_dict[k] = data_unsorted[k][time_inds]
                 else:
                     sweep_dict[k] = data_unsorted[k]
-            self._data["sweep_%d" % i] = sweep_dict
+            self._data[f"sweep_{i:d}"] = sweep_dict
         self._data["sweep_number"] = data_unsorted["sweep_number"]
         self._data["fixed_angle"] = data_unsorted["fixed_angle"]
 
@@ -531,6 +536,8 @@ class HPLBackendEntrypoint(BackendEntrypoint):
         latitude=0,
         longitude=0,
         altitude=0,
+        transition_threshold_azi=0.05,
+        transition_threshold_el=0.001,
     ):
         store_entrypoint = StoreBackendEntrypoint()
 
@@ -544,6 +551,8 @@ class HPLBackendEntrypoint(BackendEntrypoint):
             latitude=latitude,
             longitude=longitude,
             altitude=altitude,
+            transition_threshold_azi=transition_threshold_azi,
+            transition_threshold_el=transition_threshold_el,
         )
 
         ds = store_entrypoint.open_dataset(


Test patch:
diff --git a/ci/notebooktests.yml b/ci/notebooktests.yml
index c54f6c3..018057f 100644
--- a/ci/notebooktests.yml
+++ b/ci/notebooktests.yml
@@ -16,7 +16,7 @@ dependencies:
   - netCDF4
   - notebook
   - numpy
-  - open-radar-data>=0.3.0
+  - open-radar-data>=0.4.2
   - pip
   - pyproj
   - pytest
diff --git a/ci/unittests.yml b/ci/unittests.yml
index 0620223..3806f35 100644
--- a/ci/unittests.yml
+++ b/ci/unittests.yml
@@ -13,7 +13,7 @@ dependencies:
   - lat_lon_parser
   - netCDF4
   - numpy
-  - open-radar-data>=0.3.0
+  - open-radar-data>=0.4.2
   - pip
   - pyproj
   - pytest
diff --git a/examples/notebooks/conftest.py b/examples/notebooks/conftest.py
index 0e4369c..aa6d2f8 100644
--- a/examples/notebooks/conftest.py
+++ b/examples/notebooks/conftest.py
@@ -30,7 +30,7 @@ class NotebookFile(pytest.File):
             yield NotebookItem.from_parent(self, name=os.path.basename(f))
 
     def setup(self):
-        kernel = "python%d" % sys.version_info[0]
+        kernel = "python%d" % sys.version_info[0]  # noqa
         self.exproc = ExecutePreprocessor(kernel_name=kernel, timeout=600)
 
 
diff --git a/tests/io/test_hpl.py b/tests/io/test_hpl.py
index e2ac8a4..7a102ad 100644
--- a/tests/io/test_hpl.py
+++ b/tests/io/test_hpl.py
@@ -26,6 +26,27 @@ def test_open_dataset_hpl():
     with xr.open_dataset(
         DATASETS.fetch("User1_184_20240601_013257.hpl"),
         engine="hpl",
+        backend_kwargs=dict(latitude=40, longitude=-70),
+    ) as ds:
+
+        assert ds["mean_doppler_velocity"].dims == ("azimuth", "range")
+        assert ds["mean_doppler_velocity"].max() == 19.5306
+
+
+def test_open_dataset_hpl_iobase():
+    with open(DATASETS.fetch("User1_184_20240601_013257.hpl"), "r") as fi:  # noqa
+        ds = xr.open_dataset(
+            fi, engine="hpl", backend_kwargs=dict(latitude=40, longitude=-70)
+        )
+
+        assert ds["mean_doppler_velocity"].dims == ("azimuth", "range")
+        assert ds["mean_doppler_velocity"].max() == 19.5306
+
+
+def test_open_rhi():
+    with xr.open_dataset(
+        DATASETS.fetch("User1_100_20240714_122137.hpl"),
+        engine="hpl",
         backend_kwargs=dict(latitude=41.24276244459537, longitude=-70.1070364814594),
     ) as ds: