Repository: obspy/obspy

Issue:
Bug in hypodd PHA read
HypoDD PHA read does not read a catalog file that I downloaded from the OOI website: http://axial.ocean.washington.edu/#name6 because it could not handle "NaN" as a magnitude
nor seconds outside the range of 0..59.  Neither of these are explicitly allowed (or disallowed) 
in the HypoDD docs I found, but the file is of general interest and the solution simple.  Here is
the code that reproduces the error, using the files attached in [error_example.zip](https://github.com/obspy/obspy/files/4756520/error_example.zip):

```python
#!/bin/env python3
from obspy.core.inventory.inventory import read_inventory
from obspy.core.event.catalog import read_events

inv = read_inventory('OO_SP_stationonly.xml')
cat = read_events('ph2dt_example_errors.dat', 'HYPODDPHA', inventory=inv,
                  id_default='OO.{}.00.EH{}')
print(cat)
```
which results in:
```
Traceback (most recent call last):
  File "error_example.py", line 7, in <module>
    id_default='OO.{}.00.EH{}')
  File "<decorator-gen-46>", line 2, in read_events
  File "/Users/crawford/anaconda3/envs/obspy/lib/python3.6/site-packages/obspy/core/util/decorator.py", line 300, in _map_example_filename
    return func(*args, **kwargs)
  File "/Users/crawford/anaconda3/envs/obspy/lib/python3.6/site-packages/obspy/core/event/catalog.py", line 812, in read_events
    return _generic_reader(pathname_or_url, _read, format=format, **kwargs)
  File "/Users/crawford/anaconda3/envs/obspy/lib/python3.6/site-packages/obspy/core/util/base.py", line 701, in _generic_reader
    generic = callback_func(pathnames[0], **kwargs)
  File "<decorator-gen-47>", line 2, in _read
  File "/Users/crawford/anaconda3/envs/obspy/lib/python3.6/site-packages/obspy/core/util/decorator.py", line 210, in uncompress_file
    result = func(filename, *args, **kwargs)
  File "/Users/crawford/anaconda3/envs/obspy/lib/python3.6/site-packages/obspy/core/event/catalog.py", line 821, in _read
    **kwargs)
  File "/Users/crawford/anaconda3/envs/obspy/lib/python3.6/site-packages/obspy/core/util/base.py", line 469, in _read_from_plugin
    list_obj = read_format(filename, **kwargs)
  File "/Users/crawford/anaconda3/envs/obspy/lib/python3.6/site-packages/obspy/io/hypodd/pha.py", line 112, in _read_pha
    for block in text.split('#')[1:]]
  File "/Users/crawford/anaconda3/envs/obspy/lib/python3.6/site-packages/obspy/io/hypodd/pha.py", line 112, in <listcomp>
    for block in text.split('#')[1:]]
  File "/Users/crawford/anaconda3/envs/obspy/lib/python3.6/site-packages/obspy/io/hypodd/pha.py", line 51, in _block2event
    magnitude = Magnitude(mag=mg, resource_id="smi:local/magnitude/" + id_)
  File "/Users/crawford/anaconda3/envs/obspy/lib/python3.6/site-packages/obspy/core/event/base.py", line 405, in __init__
    **kwargs)
  File "/Users/crawford/anaconda3/envs/obspy/lib/python3.6/site-packages/obspy/core/event/base.py", line 226, in __init__
    setattr(self, key, value)
  File "/Users/crawford/anaconda3/envs/obspy/lib/python3.6/site-packages/obspy/core/event/base.py", line 391, in __setattr__
    raise ValueError(msg)
ValueError: On Magnitude object: Value 'nan' for 'mag' is not a finite floating point value.
```
To make it work, I had to change the first lines in `io/hypodd/pha.py:_block2event()`
Original:
```python
    lines = block.strip().splitlines()
    yr, mo, dy, hr, mn, sc, la, lo, dp, mg, eh, ez, rms, id_ = lines[0].split()
    time = UTCDateTime(int(yr), int(mo), int(dy), int(hr), int(mn), float(sc))
```
Final:
```python
    lines = block.strip().splitlines()
    yr, mo, dy, hr, mn, sc, la, lo, dp, mg, eh, ez, rms, id_ = lines[0].split()
    if mg == 'NaN':
        mg = None
    time = UTCDateTime(int(yr), int(mo), int(dy), int(hr), int(mn)) + float(sc)
```
(there is probably a smarter "if" statment that covers other possible problems).

The code now returns:
```
18 Event(s) in Catalog:
2015-03-04T01:59:55.413000Z | +45.950, -129.986 | 0.0 None
2015-03-04T02:00:35.871000Z | +45.984, -129.993 | None None
...
2015-03-29T22:40:10.109000Z | +45.927, -129.997 | -0.0 None
2015-03-29T22:40:20.110000Z | +45.953, -129.983 | -0.3 None
To see all events call 'print(CatalogObject.__str__(print_all=True))'
```
If I change only  the lines with `mg` I get the following error:
```
Traceback (most recent call last):
  File "error_example.py", line 7, in <module>
    id_default='OO.{}.00.EH{}')
  File "<decorator-gen-46>", line 2, in read_events
  File "/Users/crawford/anaconda3/envs/obspy/lib/python3.6/site-packages/obspy/core/util/decorator.py", line 300, in _map_example_filename
    return func(*args, **kwargs)
  File "/Users/crawford/anaconda3/envs/obspy/lib/python3.6/site-packages/obspy/core/event/catalog.py", line 812, in read_events
    return _generic_reader(pathname_or_url, _read, format=format, **kwargs)
  File "/Users/crawford/anaconda3/envs/obspy/lib/python3.6/site-packages/obspy/core/util/base.py", line 701, in _generic_reader
    generic = callback_func(pathnames[0], **kwargs)
  File "<decorator-gen-47>", line 2, in _read
  File "/Users/crawford/anaconda3/envs/obspy/lib/python3.6/site-packages/obspy/core/util/decorator.py", line 210, in uncompress_file
    result = func(filename, *args, **kwargs)
  File "/Users/crawford/anaconda3/envs/obspy/lib/python3.6/site-packages/obspy/core/event/catalog.py", line 821, in _read
    **kwargs)
  File "/Users/crawford/anaconda3/envs/obspy/lib/python3.6/site-packages/obspy/core/util/base.py", line 469, in _read_from_plugin
    list_obj = read_format(filename, **kwargs)
  File "/Users/crawford/anaconda3/envs/obspy/lib/python3.6/site-packages/obspy/io/hypodd/pha.py", line 114, in _read_pha
    for block in text.split('#')[1:]]
  File "/Users/crawford/anaconda3/envs/obspy/lib/python3.6/site-packages/obspy/io/hypodd/pha.py", line 114, in <listcomp>
    for block in text.split('#')[1:]]
  File "/Users/crawford/anaconda3/envs/obspy/lib/python3.6/site-packages/obspy/io/hypodd/pha.py", line 31, in _block2event
    time = UTCDateTime(int(yr), int(mo), int(dy), int(hr), int(mn), float(sc))
  File "/Users/crawford/anaconda3/envs/obspy/lib/python3.6/site-packages/obspy/core/utcdatetime.py", line 466, in __init__
    dt = datetime.datetime(*args, **kwargs)
ValueError: second must be in 0..59
```
I have attached a version of `pha.py` with these modifications

obspy version = 1.2.1, installed using Anaconda


Discussion:
Thanks for raising this issue. I can create a PR to fix it. PHA support is quite new, if you stumble upon other issues, please drop a note.

By the way, I implemented the reader, but actually I didn't use HYPODD. Are these pha files with 60.0 as seconds created by HYPODD?
Pretty sure we saw some "60 seconds" files with some other file format before.. I think it was nonlinloc?

With regard to unexpected file content ("NaN" in a float field), I would encourage talking to the person "in charge" of the file format and finding out if this is widely used (so we should implement the workaround) or rather some community code making bad use of the file format (then the institution/code producing those files should be contacted to fix it upstream).
OK, I will talk to the author about the NaN.  NORDIC files have the 60-second problem, which is why I knew quickly how to "fix" it!

_______________________________
Wayne Crawford
CNRS Researcher
Institut de Physique du Globe de Paris
1, rue Jussieu
75238 Paris Cedex 05
+33 6 5151 1054

On 10 June 2020 at 10:33:57, Tobias Megies (notifications@github.com) wrote:

Pretty sure we saw some "60 seconds" files with some other file format before.. I think it was nonlinloc?

With regard to unexpected file content ("NaN" in a float field), I would encourage talking to the person "in charge" of the file format and finding out if this is widely used (so we should implement the workaround) or rather some community code making bad use of the file format (then the institution/code producing those files should be contacted to fix it upstream).

—
You are receiving this because you authored the thread.
Reply to this email directly, view it on GitHub, or unsubscribe.
Pretty sure, thats the flag we put into UTCDateTime to handle those 60s problems (and consorts):

Might be worth to reuse that.. dunno, doesnt really matter i guess

```python
 460         except ValueError:
 461             if not strict:
 462                 self._handle_overflow(*args, **kwargs)
 463             else:
 464                 raise
 465         else:
 466             self._from_datetime(dt)
 467 
 468     def _handle_overflow(self, year, month, day, hour=0, minute=0, second=0,
 469                          microsecond=0):
```

```
~/git/obspy/obspy｢master｣$ pygrep 'strict=False'
io/nlloc/core.py
191:    time = UTCDateTime(year, mon, day, hour, min, seconds, strict=False)
331:        t = UTCDateTime(*(ymd + hm), strict=False) + float(sec)
```
>Might be worth to reuse that.. dunno, doesnt really matter i guess

I am biased (because I implemented it) but using `UTCDateTime` with `strict` set to `False` is definitely the way to go. That way we don't have to re-implement the rollover logic in every parser that might need it.
Another comment, on the documentation.  It would be nice to have  couple of lines explaining how this function "chooses" between possible seed ids.  I had a long list of stations with many networks, some stations with several different band-instrument codes for the same orientation code, etc.  Looking at the documentation for "id_default" I was afraid that they were all going to be forced to the same network code and band/instrument code.  I was preparing an "id_map" listing them all, but it seems that your function did the right thing almost all of the time: choosing the faster-sampling and higher gain option wherever possible.  I don't even know if it used the id_default I supplied.  

Could you write in the top of the docstring that your code tries to find the appropriate seed code in the station file and, if there is ambiguity, it :
   1) looks at id_map for a direct station name match
  2) looks at id_default for a direct match to the other seed_id parameters, 
  3) makes the best choice it can based on ....
I assume this is the order!

Anyway, I'm impressed by what it selected, much better than what the docstring made me fear!
Yes, good idea. I think it works the following

1. look at id_map for a direct station name match and use the specified template
2. if 1 did not succeed, look if the station is present in inventory and use its first channel as template
3. if 1 and 2 did not succeed, use specified default template (id_default)

> but it seems that your function did the right thing almost all of the time: choosing the faster-sampling and higher gain option wherever possible

Probably that is, because these higher gains are specified before the other channels in your inventory file.

One case which is not covered: station changing SEED id within the period covered by the pha file.
I actually have no clue what this is on about, guess some looking up full SEED ID from an inventory based on a net/sta only info from that PHA file?
Anyway.. looks like you have the best idea about it @trichter so you in charge now here ;-P
Would be good if somebody could look at this ASAP for 1.2.2. see #2655. If we dont get volunteers it might have to be shelved for later
I can have a look. Still need feedback if NaN values for the magnitude should get accepted.
Feedback from somebody in particular? Ping anybody you need feedback from. without looking at details I would vote to completely omit a Magnitude node, if the value as a NaN. What's a magnitude worth if it has no magnitude? And pretty sure QuakeML does not allow writing a Magnitude without a valid number. Didn't check the QuakeML schema though. The other option would be setting a magnitude object that will cause a broken file whe noutputting I guess.. Just my 2 cents :thinking: 
I checked. QuakeML requires Magnitude to have a value and it must be a double. So we are talking something QuakeML-invalid with a NaN magnitude value..
@WayneCrawford wanted to talk with the author of the files in question.

Sure, I did not intent to actually set a NaN value for the magnitude, just if these files should get accepted (and not set the magnitude for the corresponding event). For me it des not matter much. I'd say why not? But on the other hand, if it is regarded as an invalid file, it will be trivial to make it valid for everyone.
I don't care too much, whatever you folks think is best :+1: 


Resolving patch:
diff --git a/obspy/core/util/base.py b/obspy/core/util/base.py
--- a/obspy/core/util/base.py
+++ b/obspy/core/util/base.py
@@ -39,14 +39,15 @@
 # defining ObsPy modules currently used by runtests and the path function
 DEFAULT_MODULES = ['clients.filesystem', 'core', 'db', 'geodetics', 'imaging',
                    'io.ah', 'io.arclink', 'io.ascii', 'io.cmtsolution',
-                   'io.cnv', 'io.css', 'io.dmx', 'io.focmec', 'io.iaspei',
-                   'io.gcf', 'io.gse2', 'io.json', 'io.kinemetrics', 'io.kml',
-                   'io.mseed', 'io.ndk', 'io.nied', 'io.nlloc', 'io.nordic',
-                   'io.pdas', 'io.pde', 'io.quakeml', 'io.reftek', 'io.rg16',
-                   'io.sac', 'io.scardec', 'io.seg2', 'io.segy', 'io.seisan',
-                   'io.sh', 'io.shapefile', 'io.seiscomp', 'io.stationtxt',
-                   'io.stationxml', 'io.wav', 'io.win', 'io.xseed', 'io.y',
-                   'io.zmap', 'realtime', 'scripts', 'signal', 'taup']
+                   'io.cnv', 'io.css', 'io.dmx', 'io.focmec', 'io.hypodd',
+                   'io.iaspei', 'io.gcf', 'io.gse2', 'io.json',
+                   'io.kinemetrics', 'io.kml', 'io.mseed', 'io.ndk', 'io.nied',
+                   'io.nlloc', 'io.nordic', 'io.pdas', 'io.pde', 'io.quakeml',
+                   'io.reftek', 'io.rg16', 'io.sac', 'io.scardec', 'io.seg2',
+                   'io.segy', 'io.seisan', 'io.sh', 'io.shapefile',
+                   'io.seiscomp', 'io.stationtxt', 'io.stationxml', 'io.wav',
+                   'io.win', 'io.xseed', 'io.y', 'io.zmap', 'realtime',
+                   'scripts', 'signal', 'taup']
 NETWORK_MODULES = ['clients.arclink', 'clients.earthworm', 'clients.fdsn',
                    'clients.iris', 'clients.neic', 'clients.nrl',
                    'clients.seedlink', 'clients.seishub', 'clients.syngine']
diff --git a/obspy/io/hypodd/pha.py b/obspy/io/hypodd/pha.py
--- a/obspy/io/hypodd/pha.py
+++ b/obspy/io/hypodd/pha.py
@@ -26,7 +26,8 @@ def _block2event(block, seed_map, id_default, ph2comp):
     """
     lines = block.strip().splitlines()
     yr, mo, dy, hr, mn, sc, la, lo, dp, mg, eh, ez, rms, id_ = lines[0].split()
-    time = UTCDateTime(int(yr), int(mo), int(dy), int(hr), int(mn), float(sc))
+    time = UTCDateTime(int(yr), int(mo), int(dy), int(hr), int(mn), float(sc),
+                       strict=False)
     picks = []
     arrivals = []
     for line in lines[1:]:
@@ -48,13 +49,19 @@ def _block2event(block, seed_map, id_default, ph2comp):
                     longitude=float(lo),
                     depth=1000 * float(dp),
                     time=time)
-    magnitude = Magnitude(mag=mg, resource_id="smi:local/magnitude/" + id_)
+    if mg.lower() == 'nan':
+        magnitudes = []
+        preferred_magnitude_id = None
+    else:
+        magnitude = Magnitude(mag=mg, resource_id="smi:local/magnitude/" + id_)
+        magnitudes = [magnitude]
+        preferred_magnitude_id = magnitude.resource_id
     event = Event(resource_id="smi:local/event/" + id_,
                   picks=picks,
                   origins=[origin],
-                  magnitudes=[magnitude],
+                  magnitudes=magnitudes,
                   preferred_origin_id=origin.resource_id,
-                  preferred_magnitude_id=magnitude.resource_id)
+                  preferred_magnitude_id=preferred_magnitude_id)
     return event
 
 
@@ -65,7 +72,8 @@ def _is_pha(filename):
         assert line.startswith(b'#')
         assert len(line.split()) == 15
         yr, mo, dy, hr, mn, sc = line.split()[1:7]
-        UTCDateTime(int(yr), int(mo), int(dy), int(hr), int(mn), float(sc))
+        UTCDateTime(int(yr), int(mo), int(dy), int(hr), int(mn), float(sc),
+                    strict=False)
     except Exception:
         return False
     else:
@@ -84,18 +92,24 @@ def _read_pha(filename, inventory=None, id_map=None, id_default='.{}..{}',
 
     The optional parameters all deal with the problem, that the PHA format
     only stores station names for the picks, but the Pick object expects
-    a SEED id.
+    a SEED id. A SEED id template is retrieved for each station by the
+    following procedure:
+
+    1. look at id_map for a direct station name match and use the specified
+       template
+    2. if 1 did not succeed, look if the station is present in inventory and
+       use its first channel as template
+    3. if 1 and 2 did not succeed, use specified default template (id_default)
 
     :param str filename: File or file-like object in text mode.
     :type inventory: :class:`~obspy.core.inventory.inventory.Inventory`
     :param inventory: Inventory used to retrieve network code, location code
         and channel code of stations (SEED id).
-    :param dict id_map: If channel information was not found in inventory,
-        it will be looked up in this dictionary
+    :param dict id_map: Default templates for each station
         (example: `id_map={'MOX': 'GR.{}..HH{}'`).
         The values must contain three dots and two `{}` which are
         substituted by station code and component.
-    :param str id_default: Default SEED id expression.
+    :param str id_default: Default SEED id template.
         The value must contain three dots and two `{}` which are
         substituted by station code and component.
     :param dict ph2comp: mapping of phases to components
diff --git a/obspy/io/sh/evt.py b/obspy/io/sh/evt.py
--- a/obspy/io/sh/evt.py
+++ b/obspy/io/sh/evt.py
@@ -150,19 +150,26 @@ def _read_evt(filename, inventory=None, id_map=None, id_default='.{}..{}',
         ObsPy :func:`~obspy.core.event.read_events` function, call this
         instead.
 
-    :type filename: str
-    :param filename: File or file-like object in text mode.
+    The optional parameters all deal with the problem, that the EVT format
+    only stores station names for the picks, but the Pick object expects
+    a SEED id. A SEED id template is retrieved for each station by the
+    following procedure:
+
+    1. look at id_map for a direct station name match and use the specified
+       template
+    2. if 1 did not succeed, look if the station is present in inventory and
+       use its first channel as template
+    3. if 1 and 2 did not succeed, use specified default template (id_default)
+
+    :param str filename: File or file-like object in text mode.
     :type inventory: :class:`~obspy.core.inventory.inventory.Inventory`
     :param inventory: Inventory used to retrieve network code, location code
         and channel code of stations (SEED id).
-    :type id_map: dict
-    :param id_map: If channel information was not found in inventory,
-        it will be looked up in this dictionary
+    :param dict id_map: Default templates for each station
         (example: `id_map={'MOX': 'GR.{}..HH{}'`).
         The values must contain three dots and two `{}` which are
         substituted by station code and component.
-    :type id_default: str
-    :param id_default: Default SEED id expression.
+    :param str id_default: Default SEED id template.
         The value must contain three dots and two `{}` which are
         substituted by station code and component.
     :param str encoding: encoding used (default: utf-8)


Test patch:
diff --git a/obspy/io/hypodd/tests/data/60s_nan.pha b/obspy/io/hypodd/tests/data/60s_nan.pha
new file mode 100644
--- /dev/null
+++ b/obspy/io/hypodd/tests/data/60s_nan.pha
@@ -0,0 +1,6 @@
+#  2025    5   14   14   35   60.000000   40.2254   10.4496   9.4080   3.50  0.0   0.0   0.0       20202331
+FUR   3.52199909  1.0000  P
+WET   5.86199909  1.0000  S
+#  2025    5   14   15   43    5.280000   40.2227   10.4497   9.7740   NaN   0.0   0.0   0.0       20202429
+UBR   2.46399997  1.0000  P
+WERD  4.16799997  1.0000  X
\ No newline at end of file
diff --git a/obspy/io/hypodd/tests/test_pha.py b/obspy/io/hypodd/tests/test_pha.py
--- a/obspy/io/hypodd/tests/test_pha.py
+++ b/obspy/io/hypodd/tests/test_pha.py
@@ -19,6 +19,7 @@ class PHATestCase(unittest.TestCase):
     def setUp(self):
         self.path = os.path.dirname(__file__)
         self.fname = os.path.join(self.path, 'data', 'example.pha')
+        self.fname2 = os.path.join(self.path, 'data', '60s_nan.pha')
 
     def test_is_pha(self):
         self.assertEqual(pha._is_pha(self.fname), True)
@@ -53,6 +54,20 @@ def test_read_pha(self):
             self.assertEqual(arr.phase, target[sta][2])
             self.assertEqual(arr.phase, pick.phase_hint)
 
+    def test_60s_nan(self):
+        """
+        issue 2627
+        """
+        cat = read_events(self.fname2)
+        event = cat[0]
+        self.assertEqual(len(event.origins), 1)
+        self.assertEqual(len(event.magnitudes), 1)
+        ori = event.preferred_origin()
+        self.assertEqual(str(ori.time), '2025-05-14T14:36:00.000000Z')
+        event = cat[1]
+        self.assertEqual(len(event.origins), 1)
+        self.assertEqual(len(event.magnitudes), 0)
+
     def test_populate_waveform_id(self):
         inv = read_inventory()
         with warnings.catch_warnings(record=True) as ws: