Repository: obspy/obspy

Issue:
Skip invalid unicode in MiniSEED headers
The Gecko data logger (https://gecko.kelunji.net/) seems to write invalid unicode into some miniseed headers [(Example file)](https://github.com/obspy/obspy/files/2116776/2018-05-29131100GECKO.ms.zip). The error message is attached at the end of the post.

My proposed solution would be to add the errors='ignore' argument to the decode()-calls in 
https://github.com/obspy/obspy/blob/master/obspy/io/mseed/util.py#L631
and following
```python
info["station"] = data[:5].strip().decode()
info["location"] = data[5:7].strip().decode()
info["channel"] = data[7:10].strip().decode()
info["network"] = data[10:12].strip().decode()
```

as well as
https://github.com/obspy/obspy/blob/master/obspy/io/mseed/core.py#L473



Python version: 2.7 or 3.5
Obspy version: 1.1.0 (installed via anaconda)


```
In [2]: st = obspy.read('2018-05-29131100GECKO.ms')
---------------------------------------------------------------------------
UnicodeDecodeError                        Traceback (most recent call last)
<ipython-input-2-27d0c93fb1f1> in <module>()
----> 1 st = obspy.read('2018-05-29131100GECKO.ms')

<decorator-gen-146> in read(pathname_or_url, format, headonly, starttime, endtime, nearest_sample, dtype, apply_calib, check_compression, **kwargs)

~/anaconda3/envs/Py35/lib/python3.5/site-packages/obspy/core/util/decorator.py in _map_example_filename(func, *args, **kwargs)
    299                         except IOError:
    300                             pass
--> 301         return func(*args, **kwargs)
    302     return _map_example_filename
    303 

~/anaconda3/envs/Py35/lib/python3.5/site-packages/obspy/core/stream.py in read(pathname_or_url, format, headonly, starttime, endtime, nearest_sample, dtype, apply_calib, check_compression, **kwargs)
    233         pathname = pathname_or_url
    234         for file in sorted(glob(pathname)):
--> 235             st.extend(_read(file, format, headonly, **kwargs).traces)
    236         if len(st) == 0:
    237             # try to give more specific information why the stream is empty

<decorator-gen-147> in _read(filename, format, headonly, **kwargs)

~/anaconda3/envs/Py35/lib/python3.5/site-packages/obspy/core/util/decorator.py in uncompress_file(func, filename, *args, **kwargs)
    209     else:
    210         # no compressions
--> 211         result = func(filename, *args, **kwargs)
    212     return result
    213 

~/anaconda3/envs/Py35/lib/python3.5/site-packages/obspy/core/stream.py in _read(filename, format, headonly, **kwargs)
    275     """
    276     stream, format = _read_from_plugin('waveform', filename, format=format,
--> 277                                        headonly=headonly, **kwargs)
    278     # set _format identifier for each element
    279     for trace in stream:

~/anaconda3/envs/Py35/lib/python3.5/site-packages/obspy/core/util/base.py in _read_from_plugin(plugin_type, filename, format, **kwargs)
    410         raise TypeError(msg % (format_ep.name, ', '.join(eps)))
    411     # read
--> 412     list_obj = read_format(filename, **kwargs)
    413     return list_obj, format_ep.name
    414 

~/anaconda3/envs/Py35/lib/python3.5/site-packages/obspy/io/mseed/core.py in _read_mseed(mseed_object, starttime, endtime, headonly, sourcename, reclen, details, header_byteorder, verbose, **kwargs)
    281         raise ObsPyMSEEDFilesizeTooLargeError(msg)
    282 
--> 283     info = util.get_record_information(mseed_object, endian=bo)
    284 
    285     # Map the encoding to a readable string value.

~/anaconda3/envs/Py35/lib/python3.5/site-packages/obspy/io/mseed/util.py in get_record_information(file_or_file_object, offset, endian)
    540     if isinstance(file_or_file_object, (str, native_str)):
    541         with open(file_or_file_object, 'rb') as f:
--> 542             info = _get_record_information(f, offset=offset, endian=endian)
    543     else:
    544         info = _get_record_information(file_or_file_object, offset=offset,

~/anaconda3/envs/Py35/lib/python3.5/site-packages/obspy/io/mseed/util.py in _get_record_information(file_object, offset, endian)
    628     data = file_object.read(12)
    629     info["station"] = data[:5].strip().decode()
--> 630     info["location"] = data[5:7].strip().decode()
    631     info["channel"] = data[7:10].strip().decode()
    632     info["network"] = data[10:12].strip().decode()

UnicodeDecodeError: 'utf-8' codec can't decode byte 0xf0 in position 0: invalid continuation byte
```







Discussion:
I'm fairly sure that only ASCII is allowed by SEED standard here which would mean it's a corrupt file by the standard.

We could ignore/replace invalid characters and show a warning, but maybe it would be better to introduce a `kwarg` to `read()` for this (could be used in different read plugins as well), so that a proper exception is raised for the user by default but it's still possible to read the corrupt data by then explicitly switching on the "ignore decoding errors" flag.

Other opinions, @krischer?
Also, I think we should explicitly specify the encoding in all `decode()` calls in that mseed `util.py`. I'm not feeling comfortable at all as it uses different default encodings for different people/setups (for me it's failing to decode as ASCII, as it was probably originally intended when implementing this)

Resolving patch:
diff --git a/obspy/io/mseed/core.py b/obspy/io/mseed/core.py
--- a/obspy/io/mseed/core.py
+++ b/obspy/io/mseed/core.py
@@ -473,7 +473,8 @@ def allocate_data(samplecount, sampletype):
             header['mseed'] = dict((k, v.decode())
                                    if isinstance(v, bytes) else (k, v)
                                    for k, v in header['mseed'].items())
-            header = dict((k, v.decode()) if isinstance(v, bytes) else (k, v)
+            header = dict((k, util._decode_header_field(k, v))
+                          if isinstance(v, bytes) else (k, v)
                           for k, v in header.items())
             trace = Trace(header=header, data=data)
             # Append global information.
diff --git a/obspy/io/mseed/scripts/recordanalyzer.py b/obspy/io/mseed/scripts/recordanalyzer.py
--- a/obspy/io/mseed/scripts/recordanalyzer.py
+++ b/obspy/io/mseed/scripts/recordanalyzer.py
@@ -175,17 +175,22 @@ def _read_fixed_header(self):
             raise
         # Write values to dictionary.
         self.fixed_header['Sequence number'] = \
-            int(''.join(x.decode('ascii') for x in header_item[:6]))
+            int(''.join(x.decode('ascii', errors="replace")
+                for x in header_item[:6]))
         self.fixed_header['Data header/quality indicator'] = \
-            header_item[6].decode('ascii')
+            header_item[6].decode('ascii', errors="replace")
         self.fixed_header['Station identifier code'] = \
-            ''.join(x.decode('ascii') for x in header_item[8:13]).strip()
+            ''.join(x.decode('ascii', errors="replace")
+                    for x in header_item[8:13]).strip()
         self.fixed_header['Location identifier'] = \
-            ''.join(x.decode('ascii') for x in header_item[13:15]).strip()
+            ''.join(x.decode('ascii', errors="replace")
+                    for x in header_item[13:15]).strip()
         self.fixed_header['Channel identifier'] = \
-            ''.join(x.decode('ascii') for x in header_item[15:18]).strip()
+            ''.join(x.decode('ascii', errors="replace")
+                    for x in header_item[15:18]).strip()
         self.fixed_header['Network code'] = \
-            ''.join(x.decode('ascii') for x in header_item[18:20]).strip()
+            ''.join(x.decode('ascii', errors="replace")
+                    for x in header_item[18:20]).strip()
         # Construct the starttime. This is only the starttime in the fixed
         # header without any offset. See page 31 of the SEED manual for the
         # time definition.
diff --git a/obspy/io/mseed/util.py b/obspy/io/mseed/util.py
--- a/obspy/io/mseed/util.py
+++ b/obspy/io/mseed/util.py
@@ -548,6 +548,28 @@ def get_record_information(file_or_file_object, offset=0, endian=None):
     return info
 
 
+def _decode_header_field(name, content):
+    """
+    Helper function to decode header fields. Fairly fault tolerant and it
+    will also raise nice warnings in case in encounters anything wild.
+    """
+    try:
+        return content.decode("ascii", errors="strict")
+    except UnicodeError:
+        r = content.decode("ascii", errors="ignore")
+        msg = (u"Failed to decode {name} code as ASCII. "
+               u"Code in file: '{result}' (\ufffd indicates characters "
+               u"that could not be decoded). "
+               u"Will be interpreted as: '{f_result}'. "
+               u"This is an invalid MiniSEED file - please "
+               u"contact your data provider.")
+        warnings.warn(msg.format(
+            name=name,
+            result=content.decode("ascii", errors="replace"),
+            f_result=r))
+        return r
+
+
 def _get_record_information(file_object, offset=0, endian=None):
     """
     Searches the first MiniSEED record stored in file_object at the current
@@ -628,10 +650,11 @@ def _get_record_information(file_object, offset=0, endian=None):
     # Jump to the network, station, location and channel codes.
     file_object.seek(record_start + 8, 0)
     data = file_object.read(12)
-    info["station"] = data[:5].strip().decode()
-    info["location"] = data[5:7].strip().decode()
-    info["channel"] = data[7:10].strip().decode()
-    info["network"] = data[10:12].strip().decode()
+
+    info["station"] = _decode_header_field("station", data[:5].strip())
+    info["location"] = _decode_header_field("location", data[5:7].strip())
+    info["channel"] = _decode_header_field("channel", data[7:10].strip())
+    info["network"] = _decode_header_field("network", data[10:12].strip())
 
     # Use the date to figure out the byte order.
     file_object.seek(record_start + 20, 0)


Test patch:
diff --git a/obspy/io/mseed/tests/data/gecko_non_ascii_header.ms b/obspy/io/mseed/tests/data/gecko_non_ascii_header.ms
new file mode 100644
Binary files /dev/null and b/obspy/io/mseed/tests/data/gecko_non_ascii_header.ms differ
diff --git a/obspy/io/mseed/tests/test_mseed_special_issues.py b/obspy/io/mseed/tests/test_mseed_special_issues.py
--- a/obspy/io/mseed/tests/test_mseed_special_issues.py
+++ b/obspy/io/mseed/tests/test_mseed_special_issues.py
@@ -1227,6 +1227,25 @@ def test_read_file_with_non_valid_blocks_in_between(self):
             del st[0].stats.mseed
             self.assertEqual(reference, st)
 
+    def test_reading_files_with_non_ascii_headers(self):
+        """
+        Some dataloggers appear to do this.
+
+        See #2177.
+        """
+        filename = os.path.join(self.path, "data",
+                                "gecko_non_ascii_header.ms")
+        with warnings.catch_warnings(record=True) as w:
+            warnings.simplefilter("always")
+            st = read(filename)
+        # Two identical messages will be raised (but in most common cases only
+        # the first will be shown).
+        self.assertEqual(len(w), 2)
+        self.assertTrue(w[0].message.args[0].startswith(
+            "Failed to decode location code as ASCII."))
+        self.assertEqual(len(st), 1)
+        self.assertEqual(st[0].id, '.GECKO.A.CNZ')
+
 
 def suite():
     return unittest.makeSuite(MSEEDSpecialIssueTestCase, 'test')