Repository: obspy/obspy

Issue:
SEISAN Waveform import bug
It seems there is a bug when importing SEISAN waveform files. This can be checked when importing the test files in `obspy/seisan/tests` . Basicaly, the last sample of the data array usually has an unusually high value, resulting in a strange jerk/pulse in the waveform file. Because of this I usually just do

``` python
for el in st: el.data=el.data = el.data[:-1]
```

to delete the last sample...

I provided two waveform files, one in SEISAN, and another in MSEED which was converted from the SEISAN waveform using SEISAN/wavetool . However, I have tested the test waveforms already provided in the repository `obspy/seisan/tests/data` and the results is the same, altough the amplitude of the waveforms does obsfuscate this bug sometimes...



Discussion:

Resolving patch:
diff --git a/obspy/seisan/__init__.py b/obspy/seisan/__init__.py
--- a/obspy/seisan/__init__.py
+++ b/obspy/seisan/__init__.py
@@ -50,7 +50,7 @@
 The actual data is stored as numpy.ndarray in the data attribute of each trace.
 
 >>> print(st[0].data)
-[  492   519   542 ..., -6960 -6858 24000]
+[  464   492   519 ..., -7042 -6960 -6858]
 """
 from __future__ import (absolute_import, division, print_function,
                         unicode_literals)
diff --git a/obspy/seisan/core.py b/obspy/seisan/core.py
--- a/obspy/seisan/core.py
+++ b/obspy/seisan/core.py
@@ -14,6 +14,7 @@
 from future.utils import native_str
 
 import numpy as np
+import warnings
 
 from obspy import Stream, Trace, UTCDateTime
 from obspy.core import Stats
@@ -199,7 +200,11 @@ def _readline(fh, length=80):
                 dtype=dtype)
             # convert to system byte order
             data = np.require(data, stype)
-            stream.append(Trace(data=data[2:], header=header))
+            nbytes = (data.size - 2) * dtype.itemsize
+            if nbytes != data[0] or nbytes != data[-1]:
+                msg = "Mismatching byte size %d, %d, %d"
+                warnings.warn(msg % (nbytes, data[0], data[-1]))
+            stream.append(Trace(data=data[1:-1], header=header))
     fh.close()
     return stream
 


Test patch:
diff --git a/obspy/core/tests/test_waveform_plugins.py b/obspy/core/tests/test_waveform_plugins.py
--- a/obspy/core/tests/test_waveform_plugins.py
+++ b/obspy/core/tests/test_waveform_plugins.py
@@ -146,6 +146,10 @@ def test_isFormat(self):
         Tests all isFormat methods against all data test files from the other
         modules for false positives.
         """
+        KNOWN_FALSE = [
+            os.path.join('seisan', 'tests', 'data', 'SEISAN_Bug',
+                         '2011-09-06-1311-36S.A1032_001BH_Z_MSEED'),
+        ]
         formats_ep = _getEntryPoints('obspy.plugin.waveform', 'isFormat')
         formats = list(formats_ep.values())
         # Collect all false positives.
@@ -174,6 +178,8 @@ def test_isFormat(self):
                     filelist.extend([os.path.join(directory, _i) for _i in
                                      files])
                 for file in filelist:
+                    if any([n in file for n in KNOWN_FALSE]):
+                        continue
                     if isFormat(file) is True:  # pragma: no cover
                         false_positives.append((format.name, file))
         # Use try except to produce a meaningful error message.
diff --git a/obspy/seisan/tests/data/SEISAN_Bug/2011-09-06-1311-36S.A1032_001BH_Z b/obspy/seisan/tests/data/SEISAN_Bug/2011-09-06-1311-36S.A1032_001BH_Z
new file mode 100644
Binary files /dev/null and b/obspy/seisan/tests/data/SEISAN_Bug/2011-09-06-1311-36S.A1032_001BH_Z differ
diff --git a/obspy/seisan/tests/data/SEISAN_Bug/2011-09-06-1311-36S.A1032_001BH_Z_MSEED b/obspy/seisan/tests/data/SEISAN_Bug/2011-09-06-1311-36S.A1032_001BH_Z_MSEED
new file mode 100644
Binary files /dev/null and b/obspy/seisan/tests/data/SEISAN_Bug/2011-09-06-1311-36S.A1032_001BH_Z_MSEED differ
diff --git a/obspy/seisan/tests/test_core.py b/obspy/seisan/tests/test_core.py
--- a/obspy/seisan/tests/test_core.py
+++ b/obspy/seisan/tests/test_core.py
@@ -11,6 +11,7 @@
 
 import numpy as np
 
+from obspy.core import read
 from obspy.core.utcdatetime import UTCDateTime
 from obspy.seisan.core import _getVersion, isSEISAN, readSEISAN
 
@@ -73,14 +74,14 @@ def test_readSEISAN(self):
         # compare with ASCII values of trace
         # XXX: extracted ASCII file contains less values than the original
         # Seisan file!
-        self.assertEqual(list(st1[20].data[0:3665]),
+        self.assertEqual(list(st1[20].data[1:3666]),
                          np.loadtxt(datafile, dtype=np.int32).tolist())
         # 2 - little endian, 32 bit
         file = os.path.join(self.path, '2001-01-13-1742-24S.KONO__004')
         st2 = readSEISAN(file)
         st2.verify()
         self.assertEqual(len(st2), 4)
-        self.assertEqual(list(st2[0].data[0:3]), [492, 519, 542])
+        self.assertEqual(list(st2[0].data[1:4]), [492, 519, 542])
 
     def test_readSEISANHeadOnly(self):
         """
@@ -103,6 +104,18 @@ def test_readSEISANHeadOnly(self):
         self.assertAlmostEqual(st1[20].stats.delta, 0.0133, 4)
         self.assertEqual(list(st1[0].data), [])  # no data
 
+    def test_readSEISANVsReference(self):
+        """
+        Test for #970
+        """
+        _file = os.path.join(self.path, 'SEISAN_Bug',
+                             '2011-09-06-1311-36S.A1032_001BH_Z')
+        st = read(_file, format='SEISAN')
+        _file_ref = os.path.join(self.path, 'SEISAN_Bug',
+                                 '2011-09-06-1311-36S.A1032_001BH_Z_MSEED')
+        st_ref = read(_file_ref, format='MSEED')
+        self.assertTrue(np.allclose(st[0].data, st_ref[0].data))
+
 
 def suite():
     return unittest.makeSuite(CoreTestCase, 'test')