Repository: obspy/obspy

Issue:
ASCII SLIST uses only float32 
Is there any particular reason why in the obspy.io.ascii module there is assumed that any float written or read by it has to be float32? 

I'm currently working on a script that involves saving signal with response removed to an ASCII file and unfortunately the accuracy that is used by default in ObsPy is not high enough for me. 

BTW: Comparing sample to sample with same signal but saved and read to SAC gives me a 10883 errors from numpy.testing.assertion_almost_equal for a signal 12221 samples long.



Discussion:
nope - IRIS does not define it in the SLIST/TSPAIR documentation (http://ds.iris.edu/ds/nodes/dmc/data/formats/simple-ascii/) but looking at the current implementation of ```mseedtoascii.c``` (https://github.com/iris-edu/mseed2ascii/blob/f117564953ae6c1e0330483d3f176b334f841608/src/mseed2ascii.c) it seems that doubles are also supported

feel free to add a PR fixing this issue
Thanks for the green light. I will try to add possibility to manually define format of written floats in following days. 
Also, looks like scientific notation is acceptable, so the larger impact might actually be to switch to writing as `%g` (or with some custom accuracy if you will).

https://github.com/iris-edu/mseed2ascii/blob/f117564953ae6c1e0330483d3f176b334f841608/src/mseed2ascii.c#L312-L324

Resolving patch:
diff --git a/obspy/io/ascii/core.py b/obspy/io/ascii/core.py
--- a/obspy/io/ascii/core.py
+++ b/obspy/io/ascii/core.py
@@ -249,7 +249,8 @@ def _read_tspair(filename, headonly=False, **kwargs):  # @UnusedVariable
     return stream
 
 
-def _write_slist(stream, filename, **kwargs):  # @UnusedVariable
+def _write_slist(stream, filename, custom_fmt=None,
+                 **kwargs):  # @UnusedVariable
     """
     Writes a ASCII SLIST file.
 
@@ -262,6 +263,10 @@ def _write_slist(stream, filename, **kwargs):  # @UnusedVariable
     :param stream: The ObsPy Stream object to write.
     :type filename: str
     :param filename: Name of file to write.
+    :type custom_fmt: str
+    :param custom_fmt: formatter for writing sample values. Defaults to None.
+        Using this parameter will set ``TYPE`` value in header to ``CUSTOM``
+        and ObsPy will raise an exception while trying to read that file.
 
     .. rubric:: Example
 
@@ -325,9 +330,14 @@ def _write_slist(stream, filename, **kwargs):  # @UnusedVariable
                 fmt = '%d'
             elif trace.data.dtype.name.startswith('float'):
                 dtype = 'FLOAT'
-                fmt = '%f'
+                fmt = '%+.10e'
+
             else:
                 raise NotImplementedError
+            # fmt
+            if custom_fmt is not None:
+                dtype = _determine_dtype(custom_fmt)
+                fmt = custom_fmt
             # unit
             try:
                 unit = stats.ascii.unit
@@ -350,7 +360,8 @@ def _write_slist(stream, filename, **kwargs):  # @UnusedVariable
                          '\n').encode('ascii', 'strict'))
 
 
-def _write_tspair(stream, filename, **kwargs):  # @UnusedVariable
+def _write_tspair(stream, filename, custom_fmt=None,
+                  **kwargs):  # @UnusedVariable
     """
     Writes a ASCII TSPAIR file.
 
@@ -363,6 +374,10 @@ def _write_tspair(stream, filename, **kwargs):  # @UnusedVariable
     :param stream: The ObsPy Stream object to write.
     :type filename: str
     :param filename: Name of file to write.
+    :type custom_fmt: str
+    :param custom_fmt: formatter for writing sample values. Defaults to None.
+        Using this parameter will set ``TYPE`` value in header to ``CUSTOM``
+        and ObsPy will raise an exception while trying to read that file.
 
     .. rubric:: Example
 
@@ -433,9 +448,11 @@ def _write_tspair(stream, filename, **kwargs):  # @UnusedVariable
                 fmt = '%d'
             elif trace.data.dtype.name.startswith('float'):
                 dtype = 'FLOAT'
-                fmt = '%f'
-            else:
-                raise NotImplementedError
+                fmt = '%+.10e'
+            # fmt
+            if custom_fmt is not None:
+                dtype = _determine_dtype(custom_fmt)
+                fmt = custom_fmt
             # unit
             try:
                 unit = stats.ascii.unit
@@ -453,6 +470,29 @@ def _write_tspair(stream, filename, **kwargs):  # @UnusedVariable
                 fh.write(line.encode('ascii', 'strict'))
 
 
+def _determine_dtype(custom_fmt):
+    """
+    :type custom_fmt: str
+    :param custom_fmt: Python string formatter.
+    :rtype: str
+    :return: Datatype string for writing in header. Currently supported
+        are 'INTEGER', 'FLOAT' and `CUSTOM`.
+    :raises ValueError: if provided string is empty.
+    """
+    floats = ('e', 'f', 'g')
+    ints = ('d', 'i')
+    try:
+        if custom_fmt[-1].lower() in floats:
+            return 'FLOAT'
+        elif custom_fmt[-1].lower() in ints:
+            return 'INTEGER'
+        else:
+            return 'CUSTOM'
+    except IndexError:
+        raise ValueError('Provided string is not valid for determining ' +
+                         'datatype. Provide a proper Python string formatter')
+
+
 def _parse_data(data, data_type):
     """
     Simple function to read data contained in a StringIO object to a NumPy
@@ -467,7 +507,7 @@ def _parse_data(data, data_type):
     if data_type == "INTEGER":
         dtype = np.int_
     elif data_type == "FLOAT":
-        dtype = np.float32
+        dtype = np.float64
     else:
         raise NotImplementedError
     # Seek to the beginning of the StringIO.


Test patch:
diff --git a/obspy/io/ascii/tests/data/slist_float.ascii b/obspy/io/ascii/tests/data/slist_float.ascii
--- a/obspy/io/ascii/tests/data/slist_float.ascii
+++ b/obspy/io/ascii/tests/data/slist_float.ascii
@@ -1,3 +1,3 @@
 TIMESERIES XX_TEST__BHZ_R, 12 samples, 40 sps, 2008-01-15T00:00:00.025000, SLIST, FLOAT, Counts
-       185.01        181.02        185.03        189.04        194.05        205.06  
-       209.07        214.08        222.09        225.98        226.99        219.00  
++1.8500999450e+02	+1.8102000430e+02	+1.8502999880e+02	+1.8903999330e+02	+1.9405000310e+02	+2.0505999760e+02
++2.0907000730e+02	+2.1408000180e+02	+2.2208999630e+02	+2.2597999570e+02	+2.2699000550e+02	+2.1900000000e+02
diff --git a/obspy/io/ascii/tests/data/slist_float_custom_fmt.ascii b/obspy/io/ascii/tests/data/slist_float_custom_fmt.ascii
new file mode 100644
--- /dev/null
+++ b/obspy/io/ascii/tests/data/slist_float_custom_fmt.ascii
@@ -0,0 +1,3 @@
+TIMESERIES XX_TEST__BHZ_R, 12 samples, 40 sps, 2008-01-15T00:00:00.025000, SLIST, FLOAT, Counts
+185.00999450000000	181.02000430000001	185.02999879999999	189.03999329999999	194.05000310000000	205.05999760000000
+209.07000729999999	214.08000179999999	222.08999630000000	225.97999569999999	226.99000550000000	219.00000000000000
diff --git a/obspy/io/ascii/tests/data/tspair_float.ascii b/obspy/io/ascii/tests/data/tspair_float.ascii
--- a/obspy/io/ascii/tests/data/tspair_float.ascii
+++ b/obspy/io/ascii/tests/data/tspair_float.ascii
@@ -1,13 +1,13 @@
 TIMESERIES XX_TEST__BHZ_R, 12 samples, 40 sps, 2008-01-15T00:00:00.025000, TSPAIR, FLOAT, Counts
-2008-01-15T00:00:00.025000  185.01
-2008-01-15T00:00:00.050000  181.02
-2008-01-15T00:00:00.075000  185.03
-2008-01-15T00:00:00.100000  189.04
-2008-01-15T00:00:00.125000  194.05
-2008-01-15T00:00:00.150000  205.06
-2008-01-15T00:00:00.175000  209.07
-2008-01-15T00:00:00.200000  214.08
-2008-01-15T00:00:00.225000  222.09
-2008-01-15T00:00:00.250000  225.98
-2008-01-15T00:00:00.275000  226.99
-2008-01-15T00:00:00.300000  219.00
+2008-01-15T00:00:00.025000  +1.8500999450e+02
+2008-01-15T00:00:00.050000  +1.8102000430e+02
+2008-01-15T00:00:00.075000  +1.8502999880e+02
+2008-01-15T00:00:00.100000  +1.8903999330e+02
+2008-01-15T00:00:00.125000  +1.9405000310e+02
+2008-01-15T00:00:00.150000  +2.0505999760e+02
+2008-01-15T00:00:00.175000  +2.0907000730e+02
+2008-01-15T00:00:00.200000  +2.1408000180e+02
+2008-01-15T00:00:00.225000  +2.2208999630e+02
+2008-01-15T00:00:00.250000  +2.2597999570e+02
+2008-01-15T00:00:00.275000  +2.2699000550e+02
+2008-01-15T00:00:00.300000  +2.1900000000e+02
diff --git a/obspy/io/ascii/tests/data/tspair_float_custom_fmt.ascii b/obspy/io/ascii/tests/data/tspair_float_custom_fmt.ascii
new file mode 100644
--- /dev/null
+++ b/obspy/io/ascii/tests/data/tspair_float_custom_fmt.ascii
@@ -0,0 +1,13 @@
+TIMESERIES XX_TEST__BHZ_R, 12 samples, 40 sps, 2008-01-15T00:00:00.025000, TSPAIR, FLOAT, Counts
+2008-01-15T00:00:00.025000  185.00999450000000
+2008-01-15T00:00:00.050000  181.02000430000001
+2008-01-15T00:00:00.075000  185.02999879999999
+2008-01-15T00:00:00.100000  189.03999329999999
+2008-01-15T00:00:00.125000  194.05000310000000
+2008-01-15T00:00:00.150000  205.05999760000000
+2008-01-15T00:00:00.175000  209.07000729999999
+2008-01-15T00:00:00.200000  214.08000179999999
+2008-01-15T00:00:00.225000  222.08999630000000
+2008-01-15T00:00:00.250000  225.97999569999999
+2008-01-15T00:00:00.275000  226.99000550000000
+2008-01-15T00:00:00.300000  219.00000000000000
diff --git a/obspy/io/ascii/tests/test_ascii.py b/obspy/io/ascii/tests/test_ascii.py
--- a/obspy/io/ascii/tests/test_ascii.py
+++ b/obspy/io/ascii/tests/test_ascii.py
@@ -9,8 +9,9 @@
 import numpy as np
 
 from obspy import Trace, UTCDateTime, read
-from obspy.io.ascii.core import (_is_slist, _is_tspair, _read_slist,
-                                 _read_tspair, _write_slist, _write_tspair)
+from obspy.io.ascii.core import (_determine_dtype, _is_slist, _is_tspair,
+                                 _read_slist, _read_tspair, _write_slist,
+                                 _write_tspair)
 from obspy.core.util import NamedTemporaryFile
 
 
@@ -278,6 +279,63 @@ def test_write_tspair(self):
             tmpfile = tf.name
             # write
             _write_tspair(stream_orig, tmpfile)
+            # look at the raw data
+            with open(tmpfile, 'rt') as f:
+                lines = f.readlines()
+            self.assertEqual(
+                lines[0].strip(),
+                'TIMESERIES XX_TEST__BHZ_R, 12 samples, 40 sps, ' +
+                '2008-01-15T00:00:00.025000, TSPAIR, FLOAT, Counts')
+            self.assertEqual(
+                lines[1].strip(),
+                '2008-01-15T00:00:00.025000  +1.8500999450e+02')
+            # read again
+            stream = _read_tspair(tmpfile)
+            stream.verify()
+            self.assertEqual(stream[0].stats.network, 'XX')
+            self.assertEqual(stream[0].stats.station, 'TEST')
+            self.assertEqual(stream[0].stats.location, '')
+            self.assertEqual(stream[0].stats.channel, 'BHZ')
+            self.assertEqual(stream[0].stats.sampling_rate, 40.0)
+            self.assertEqual(stream[0].stats.npts, 12)
+            self.assertEqual(stream[0].stats.starttime,
+                             UTCDateTime("2008-01-15T00:00:00.025000"))
+            self.assertEqual(stream[0].stats.calib, 1.0e-00)
+            self.assertEqual(stream[0].stats.mseed.dataquality, 'R')
+            data = [185.01, 181.02, 185.03, 189.04, 194.05, 205.06,
+                    209.07, 214.08, 222.09, 225.98, 226.99, 219.00]
+            np.testing.assert_array_almost_equal(stream[0].data, data,
+                                                 decimal=2)
+            # compare raw header
+            with open(testfile, 'rt') as f:
+                lines_orig = f.readlines()
+            with open(tmpfile, 'rt') as f:
+                lines_new = f.readlines()
+        self.assertEqual(lines_orig[0], lines_new[0])
+
+    def test_write_tspair_custom_fmt(self):
+        """
+        Write TSPAIR file test via obspy.core.ascii._write_tspair.
+        """
+        # float
+        testfile_orig = os.path.join(self.path, 'data', 'tspair_float.ascii')
+        testfile = os.path.join(self.path, 'data',
+                                'tspair_float_custom_fmt.ascii')
+        stream_orig = _read_tspair(testfile_orig)
+        with NamedTemporaryFile() as tf:
+            tmpfile = tf.name
+            # write
+            _write_tspair(stream_orig, tmpfile, custom_fmt='%3.14f')
+            # look at the raw data
+            with open(tmpfile, 'rt') as f:
+                lines = f.readlines()
+            self.assertEqual(
+                lines[0].strip(),
+                'TIMESERIES XX_TEST__BHZ_R, 12 samples, 40 sps, ' +
+                '2008-01-15T00:00:00.025000, TSPAIR, FLOAT, Counts')
+            self.assertEqual(
+                lines[1].strip(),
+                '2008-01-15T00:00:00.025000  185.00999450000000')
             # read again
             stream = _read_tspair(tmpfile)
             stream.verify()
@@ -302,6 +360,29 @@ def test_write_tspair(self):
                 lines_new = f.readlines()
         self.assertEqual(lines_orig[0], lines_new[0])
 
+    def test_write_tspair_custom_fmt_custom(self):
+        """
+        Write TSPAIR file test via obspy.core.ascii._write_tspair.
+        """
+        # float
+        testfile_orig = os.path.join(self.path, 'data', 'tspair_float.ascii')
+        stream_orig = _read_tspair(testfile_orig)
+        with NamedTemporaryFile() as tf:
+            tmpfile = tf.name
+            # write
+            _write_tspair(stream_orig, tmpfile, custom_fmt='%+r')
+            self.assertRaises(NotImplementedError, _read_tspair, tmpfile)
+            # look at the raw data
+            with open(tmpfile, 'rt') as f:
+                lines = f.readlines()
+            self.assertEqual(
+                lines[0].strip(),
+                'TIMESERIES XX_TEST__BHZ_R, 12 samples, 40 sps, ' +
+                '2008-01-15T00:00:00.025000, TSPAIR, CUSTOM, Counts')
+            self.assertEqual(
+                lines[1].strip(),
+                '2008-01-15T00:00:00.025000  185.0099945')
+
     def test_write_tspair_file_multiple_traces(self):
         """
         Write TSPAIR file test via obspy.core.ascii._write_tspair.
@@ -379,8 +460,57 @@ def test_write_slist(self):
                 '2008-01-15T00:00:00.025000, SLIST, FLOAT, Counts')
             self.assertEqual(
                 lines[1].strip(),
-                '185.009995\t181.020004\t185.029999\t189.039993\t' +
-                '194.050003\t205.059998')
+                '+1.8500999450e+02\t+1.8102000430e+02\t+1.8502999880e+02\t' +
+                '+1.8903999330e+02\t+1.9405000310e+02\t+2.0505999760e+02')
+            # read again
+            stream = _read_slist(tmpfile)
+            stream.verify()
+            self.assertEqual(stream[0].stats.network, 'XX')
+            self.assertEqual(stream[0].stats.station, 'TEST')
+            self.assertEqual(stream[0].stats.location, '')
+            self.assertEqual(stream[0].stats.channel, 'BHZ')
+            self.assertEqual(stream[0].stats.sampling_rate, 40.0)
+            self.assertEqual(stream[0].stats.npts, 12)
+            self.assertEqual(stream[0].stats.starttime,
+                             UTCDateTime("2008-01-15T00:00:00.025000"))
+            self.assertEqual(stream[0].stats.calib, 1.0e-00)
+            self.assertEqual(stream[0].stats.mseed.dataquality, 'R')
+            data = [185.01, 181.02, 185.03, 189.04, 194.05, 205.06,
+                    209.07, 214.08, 222.09, 225.98, 226.99, 219.00]
+            np.testing.assert_array_almost_equal(stream[0].data, data,
+                                                 decimal=2)
+            # compare raw header
+            with open(testfile, 'rt') as f:
+                lines_orig = f.readlines()
+            with open(tmpfile, 'rt') as f:
+                lines_new = f.readlines()
+        self.assertEqual(lines_orig[0], lines_new[0])
+
+    def test_write_slist_custom_fmt_float(self):
+        """
+        Write SLIST file test via obspy.core.ascii._write_tspair.
+        """
+        # float
+        testfile_orig = os.path.join(self.path, 'data', 'slist_float.ascii')
+        testfile = os.path.join(self.path, 'data',
+                                'slist_float_custom_fmt.ascii')
+        stream_orig = _read_slist(testfile_orig)
+        with NamedTemporaryFile() as tf:
+            tmpfile = tf.name
+            # write
+            _write_slist(stream_orig, tmpfile, custom_fmt='%3.14f')
+            # look at the raw data
+            with open(tmpfile, 'rt') as f:
+                lines = f.readlines()
+            self.assertEqual(
+                lines[0].strip(),
+                'TIMESERIES XX_TEST__BHZ_R, 12 samples, 40 sps, ' +
+                '2008-01-15T00:00:00.025000, SLIST, FLOAT, Counts')
+            self.assertEqual(
+                lines[1].strip(),
+                '185.00999450000000\t181.02000430000001\t' +
+                '185.02999879999999\t189.03999329999999\t' +
+                '194.05000310000000\t205.05999760000000')
             # read again
             stream = _read_slist(tmpfile)
             stream.verify()
@@ -405,6 +535,31 @@ def test_write_slist(self):
                 lines_new = f.readlines()
         self.assertEqual(lines_orig[0], lines_new[0])
 
+    def test_write_slist_custom_fmt_custom(self):
+        """
+        Write SLIST file test via obspy.core.ascii._write_tspair.
+        """
+        # float
+        testfile_orig = os.path.join(self.path, 'data', 'slist_float.ascii')
+        stream_orig = _read_slist(testfile_orig)
+        with NamedTemporaryFile() as tf:
+            tmpfile = tf.name
+            # write
+            _write_slist(stream_orig, tmpfile, custom_fmt='%+r')
+            self.assertRaises(NotImplementedError, _read_slist, tmpfile)
+            # look at the raw data
+            with open(tmpfile, 'rt') as f:
+                lines = f.readlines()
+            self.assertEqual(
+                lines[0].strip(),
+                'TIMESERIES XX_TEST__BHZ_R, 12 samples, 40 sps, ' +
+                '2008-01-15T00:00:00.025000, SLIST, CUSTOM, Counts')
+            self.assertEqual(
+                lines[1].strip(),
+                '185.0099945\t181.02000430000001\t' +
+                '185.02999879999999\t189.03999329999999\t' +
+                '194.0500031\t205.0599976')
+
     def test_write_slist_file_multiple_traces(self):
         """
         Write SLIST file test via obspy.core.ascii._write_tspair.
@@ -494,6 +649,40 @@ def test_float_sampling_rates_write_and_read(self):
                 self.assertEqual(tr.stats.sampling_rate,
                                  got.stats.sampling_rate)
 
+    def test_determine_dtype(self):
+        """
+        Tests _determine_dtype for properly returned types
+        """
+        float_formats = ['%+10.10e', '%+.10e', '%.3e',
+                         '%+10.10E', '%+.10E', '%.3E',
+                         '%+10.10f', '%+.10f', '%.3f',
+                         '%+10.10F', '%+.10F', '%.3F',
+                         '%+10.10g', '%+.10g', '%.3g',
+                         '%+10.10G', '%+.10G', '%.3G']
+
+        int_formats = ['%+10.10i', '%+.10i', '%.3i',
+                       '%+10.10I', '%+.10I', '%.3I',
+                       '%+10.10d', '%+.10d', '%.3d',
+                       '%+10.10D', '%+.10D', '%.3D']
+
+        custom_formats = ['%+10.10s', '%+.10s', '%.3s',
+                          '%+10.10x', '%+.10x', '%.3x',
+                          '%+10.10k', '%+.10k', '%.3k',
+                          '%+10.10z', '%+.10z', '%.3z',
+                          '%+10.10w', '%+.10w', '%.3w',
+                          '%+10.10q', '%+.10q', '%.3q']
+
+        for format in float_formats:
+            self.assertEqual('FLOAT', _determine_dtype(format))
+
+        for format in int_formats:
+            self.assertEqual('INTEGER', _determine_dtype(format))
+
+        for format in custom_formats:
+            self.assertEqual('CUSTOM', _determine_dtype(format))
+
+        self.assertRaises(ValueError, _determine_dtype, '')
+
 
 def suite():
     return unittest.makeSuite(ASCIITestCase, 'test')