Repository: PIC-IRIS/PH5 (language: python)

Issue:
seg2toph5 not working after server updates
**Describe the bug**
Command-line and GUI seg2toph5 ingestion attempts result in an error like the following:
`[2021-03-31 14:29:53,702] - ph5.utilities.seg2toph5 - ERROR: list indices must be integers, not unicode. Can't process /external/ph5/Working/Holly_seg2_test/RAW/15001.dat`
I also got the same error with a few SEG-2 files that were used in the 18-030 PH5.

**Environment (please complete the following information):**
 - OS: ph51
 - Program Version seg2toph5 2019.42

**To Reproduce**
Run pforma with a list of one or more SEG-2 (.dat) files, or initialize a ph5 and run `seg2toph5 -f list-of-files -n master.ph5 -M num_minis -S starting_mini`

**Expected behavior**
Expect ingestion of SEG-2 format files to occur.

**Additional context**
Recent updates included moving to obspy 1.2.2.
The ingestion works on a server that has not received these updates.
Other ingestion methods work (do not generate an error) on the updated server.

The files I was using do not need to go into a PH5 archive.
fix over-indent that create more loop in update_external_references
### What does this PR do?
Fix seg2toph5, time increasing to finish each file  #395 

### Description
In update_external_references there are 2 for loops that are supposed to be independently. However there was a mistake that give more indent to the second loop that make it nested inside the first loop. 
Fixing the indentation error will solve the issue.




Discussion / hints:

Fix patch:
diff --git a/CHANGELOG.txt b/CHANGELOG.txt
index a96f05d1..06cc18bf 100644
--- a/CHANGELOG.txt
+++ b/CHANGELOG.txt
@@ -88,6 +88,7 @@ ph5.utilities.seg2toph5
  * fix bug missing variable
  * Make compatible for pforma
  * Add flag -r/--raw
+ * Make compatible with both Obspy 1.1.1 and 1.2.2
 ph5.utilities.pforma_io
  * add seg2 to pforma
  * fix bug that duplicate index_t and array_t when copy tables from A/master.ph5 to Sigma/master.ph5
diff --git a/ph5/utilities/seg2toph5.py b/ph5/utilities/seg2toph5.py
index be4f9a57..0402f6c5 100644
--- a/ph5/utilities/seg2toph5.py
+++ b/ph5/utilities/seg2toph5.py
@@ -15,7 +15,7 @@
 
 from obspy import read as readSEG2
 
-PROG_VERSION = "2019.42"
+PROG_VERSION = "2021.97"
 LOGGER = logging.getLogger(__name__)
 
 MAX_PH5_BYTES = 1073741824 * 1.  # 1 GB (1024 X 1024 X 1024 X 2)
@@ -252,14 +252,11 @@ def smallest():
 
 
 def update_external_references():
-    global EX, INDEX_T_DAS
-
     LOGGER.info("Updating external references...")
     n = 0
     for i in INDEX_T_DAS.rows:
         external_file = i['external_file_name_s'][2:]
         external_path = i['hdf5_path_s']
-        i['serial_number_s']
         target = external_file + ':' + external_path
         external_group = external_path.split('/')[3]
 
@@ -278,28 +275,27 @@ def update_external_references():
         except Exception as e:
             LOGGER.error(e)
 
-        n = 0
-        for i in INDEX_T_MAP.rows:
-            external_file = i['external_file_name_s'][2:]
-            external_path = i['hdf5_path_s']
-            i['serial_number_s']
-            target = external_file + ':' + external_path
-            external_group = external_path.split('/')[3]
-
-            # Nuke old node
-            try:
-                group_node = EX.ph5.get_node(external_path)
-                group_node.remove()
-            except Exception as e:
-                pass
-
-            # Re-create node
-            try:
-                EX.ph5.create_external_link(
-                    '/Experiment_g/Maps_g', external_group, target)
-                n += 1
-            except Exception as e:
-                LOGGER.error(e.message)
+    n = 0
+    for i in INDEX_T_MAP.rows:
+        external_file = i['external_file_name_s'][2:]
+        external_path = i['hdf5_path_s']
+        target = external_file + ':' + external_path
+        external_group = external_path.split('/')[3]
+
+        # Nuke old node
+        try:
+            group_node = EX.ph5.get_node(external_path)
+            group_node.remove()
+        except Exception as e:
+            pass
+
+        # Re-create node
+        try:
+            EX.ph5.create_external_link(
+                '/Experiment_g/Maps_g', external_group, target)
+            n += 1
+        except Exception as e:
+            LOGGER.error(e.message)
     LOGGER.info("Done, {0} nodes recreated.\n".format(n))
 
 
@@ -465,8 +461,17 @@ def process(hdr, header_type):
                     td[k] = trace.stats.seg2[k]
                 else:
                     tdd = {}
-                    for j in trace.stats.seg2[k]:
-                        tdd[j] = trace.stats.seg2[k][j]
+
+                    if isinstance(trace.stats.seg2[k], list):
+                        # list: Obspy 1.2.2
+                        for j in trace.stats.seg2[k]:
+                            k, v = j.split()
+                            tdd[k] = v
+                    else:
+                        # AtribDict: Obspy 1.1.1
+                        for j in trace.stats.seg2[k]:
+                            tdd[j] = trace.stats.seg2[k][j]
+
                     td[k] = tdd
 
         log_array, name = getLOG(CURRENT_DAS)
@@ -577,6 +582,11 @@ def main():
     seconds = time.time() - then
     print "Done...{0:b}".format(int(seconds / 6.))  # Minutes X 10
     LOGGER.info("Done...{0:b}".format(int(seconds / 6.)))
+    try:
+        EX.ph5close()
+        EXREC.ph5close()
+    except NameError:
+        pass
 
 
 if __name__ == '__main__':


Test patch:
diff --git a/ph5/test_data/seg2/15001.dat b/ph5/test_data/seg2/15001.dat
new file mode 100644
index 00000000..da449b71
Binary files /dev/null and b/ph5/test_data/seg2/15001.dat differ
diff --git a/ph5/utilities/tests/test_seg2toph5.py b/ph5/utilities/tests/test_seg2toph5.py
new file mode 100644
index 00000000..014ca2e3
--- /dev/null
+++ b/ph5/utilities/tests/test_seg2toph5.py
@@ -0,0 +1,119 @@
+'''
+Tests for seg2toph5
+'''
+import os
+import sys
+import unittest
+import logging
+
+from mock import patch
+from testfixtures import OutputCapture, LogCapture
+
+from ph5.utilities import seg2toph5, initialize_ph5
+from ph5.core.tests.test_base import LogTestCase, TempDirTestCase,\
+    initialize_ex
+from ph5.core import ph5api
+
+
+class TestSeg2toPH5_main(TempDirTestCase, LogTestCase):
+    def tearDown(self):
+        self.ph5object.ph5close()
+        super(TestSeg2toPH5_main, self).tearDown()
+
+    def test_main(self):
+        # check external links created
+        data_nodes = ['Das_g_0000SV01', 'Das_g_0000SV02', 'Das_g_0000SV03',
+                      'Das_g_0000SV04', 'Das_g_0000SV05', 'Das_g_0000SV06',
+                      'Das_g_0000SV07', 'Das_g_0000SV08', 'Das_g_0000SV09',
+                      'Das_g_0000SV10', 'Das_g_0000SV11', 'Das_g_0000SV12',
+                      'Das_g_0000SV13', 'Das_g_0000SV14', 'Das_g_0000SV15',
+                      'Das_g_0000SV16', 'Das_g_0000SV17', 'Das_g_0000SV18',
+                      'Das_g_0000SV19', 'Das_g_0000SV20', 'Das_g_0000SV21',
+                      'Das_g_0000SV22', 'Das_g_0000SV23', 'Das_g_0000SV24',
+                      'Das_g_0000SV25', 'Das_g_0000SV26', 'Das_g_0000SV27',
+                      'Das_g_0000SV28', 'Das_g_0000SV29', 'Das_g_0000SV30',
+                      'Das_g_0000SV31', 'Das_g_0000SV32', 'Das_g_0000SV33',
+                      'Das_g_0000SV34', 'Das_g_0000SV35', 'Das_g_0000SV36',
+                      'Das_g_0000SV37', 'Das_g_0000SV38', 'Das_g_0000SV39',
+                      'Das_g_0000SV40', 'Das_g_0000SV41', 'Das_g_0000SV42',
+                      'Das_g_0000SV43', 'Das_g_0000SV44', 'Das_g_0000SV45',
+                      'Das_g_0000SV46', 'Das_g_0000SV47', 'Das_g_0000SV48',
+                      'Das_g_0000SV49', 'Das_g_0000SV50', 'Das_g_0000SV51',
+                      'Das_g_0000SV52', 'Das_g_0000SV53', 'Das_g_0000SV54',
+                      'Das_g_0000SV55', 'Das_g_0000SV56', 'Das_g_0000SV57',
+                      'Das_g_0000SV58', 'Das_g_0000SV59', 'Das_g_0000SV60']
+        testargs = ['initialize_ph5', '-n', 'master.ph5']
+        with patch.object(sys, 'argv', testargs):
+            initialize_ph5.main()
+        # add seg2 to ph5
+        testargs = ['seg2toph5', '-n', 'master.ph5', '-r',
+                    os.path.join(self.home, "ph5/test_data/seg2/15001.dat")]
+        with patch.object(sys, 'argv', testargs):
+            with OutputCapture():
+                with LogCapture() as log:
+                    log.setLevel(logging.ERROR)
+                    seg2toph5.main()
+
+        # before commit caf6978, there would be 1 error log if run this in
+        # environment that uses Obspy 1.2.2
+        self.assertEqual(len(log.records), 0)
+        self.ph5object = ph5api.PH5(path=self.tmpdir, nickname='master.ph5')
+
+        target_p1 = 'miniPH5_00001.ph5:/Experiment_g/Maps_g/'
+        targets = [target_p1 + n for n in data_nodes]
+
+        node = self.ph5object.ph5.get_node("/Experiment_g/Maps_g/")
+        i = 0
+        ret_targets = []
+        for n in self.ph5object.ph5.list_nodes(node):
+            if hasattr(n, 'target'):
+                ret_targets.append(n.target)
+                i += 1
+        self.assertEqual(ret_targets, targets)
+
+        target_p1 = 'miniPH5_00001.ph5:/Experiment_g/Receivers_g/'
+        targets = [target_p1 + n for n in data_nodes]
+
+        node = self.ph5object.ph5.get_node("/Experiment_g/Receivers_g/")
+        i = 0
+        ret_targets = []
+        for n in self.ph5object.ph5.list_nodes(node):
+            if hasattr(n, 'target'):
+                ret_targets.append(n.target)
+                i += 1
+        self.assertEqual(ret_targets, targets)
+
+    def test_update_external_references(self):
+        self.ph5object = seg2toph5.EX = \
+            initialize_ex('master.ph5', '.', True)
+        keys = ['external_file_name_s', 'hdf5_path_s', 'serial_number_s']
+        INDEX_T_DAS_rows = \
+            [{'external_file_name_s': './miniPH5_00001.ph5',
+              'hdf5_path_s': '/Experiment_g/Receivers_g/Das_g_0000SV01'}]
+        seg2toph5.INDEX_T_DAS = seg2toph5.Rows_Keys(INDEX_T_DAS_rows, keys)
+
+        INDEX_T_MAP_rows = \
+            [{'external_file_name_s': './miniPH5_00001.ph5',
+              'hdf5_path_s': '/Experiment_g/Maps_g/Das_g_0000SV01'}]
+        seg2toph5.INDEX_T_MAP = seg2toph5.Rows_Keys(INDEX_T_MAP_rows, keys)
+
+        seg2toph5.update_external_references()
+
+        # check if external links are created
+        node = self.ph5object.ph5.get_node("/Experiment_g/Receivers_g/")
+        target = 'miniPH5_00001.ph5:/Experiment_g/Receivers_g/Das_g_0000SV01'
+        for n in self.ph5object.ph5.list_nodes(node):
+            if hasattr(n, 'target'):
+                self.assertEqual(n.target, target)
+                break
+
+        node = self.ph5object.ph5.get_node("/Experiment_g/Maps_g/")
+        target = 'miniPH5_00001.ph5:/Experiment_g/Maps_g/Das_g_0000SV01'
+        for n in self.ph5object.ph5.list_nodes(node):
+            if hasattr(n, 'target'):
+                self.assertEqual(n.target, target)
+                break
+
+
+if __name__ == "__main__":
+    unittest.main()