Repository: SCECcode/pycsep (language: python)

Issue:
need test for JMA catalog
need a test for the creation of the JMA catalog, using random data and snippet of test catalog. @gen2 


Discussion / hints:
closed by #21 

Fix patch:
diff --git a/csep/__init__.py b/csep/__init__.py
index e2cc817e..cc992256 100644
--- a/csep/__init__.py
+++ b/csep/__init__.py
@@ -117,8 +117,8 @@ def load_catalog(filename, type='csep-csv', format='native', loader=None, apply_
     Returns (:class:`~csep.core.catalogs.AbstractBaseCatalog`)
     """
 
-    if type not in ('ucerf3', 'csep-csv', 'zmap', 'jma-csv', 'ndk') and loader is None:
-        raise ValueError("type must be one of the following: ('ucerf3', 'csep-csv', 'zmap', 'jma-csv', 'ndk').")
+    if type not in ('ucerf3', 'csep-csv', 'zmap', 'jma-csv', 'ingv_horus', 'ingv_emrcmt', 'ndk') and loader is None:
+        raise ValueError("type must be one of the following: ('ucerf3', 'csep-csv', 'zmap', 'jma-csv', 'ndk', 'ingv_horus', 'ingv_emrcmt').")
 
     # map to correct catalog class, at some point these could be abstracted into configuration file
     # this maps a human readable string to the correct catalog class and the correct loader function
@@ -142,6 +142,14 @@ def load_catalog(filename, type='csep-csv', format='native', loader=None, apply_
         'ndk': {
             'class': catalogs.CSEPCatalog,
             'loader': readers.ndk
+        },
+        'ingv_horus': {
+            'class': catalogs.CSEPCatalog,
+            'loader': readers.ingv_horus
+        },
+        'ingv_emrcmt': {
+            'class': catalogs.CSEPCatalog,
+            'loader': readers.ingv_emrcmt
         }
     }
 
diff --git a/csep/utils/readers.py b/csep/utils/readers.py
index e336e93d..cb971cef 100644
--- a/csep/utils/readers.py
+++ b/csep/utils/readers.py
@@ -495,13 +495,20 @@ def ingv_emrcmt(fname):
            'depth': 6,
            'Mw': 61}
 
+    def is_header_line(line):
+        if line[0] == 'ev_id':
+            return True
+        else:
+            return False
+
     out = []
     evcat_id = []
     n_event = 0
     with open(fname) as file_:
         reader = csv.reader(file_)
-        
         for n, line in enumerate(reader):
+            if is_header_line(line):
+                continue
             try:
                 date = line[ind['date']].replace('-', '/')
                 time = line[ind['time']].replace(' ', '0')
@@ -549,7 +556,6 @@ def ingv_emrcmt(fname):
         
     return out
 
-
 def ingv_horus(fname):
     """
     Reader for the INGV (Istituto Nazionale di Geofisica e Vulcanologia - Italy)


Test patch:
diff --git a/tests/artifacts/ingv_catalogs/EuroMedCentrMomTensors.csv b/tests/artifacts/ingv_catalogs/EuroMedCentrMomTensors.csv
index a9245bf8..34f374a8 100644
--- a/tests/artifacts/ingv_catalogs/EuroMedCentrMomTensors.csv
+++ b/tests/artifacts/ingv_catalogs/EuroMedCentrMomTensors.csv
@@ -1,5 +1,6 @@
-"C201901150125A","2019-01-15","01:25: 5"," 0","  38.93","   20.55","  10.0","4.5","0.0","GREECE                   ","ING","  0","    0","   0","  1","    1","  40"," 21","   24","  35"," 1","  3.2"," 0.5","  38.87"," 0.02","   20.54"," 0.02","  22.6","  1.7","FREE "," 1.00","D-C201901150125A","22","  1.980"," 0.610"," -0.150"," 0.400"," -1.830"," 0.430"," -3.350"," 0.530"," -0.200"," 0.370","  6.340"," 0.290","   6.920"," 30"," 143","   0.920"," 57"," 296","  -7.840"," 12","  46","   7.380"," 181"," 60","  167"," 278"," 79","   31","4.52","D","A"
-"S201005021224A","2010-05-02","12:24:37","0","35.61","24.69","89.00","4.5","0.0","CRETE"," GR","0","0","0","0","0","0","0","11","15","35","1.0","3.6","1.90","35.53","0.10","24.46","0.050","74.700","10","0.0"," 1.00","0","22.000","0.500","0.360","-3.200","0.450","2.700","0.500","-0.890","0.250","-0.240","0.220","-4.790","0.500","5","3","99.999","1","82","99.999","-6","8","99.999","6","16","83","-4","106","86","-9.9"," ","D"
-"S201008080915A","2010-08-08","09:15:00","9","40.52","23.52","10.00","4.5","0.0","GREECE                   ","ING","0","0","0","0","0","0","13","16","30","1","1.8","0.3","40.55","0.02","23.58","0.05","14.100","2.000","FREE","1.0","D-S201008080915A","22","-2.800","0.490","3.560","0.350","-0.770","0.250","-2.950","0.530","-0.540","0.360","-0.120","0.210","4.720","21","179","-0.670","9","86","-4.050","67","335","4.380","285","25","-69","82","67","-99","4.4","D","A"
-"S201912120305A","2019-12-12","03: 5:","0","34.80","26.32","20.00","5.0","0.0","CRETE--GREECE------------","ING","0","0","0","0","0","0","28","38","35","1","-1.5","0.5","34.63","0.03","26.44","0.02","36.300","1.100","FREE","1.3","D-S201912120305A","23","-1.140","0.160","-0.830","0.100","1.970","0.090","0.430","0.080","-1.120","0.060","0.230","0.090","2.330","18","92","-0.590","27","353","-1.750","57","211","2.040","216","36","-40","341","68","-119","4.8","Q","A"
-"S201008080915A","2010-08-08","09:15:00","9","40.52","23.52","10.00","4.5","0.0","GREECE                   ","ING","0","0","0","0","0","0","13","16","30","1","1.8","0.3","40.55","0.02","23.58","0.05","14.100","2.000","FREE","1.0","D-S201008080915A","22","-2.800","0.490","3.560","0.350","-0.770","0.250","-2.950","0.530","-0.540","0.360","-0.120","0.210","4.720","21","179","-0.670","9","86","-4.050","67","335","4.380","285","25","-69","82","67","-99","4.4","D","A"
+ev_id,date,time_orig,time_orig_decimals,lat_orig,long_orig,depth,Mb,Ms,region,source_type,bwave_n_stations,bwave_n_inv,bwave_cutoff,mwave_n_stations,mwave_n_inv,mwave_cutoff,swave_n_stations,swave_n_inv,swave_cutoff,dcouple,centroid_time,delta_centroid_time,centroid_lat,delta_centroid_lat,centroid_long,delta_centroid_long,centroid_depth,delta_centroid_depth,prof,half_dur,inv_date_string,exp,mrr,delta_mrr,mss,delta_mss,mee,delta_mee,mrs,delta_mrs,mre,delta_mre,mse,delta_mse,eigen_t,plunge_t,strike_t,eigen_n,plunge_n,strike_n,eigen_p,plunge_p,strike_p,scalar_moment,strike1,dip1,rake1,strike2,dip2,rake2,Mw,status_flag,quality_flag
+C201901150125A,2019-01-15,01:25: 5,0,38.93,20.55,10,4.5,0,GREECE                   ,ING,0,0,0,1,1,40,21,24,35,1,3.2,0.5,38.87,0.02,20.54,0.02,22.6,1.7,FREE ,1,D-C201901150125A,22,1.98,0.61,-0.15,0.4,-1.83,0.43,-3.35,0.53,-0.2,0.37,6.34,0.29,6.92,30,143,0.92,57,296,-7.84,12,46,7.38,181,60,167,278,79,31,4.52,D,A
+S201005021224A,2010-05-02,12:24:37,0,35.61,24.69,89,4.5,0,CRETE, GR,0,0,0,0,0,0,0,11,15,35,1,3.6,1.9,35.53,0.1,24.46,0.05,74.7,10,0,1,0,22,0.5,0.36,-3.2,0.45,2.7,0.5,-0.89,0.25,-0.24,0.22,-4.79,0.5,5,3,99.999,1,82,99.999,-6,8,99.999,6,16,83,-4,106,86,-9.9, ,D
+S201008080915A,2010-08-08,09:15:00,9,40.52,23.52,10,4.5,0,GREECE                   ,ING,0,0,0,0,0,0,13,16,30,1,1.8,0.3,40.55,0.02,23.58,0.05,14.1,2,FREE,1,D-S201008080915A,22,-2.8,0.49,3.56,0.35,-0.77,0.25,-2.95,0.53,-0.54,0.36,-0.12,0.21,4.72,21,179,-0.67,9,86,-4.05,67,335,4.38,285,25,-69,82,67,-99,4.4,D,A
+S201912120305A,2019-12-12,03: 5:,0,34.8,26.32,20,5,0,CRETE--GREECE------------,ING,0,0,0,0,0,0,28,38,35,1,-1.5,0.5,34.63,0.03,26.44,0.02,36.3,1.1,FREE,1.3,D-S201912120305A,23,-1.14,0.16,-0.83,0.1,1.97,0.09,0.43,0.08,-1.12,0.06,0.23,0.09,2.33,18,92,-0.59,27,353,-1.75,57,211,2.04,216,36,-40,341,68,-119,4.8,Q,A
+S201008080915A,2010-08-08,09:15:00,9,40.52,23.52,10,4.5,0,GREECE                   ,ING,0,0,0,0,0,0,13,16,30,1,1.8,0.3,40.55,0.02,23.58,0.05,14.1,2,FREE,1,D-S201008080915A,22,-2.8,0.49,3.56,0.35,-0.77,0.25,-2.95,0.53,-0.54,0.36,-0.12,0.21,4.72,21,179,-0.67,9,86,-4.05,67,335,4.38,285,25,-69,82,67,-99,4.4,D,A
diff --git a/tests/artifacts/ingv_catalogs/EuroMedCentrMomTensors_old.csv b/tests/artifacts/ingv_catalogs/EuroMedCentrMomTensors_old.csv
new file mode 100644
index 00000000..2d8e51f8
--- /dev/null
+++ b/tests/artifacts/ingv_catalogs/EuroMedCentrMomTensors_old.csv
@@ -0,0 +1,5 @@
+C201901150125A,2019-01-15,01:25: 5,0,38.93,20.55,10,4.5,0,GREECE                   ,ING,0,0,0,1,1,40,21,24,35,1,3.2,0.5,38.87,0.02,20.54,0.02,22.6,1.7,FREE ,1,D-C201901150125A,22,1.98,0.61,-0.15,0.4,-1.83,0.43,-3.35,0.53,-0.2,0.37,6.34,0.29,6.92,30,143,0.92,57,296,-7.84,12,46,7.38,181,60,167,278,79,31,4.52,D,A
+S201005021224A,2010-05-02,12:24:37,0,35.61,24.69,89,4.5,0,CRETE, GR,0,0,0,0,0,0,0,11,15,35,1,3.6,1.9,35.53,0.1,24.46,0.05,74.7,10,0,1,0,22,0.5,0.36,-3.2,0.45,2.7,0.5,-0.89,0.25,-0.24,0.22,-4.79,0.5,5,3,99.999,1,82,99.999,-6,8,99.999,6,16,83,-4,106,86,-9.9, ,D
+S201008080915A,2010-08-08,09:15:00,9,40.52,23.52,10,4.5,0,GREECE                   ,ING,0,0,0,0,0,0,13,16,30,1,1.8,0.3,40.55,0.02,23.58,0.05,14.1,2,FREE,1,D-S201008080915A,22,-2.8,0.49,3.56,0.35,-0.77,0.25,-2.95,0.53,-0.54,0.36,-0.12,0.21,4.72,21,179,-0.67,9,86,-4.05,67,335,4.38,285,25,-69,82,67,-99,4.4,D,A
+S201912120305A,2019-12-12,03: 5:,0,34.8,26.32,20,5,0,CRETE--GREECE------------,ING,0,0,0,0,0,0,28,38,35,1,-1.5,0.5,34.63,0.03,26.44,0.02,36.3,1.1,FREE,1.3,D-S201912120305A,23,-1.14,0.16,-0.83,0.1,1.97,0.09,0.43,0.08,-1.12,0.06,0.23,0.09,2.33,18,92,-0.59,27,353,-1.75,57,211,2.04,216,36,-40,341,68,-119,4.8,Q,A
+S201008080915A,2010-08-08,09:15:00,9,40.52,23.52,10,4.5,0,GREECE                   ,ING,0,0,0,0,0,0,13,16,30,1,1.8,0.3,40.55,0.02,23.58,0.05,14.1,2,FREE,1,D-S201008080915A,22,-2.8,0.49,3.56,0.35,-0.77,0.25,-2.95,0.53,-0.54,0.36,-0.12,0.21,4.72,21,179,-0.67,9,86,-4.05,67,335,4.38,285,25,-69,82,67,-99,4.4,D,A
diff --git a/tests/test_ingv_readers.py b/tests/test_ingv_readers.py
index 5dea152d..f48a00e6 100755
--- a/tests/test_ingv_readers.py
+++ b/tests/test_ingv_readers.py
@@ -11,6 +11,12 @@ def get_horus_path():
                             'HORUS_Ita_Catalog.txt')
     return data_dir
 
+def get_emrcmt_path_old():
+    root_dir = os.path.dirname(os.path.abspath(__file__))
+    data_dir = os.path.join(root_dir, 'artifacts', 'ingv_catalogs',
+                            'EuroMedCentrMomTensors_old.csv')
+    return data_dir
+
 def get_emrcmt_path():
     root_dir = os.path.dirname(os.path.abspath(__file__))
     data_dir = os.path.join(root_dir, 'artifacts', 'ingv_catalogs',
@@ -20,40 +26,42 @@ def get_emrcmt_path():
 class TestReadCatalog(unittest.TestCase):
 
     def test_cat_emrcmt(self):
-        filepath = get_emrcmt_path()
-        catalog_tuples = readers.ingv_emrcmt(filepath)
-    
-        #Test removal of wrong mw events and repeated events
-        self.assertEqual(len(catalog_tuples),3)   
-        #Test the fixing of date and time typos (epoch obtained manually)
-        self.assertEqual(catalog_tuples[0][1],1547515505000)
-        self.assertEqual(catalog_tuples[2][1],1576119900000)
-        
-        #Test reader
-        target_lat = []
-        target_lon = []
-        target_depth = []
-        target_mw = []
-        with open(filepath) as file_:
-            for targetline in file_.readlines():
-                targetline_ = targetline.split(',')
-                target_lon.append(float(targetline_[5].replace('"', '')))
-                target_lat.append(float(targetline_[4].replace('"', '')))
-                target_depth.append(float(targetline_[6].replace('"', '')))
-                target_mw.append(float(targetline_[-3].replace('"', '')))
-
-        for event in catalog_tuples:
-            self.assertIn(event[2], target_lat)
-            self.assertIn(event[3], target_lon)
-            self.assertIn(event[4], target_depth)
-            self.assertIn(event[5], target_mw)
+        filepaths = [get_emrcmt_path(), get_emrcmt_path_old()]
+        for filepath in filepaths:
+            catalog_tuples = readers.ingv_emrcmt(filepath)
+
+            # Test removal of wrong mw events and repeated events
+            self.assertEqual(len(catalog_tuples), 3)
+            # Test the fixing of date and time typos (epoch obtained manually)
+            self.assertEqual(catalog_tuples[0][1], 1547515505000)
+            self.assertEqual(catalog_tuples[2][1], 1576119900000)
+
+            # Test reader
+            target_lat = []
+            target_lon = []
+            target_depth = []
+            target_mw = []
+            with open(filepath) as file_:
+                for targetline in file_.readlines():
+                    targetline_ = targetline.split(',')
+                    if targetline_[0] == 'ev_id':
+                        continue
+                    target_lon.append(float(targetline_[5].replace('"', '')))
+                    target_lat.append(float(targetline_[4].replace('"', '')))
+                    target_depth.append(float(targetline_[6].replace('"', '')))
+                    target_mw.append(float(targetline_[-3].replace('"', '')))
+            for event in catalog_tuples:
+                self.assertIn(event[2], target_lat)
+                self.assertIn(event[3], target_lon)
+                self.assertIn(event[4], target_depth)
+                self.assertIn(event[5], target_mw)
 
     def test_cat_horus(self):
         filepath = get_horus_path()
         catalog_tuples = readers.ingv_horus(filepath)
-        
+
         self.assertEqual(len(catalog_tuples), 2)
-        #Test the fixing of over times typos (e.g. 61 sec) 
+        #Test the fixing of over times typos (e.g. 61 sec)
         self.assertEqual(catalog_tuples[1][1], 1600772464000)