Repository: spacetelescope/jwql

Issue:
Update filename parser to handle segment files of Guider data
It sounds like in addition to TSO data, FGS track and fine guide data can (optionally) be placed into segment files. At the moment the JWQL filename parser is unable to handle those filenames though. 

From Mark Kyprianou:
https://jwql.stsci.edu entered 1118 as the proposal :
 
ValueError at /
Provided file jw01118005001_gs-fg_2022150070312-seg002_uncal.fits does not follow JWST naming conventions.  See https://jwst-docs.stsci.edu/understanding-jwst-data-files/jwst-data-file-naming-conventions for further information.

Request Method:
POST
Request URL:
http://localhost/
Django Version:
3.1.7
Exception Type:
ValueError
Exception Value:
Provided file jw01118005001_gs-fg_2022150070312-seg002_uncal.fits does not follow JWST naming conventions.  See https://jwst-docs.stsci.edu/understanding-jwst-data-files/jwst-data-file-naming-conventions for further information.
Exception Location:
/home/svc_jwqladm_ops/jwql/jwql/jwql/utils/utils.py, line 432, in filename_parser
Python Executable:
/home/svc_jwqladm_ops/jwql/miniconda3/envs/jwql-3.7/bin/python
Python Version:
3.7.10


Discussion:

Resolving patch:
diff --git a/jwql/utils/constants.py b/jwql/utils/constants.py
--- a/jwql/utils/constants.py
+++ b/jwql/utils/constants.py
@@ -168,6 +168,25 @@
                        "pom": "NIS_IMAGE", "wfss": "NIS_WFSS"},
             "fgs": {"imaging": "FGS_IMAGE"}}
 
+# Filename Component Lengths
+FILE_AC_CAR_ID_LEN = 4
+FILE_AC_O_ID_LEN = 3
+FILE_ACT_LEN = 2
+FILE_DATETIME_LEN = 13
+FILE_EPOCH_LEN = 1
+FILE_GUIDESTAR_ATTMPT_LEN = 1
+FILE_OBS_LEN = 3
+FILE_PARALLEL_SEQ_ID_LEN = 1
+FILE_PROG_ID_LEN = 5
+FILE_SEG_LEN = 3
+FILE_SOURCE_ID_LEN = 5
+FILE_TARG_ID_LEN = 3
+FILE_VISIT_GRP_LEN = 2
+FILE_VISIT_LEN = 3
+
+# MSA metadata file do not have a standard suffix attached
+FILETYPE_WO_STANDARD_SUFFIX = 'msa.fits'
+
 FLAT_EXP_TYPES = {'nircam': ['NRC_FLAT'],
                   'niriss': ['NIS_LAMP'],
                   'miri': ['MIR_FLATIMAGE', 'MIR_FLATMRS'],
diff --git a/jwql/utils/utils.py b/jwql/utils/utils.py
--- a/jwql/utils/utils.py
+++ b/jwql/utils/utils.py
@@ -37,7 +37,12 @@
 import jsonschema
 
 from jwql.utils import permissions
-from jwql.utils.constants import FILE_SUFFIX_TYPES, JWST_INSTRUMENT_NAMES_SHORTHAND
+from jwql.utils.constants import FILE_AC_CAR_ID_LEN, FILE_AC_O_ID_LEN, FILE_ACT_LEN, \
+                                 FILE_DATETIME_LEN, FILE_EPOCH_LEN, FILE_GUIDESTAR_ATTMPT_LEN, \
+                                 FILE_OBS_LEN, FILE_PARALLEL_SEQ_ID_LEN, FILE_PROG_ID_LEN, \
+                                 FILE_SEG_LEN, FILE_SOURCE_ID_LEN, FILE_SUFFIX_TYPES, FILE_TARG_ID_LEN, \
+                                 FILE_VISIT_GRP_LEN, FILE_VISIT_LEN, FILETYPE_WO_STANDARD_SUFFIX, \
+                                 JWST_INSTRUMENT_NAMES_SHORTHAND
 
 __location__ = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
 
@@ -278,12 +283,12 @@ def filename_parser(filename):
     # e.g. "jw80500012009_01101_00012_nrcalong_uncal.fits"
     stage_1_and_2 = \
         r"jw" \
-        r"(?P<program_id>\d{5})"\
-        r"(?P<observation>\d{3})"\
-        r"(?P<visit>\d{3})"\
-        r"_(?P<visit_group>\d{2})"\
-        r"(?P<parallel_seq_id>\d{1})"\
-        r"(?P<activity>\w{2})"\
+        r"(?P<program_id>\d{" + f"{FILE_PROG_ID_LEN}" + "})"\
+        r"(?P<observation>\d{" + f"{FILE_OBS_LEN}" + "})"\
+        r"(?P<visit>\d{" + f"{FILE_VISIT_LEN}" + "})"\
+        r"_(?P<visit_group>\d{" + f"{FILE_VISIT_GRP_LEN}" + "})"\
+        r"(?P<parallel_seq_id>\d{" + f"{FILE_PARALLEL_SEQ_ID_LEN}" + "})"\
+        r"(?P<activity>\w{" f"{FILE_ACT_LEN}" + "})"\
         r"_(?P<exposure_id>\d+)"\
         r"_(?P<detector>((?!_)[\w])+)"
 
@@ -291,23 +296,32 @@ def filename_parser(filename):
     # e.g. "jw94015002002_02108_00001_mirimage_o002_crf.fits"
     stage_2c = \
         r"jw" \
-        r"(?P<program_id>\d{5})" \
-        r"(?P<observation>\d{3})" \
-        r"(?P<visit>\d{3})" \
-        r"_(?P<visit_group>\d{2})" \
-        r"(?P<parallel_seq_id>\d{1})" \
-        r"(?P<activity>\w{2})" \
+        r"(?P<program_id>\d{" + f"{FILE_PROG_ID_LEN}" + "})" \
+        r"(?P<observation>\d{" + f"{FILE_OBS_LEN}" + "})" \
+        r"(?P<visit>\d{" + f"{FILE_VISIT_LEN}" + "})" \
+        r"_(?P<visit_group>\d{" + f"{FILE_VISIT_GRP_LEN}" + "})" \
+        r"(?P<parallel_seq_id>\d{" + f"{FILE_PARALLEL_SEQ_ID_LEN}" + "})" \
+        r"(?P<activity>\w{" + f"{FILE_ACT_LEN}" + "})" \
         r"_(?P<exposure_id>\d+)" \
         r"_(?P<detector>((?!_)[\w])+)"\
-        r"_(?P<ac_id>(o\d{3}|(c|a|r)\d{4}))"
+        r"_(?P<ac_id>(o\d{" + f"{FILE_AC_O_ID_LEN}" + r"}|(c|a|r)\d{" + f"{FILE_AC_CAR_ID_LEN}" + "}))"
+
+    # Stage 2 MSA metadata file. Created by APT and loaded in
+    # assign_wcs. e.g. "jw01118008001_01_msa.fits"
+    stage_2_msa = \
+        r"jw" \
+        r"(?P<program_id>\d{" + f"{FILE_PROG_ID_LEN}" + "})"\
+        r"(?P<observation>\d{" + f"{FILE_OBS_LEN}" + "})"\
+        r"(?P<visit>\d{" + f"{FILE_VISIT_LEN}" + "})"\
+        r"(_.._msa.fits)"
 
     # Stage 3 filenames with target ID
     # e.g. "jw80600-o009_t001_miri_f1130w_i2d.fits"
     stage_3_target_id = \
         r"jw" \
-        r"(?P<program_id>\d{5})"\
-        r"-(?P<ac_id>(o\d{3}|(c|a|r)\d{4}))"\
-        r"_(?P<target_id>(t)\d{3})"\
+        r"(?P<program_id>\d{" + f"{FILE_PROG_ID_LEN}" + "})"\
+        r"-(?P<ac_id>(o\d{" + f"{FILE_AC_O_ID_LEN}" + r"}|(c|a|r)\d{" + f"{FILE_AC_CAR_ID_LEN}" + "}))"\
+        r"_(?P<target_id>(t)\d{" + f"{FILE_TARG_ID_LEN}" + "})"\
         r"_(?P<instrument>(nircam|niriss|nirspec|miri|fgs))"\
         r"_(?P<optical_elements>((?!_)[\w-])+)"
 
@@ -315,9 +329,9 @@ def filename_parser(filename):
     # e.g. "jw80600-o009_s00001_miri_f1130w_i2d.fits"
     stage_3_source_id = \
         r"jw" \
-        r"(?P<program_id>\d{5})"\
-        r"-(?P<ac_id>(o\d{3}|(c|a|r)\d{4}))"\
-        r"_(?P<source_id>(s)\d{5})"\
+        r"(?P<program_id>\d{" + f"{FILE_PROG_ID_LEN}" + "})"\
+        r"-(?P<ac_id>(o\d{" + f"{FILE_AC_O_ID_LEN}" + r"}|(c|a|r)\d{" + f"{FILE_AC_CAR_ID_LEN}" + "}))"\
+        r"_(?P<source_id>(s)\d{" + f"{FILE_SOURCE_ID_LEN}" + "})"\
         r"_(?P<instrument>(nircam|niriss|nirspec|miri|fgs))"\
         r"_(?P<optical_elements>((?!_)[\w-])+)"
 
@@ -325,10 +339,10 @@ def filename_parser(filename):
     # e.g. "jw80600-o009_t001-epoch1_miri_f1130w_i2d.fits"
     stage_3_target_id_epoch = \
         r"jw" \
-        r"(?P<program_id>\d{5})"\
-        r"-(?P<ac_id>(o\d{3}|(c|a|r)\d{4}))"\
-        r"_(?P<target_id>(t)\d{3})"\
-        r"-epoch(?P<epoch>\d{1})"\
+        r"(?P<program_id>\d{" + f"{FILE_PROG_ID_LEN}" + "})"\
+        r"-(?P<ac_id>(o\d{" + f"{FILE_AC_O_ID_LEN}" + r"}|(c|a|r)\d{" + f"{FILE_AC_CAR_ID_LEN}" + "}))"\
+        r"_(?P<target_id>(t)\d{" + f"{FILE_TARG_ID_LEN}" + "})"\
+        r"-epoch(?P<epoch>\d{" + f"{FILE_EPOCH_LEN}" + "})"\
         r"_(?P<instrument>(nircam|niriss|nirspec|miri|fgs))"\
         r"_(?P<optical_elements>((?!_)[\w-])+)"
 
@@ -336,10 +350,10 @@ def filename_parser(filename):
     # e.g. "jw80600-o009_s00001-epoch1_miri_f1130w_i2d.fits"
     stage_3_source_id_epoch = \
         r"jw" \
-        r"(?P<program_id>\d{5})"\
-        r"-(?P<ac_id>(o\d{3}|(c|a|r)\d{4}))"\
-        r"_(?P<source_id>(s)\d{5})"\
-        r"-epoch(?P<epoch>\d{1})"\
+        r"(?P<program_id>\d{" + f"{FILE_PROG_ID_LEN}" + "})"\
+        r"-(?P<ac_id>(o\d{" + f"{FILE_AC_O_ID_LEN}" + r"}|(c|a|r)\d{" + f"{FILE_AC_CAR_ID_LEN}" + "}))"\
+        r"_(?P<source_id>(s)\d{" + f"{FILE_SOURCE_ID_LEN}" + "})"\
+        r"-epoch(?P<epoch>\d{" + f"{FILE_EPOCH_LEN}" + "})"\
         r"_(?P<instrument>(nircam|niriss|nirspec|miri|fgs))"\
         r"_(?P<optical_elements>((?!_)[\w-])+)"
 
@@ -347,14 +361,14 @@ def filename_parser(filename):
     # e.g. "jw00733003001_02101_00002-seg001_nrs1_rate.fits"
     time_series = \
         r"jw" \
-        r"(?P<program_id>\d{5})"\
-        r"(?P<observation>\d{3})"\
-        r"(?P<visit>\d{3})"\
-        r"_(?P<visit_group>\d{2})"\
-        r"(?P<parallel_seq_id>\d{1})"\
-        r"(?P<activity>\w{2})"\
+        r"(?P<program_id>\d{" + f"{FILE_PROG_ID_LEN}" + "})"\
+        r"(?P<observation>\d{" + f"{FILE_OBS_LEN}" + "})"\
+        r"(?P<visit>\d{" + f"{FILE_VISIT_LEN}" + "})"\
+        r"_(?P<visit_group>\d{" + f"{FILE_VISIT_GRP_LEN}" + "})"\
+        r"(?P<parallel_seq_id>\d{" + f"{FILE_PARALLEL_SEQ_ID_LEN}" + "})"\
+        r"(?P<activity>\w{" + f"{FILE_ACT_LEN}" + "})"\
         r"_(?P<exposure_id>\d+)"\
-        r"-seg(?P<segment>\d{3})"\
+        r"-seg(?P<segment>\d{" + f"{FILE_SEG_LEN}" + "})"\
         r"_(?P<detector>\w+)"
 
     # Guider filenames
@@ -362,39 +376,54 @@ def filename_parser(filename):
     # "jw00799003001_gs-acq1_2019154181705_stream.fits"
     guider = \
         r"jw" \
-        r"(?P<program_id>\d{5})" \
-        r"(?P<observation>\d{3})" \
-        r"(?P<visit>\d{3})" \
+        r"(?P<program_id>\d{" + f"{FILE_PROG_ID_LEN}" + "})" \
+        r"(?P<observation>\d{" + f"{FILE_OBS_LEN}" + "})" \
+        r"(?P<visit>\d{" + f"{FILE_VISIT_LEN}" + "})" \
+        r"_gs-(?P<guider_mode>(id|acq1|acq2|track|fg))" \
+        r"_((?P<date_time>\d{" + f"{FILE_DATETIME_LEN}" + r"})|(?P<guide_star_attempt_id>\d{" + f"{FILE_GUIDESTAR_ATTMPT_LEN}" + "}))"
+
+    # Segment guider filenames
+    # e.g. "jw01118005001_gs-fg_2022150070312-seg002_uncal.fits"
+    guider_segment = \
+        r"jw" \
+        r"(?P<program_id>\d{" + f"{FILE_PROG_ID_LEN}" + "})" \
+        r"(?P<observation>\d{" + f"{FILE_OBS_LEN}" + "})" \
+        r"(?P<visit>\d{" + f"{FILE_VISIT_LEN}" + "})" \
         r"_gs-(?P<guider_mode>(id|acq1|acq2|track|fg))" \
-        r"_((?P<date_time>\d{13})|(?P<guide_star_attempt_id>\d{1}))"
+        r"_((?P<date_time>\d{" + f"{FILE_DATETIME_LEN}" + r"})|(?P<guide_star_attempt_id>\d{" + f"{FILE_GUIDESTAR_ATTMPT_LEN}" + "}))" \
+        r"-seg(?P<segment>\d{" + f"{FILE_SEG_LEN}" + "})"
 
     # Build list of filename types
     filename_types = [
         stage_1_and_2,
         stage_2c,
+        stage_2_msa,
         stage_3_target_id,
         stage_3_source_id,
         stage_3_target_id_epoch,
         stage_3_source_id_epoch,
         time_series,
-        guider]
+        guider,
+        guider_segment]
 
     filename_type_names = [
         'stage_1_and_2',
         'stage_2c',
+        'stage_2_msa',
         'stage_3_target_id',
         'stage_3_source_id',
         'stage_3_target_id_epoch',
         'stage_3_source_id_epoch',
         'time_series',
-        'guider'
+        'guider',
+        'guider_segment'
     ]
 
     # Try to parse the filename
     for filename_type, filename_type_name in zip(filename_types, filename_type_names):
 
-        # If full filename, try using suffix
-        if not file_root_name:
+        # If full filename, try using suffix, except for *msa.fits files
+        if not file_root_name and FILETYPE_WO_STANDARD_SUFFIX not in filename:
             filename_type += r"_(?P<suffix>{}).*".format('|'.join(FILE_SUFFIX_TYPES))
         # If not, make sure the provided regex matches the entire filename root
         else:
@@ -417,12 +446,15 @@ def filename_parser(filename):
 
         # Also, add the instrument if not already there
         if 'instrument' not in filename_dict.keys():
-            if name_match == 'guider':
+            if name_match in ['guider', 'guider_segment']:
                 filename_dict['instrument'] = 'fgs'
             elif 'detector' in filename_dict.keys():
                 filename_dict['instrument'] = JWST_INSTRUMENT_NAMES_SHORTHAND[
                     filename_dict['detector'][:3].lower()
                 ]
+            elif name_match == 'stage_2_msa':
+                    filename_dict['instrument'] = 'nirspec'
+
 
     # Raise error if unable to parse the filename
     except AttributeError:


Test patch:
diff --git a/jwql/tests/test_utils.py b/jwql/tests/test_utils.py
--- a/jwql/tests/test_utils.py
+++ b/jwql/tests/test_utils.py
@@ -72,6 +72,14 @@
       'visit': '001',
       'visit_group': '02'}),
 
+    # Test stage 2 MSA metadata filename
+    ('jw01118008001_01_msa.fits',
+        {'filename_type': 'stage_2_msa',
+         'instrument': 'nirspec',
+         'observation': '008',
+         'program_id': '01118',
+         'visit': '001'}),
+
     # Test full stage 2c filename
     ('jw94015002002_02108_00001_mirimage_o002_crf.fits',
      {'ac_id': 'o002',
@@ -244,8 +252,20 @@
       'instrument': 'fgs',
       'observation': '011',
       'program_id': '00729',
-      'visit': '001'})
+      'visit': '001'}),
 
+    # Test segmented guider file
+    ('jw01118005001_gs-fg_2022150070312-seg002_uncal.fits',
+     {'date_time': '2022150070312',
+      'filename_type': 'guider_segment',
+      'guide_star_attempt_id': None,
+      'guider_mode': 'fg',
+      'instrument': 'fgs',
+      'observation': '005',
+      'program_id': '01118',
+      'segment': '002',
+      'suffix': 'uncal',
+      'visit': '001'}),
 ]