Repository: spacetelescope/jwql

Issue:
Update filename_parser to handle time series and stage 3 filenames
There have been recent updates to JWST filename conventions, specifically for time series observations and stage 3 data products, as described in [this](https://jwst-docs.stsci.edu/display/JDAT/File+Naming+Conventions+and+Data+Products) and [this](https://jwst-pipeline.readthedocs.io/en/stable/jwst/introduction.html#output-file-and-associations).  We should update `utils.filename_parser()` to handle this.


Discussion:

Resolving patch:
diff --git a/jwql/utils/utils.py b/jwql/utils/utils.py
--- a/jwql/utils/utils.py
+++ b/jwql/utils/utils.py
@@ -31,7 +31,8 @@
 __location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__)))
 
 FILE_SUFFIX_TYPES = ['uncal', 'cal', 'rateints', 'rate', 'trapsfilled',
-                     'i2d', 'x1d', 's2d', 's3d', 'dark', 'ami', 'crf']
+                     'i2d', 'x1d', 's2d', 's3d', 'dark', 'ami', 'crf',
+                     'crfints']
 JWST_INSTRUMENTS = sorted(['NIRISS', 'NIRCam', 'NIRSpec', 'MIRI', 'FGS'])
 JWST_DATAPRODUCTS = ['IMAGE', 'SPECTRUM', 'SED', 'TIMESERIES', 'VISIBILITY',
                      'EVENTLIST', 'CUBE', 'CATALOG', 'ENGINEERING', 'NULL']
@@ -78,7 +79,7 @@ def ensure_dir_exists(fullpath):
 
 def filename_parser(filename):
     """Return a dictionary that contains the properties of a given
-    JWST file (e.g. program ID, visit number, detector, etc.)
+    JWST file (e.g. program ID, visit number, detector, etc.).
 
     Parameters
     ----------
@@ -95,32 +96,94 @@ def filename_parser(filename):
     ValueError
         When the provided file does not follow naming conventions
     """
-    filename = os.path.basename(filename)
 
+    filename = os.path.basename(filename)
     file_root_name = (len(filename.split('.')) < 2)
 
-    regex_string_to_compile = r"[a-z]+" \
-                               "(?P<program_id>\d{5})"\
-                               "(?P<observation>\d{3})"\
-                               "(?P<visit>\d{3})"\
-                               "_(?P<visit_group>\d{2})"\
-                               "(?P<parallel_seq_id>\d{1})"\
-                               "(?P<activity>\w{2})"\
-                               "_(?P<exposure_id>\d+)"\
-                               "_(?P<detector>\w+)"
-
-    if not file_root_name:
-        regex_string_to_compile += r"_(?P<suffix>{}).*".format('|'.join(FILE_SUFFIX_TYPES))
-
-    elements = \
-        re.compile(regex_string_to_compile)
-
-    jwst_file = elements.match(filename)
-
-    if jwst_file is not None:
+    # Stage 1 and 2 filenames, e.g. "jw80500012009_01101_00012_nrcalong_uncal.fits"
+    stage_1_and_2 = \
+        r"jw" \
+        r"(?P<program_id>\d{5})"\
+        r"(?P<observation>\d{3})"\
+        r"(?P<visit>\d{3})"\
+        r"_(?P<visit_group>\d{2})"\
+        r"(?P<parallel_seq_id>\d{1})"\
+        r"(?P<activity>\w{2})"\
+        r"_(?P<exposure_id>\d+)"\
+        r"_(?P<detector>((?!_)[\w])+)"
+
+    # Stage 3 filenames with target ID, e.g. "jw80600-o009_t001_miri_f1130w_i2d.fits"
+    stage_3_target_id = \
+        r"jw" \
+        r"(?P<program_id>\d{5})"\
+        r"-(?P<ac_id>(o|c|a|r)\d{3,4})"\
+        r"_(?P<target_id>(t)\d{3})"\
+        r"_(?P<instrument>(nircam|niriss|nirspec|miri|fgs))"\
+        r"_(?P<optical_elements>((?!_)[\w-])+)"
+
+    # Stage 3 filenames with source ID, e.g. "jw80600-o009_s00001_miri_f1130w_i2d.fits"
+    stage_3_source_id = \
+        r"jw" \
+        r"(?P<program_id>\d{5})"\
+        r"-(?P<ac_id>(o|c|a|r)\d{3,4})"\
+        r"_(?P<source_id>(s)\d{5})"\
+        r"_(?P<instrument>(nircam|niriss|nirspec|miri|fgs))"\
+        r"_(?P<optical_elements>((?!_)[\w-])+)"
+
+    # Stage 3 filenames with target ID and epoch, e.g. "jw80600-o009_t001-epoch1_miri_f1130w_i2d.fits"
+    stage_3_target_id_epoch = \
+        r"jw" \
+        r"(?P<program_id>\d{5})"\
+        r"-(?P<ac_id>(o|c|a|r)\d{3,4})"\
+        r"_(?P<target_id>(t)\d{3})"\
+        r"-epoch(?P<epoch>\d{1})"\
+        r"_(?P<instrument>(nircam|niriss|nirspec|miri|fgs))"\
+        r"_(?P<optical_elements>((?!_)[\w-])+)"
+
+    # Stage 3 filenames with source ID and epoch, e.g. "jw80600-o009_s00001-epoch1_miri_f1130w_i2d.fits"
+    stage_3_source_id_epoch = \
+        r"jw" \
+        r"(?P<program_id>\d{5})"\
+        r"-(?P<ac_id>(o|c|a|r)\d{3,4})"\
+        r"_(?P<source_id>(s)\d{5})"\
+        r"-epoch(?P<epoch>\d{1})"\
+        r"_(?P<instrument>(nircam|niriss|nirspec|miri|fgs))"\
+        r"_(?P<optical_elements>((?!_)[\w-])+)"
+
+    # Time series filenames, e.g. "jw00733003001_02101_00002-seg001_nrs1_rate.fits"
+    time_series = \
+        r"jw" \
+        r"(?P<program_id>\d{5})"\
+        r"(?P<observation>\d{3})"\
+        r"(?P<visit>\d{3})"\
+        r"_(?P<visit_group>\d{2})"\
+        r"(?P<parallel_seq_id>\d{1})"\
+        r"(?P<activity>\w{2})"\
+        r"_(?P<exposure_id>\d+)"\
+        r"-seg(?P<segment>\d{3})"\
+        r"_(?P<detector>\w+)"
+
+    # Build list of filename types
+    filename_types = [stage_1_and_2, stage_3_target_id, stage_3_source_id, stage_3_target_id_epoch, stage_3_source_id_epoch, time_series]
+
+    # Try to parse the filename
+    for filename_type in filename_types:
+
+        # If full filename, try using suffix
+        if not file_root_name:
+            filename_type += r"_(?P<suffix>{}).*".format('|'.join(FILE_SUFFIX_TYPES))
+
+        elements = re.compile(filename_type)
+        jwst_file = elements.match(filename)
+        if jwst_file is not None:
+            break
+
+    # Raise error if unable to parse the filename
+    try:
         filename_dict = jwst_file.groupdict()
-    else:
-        raise ValueError('Provided file {} does not follow JWST naming conventions (jw<PPPPP><OOO><VVV>_<GGSAA>_<EEEEE>_<detector>_<suffix>.fits)'.format(filename))
+    except AttributeError:
+        jdox_url = 'https://jwst-docs.stsci.edu/display/JDAT/File+Naming+Conventions+and+Data+Products'
+        raise ValueError('Provided file {} does not follow JWST naming conventions.  See {} for further information.'.format(filename, jdox_url))
 
     return filename_dict
 


Test patch:
diff --git a/jwql/tests/test_utils.py b/jwql/tests/test_utils.py
--- a/jwql/tests/test_utils.py
+++ b/jwql/tests/test_utils.py
@@ -5,6 +5,7 @@
 -------
 
     - Lauren Chambers
+    - Matthew Bourque
 
 Use
 ---
@@ -21,80 +22,164 @@
 
 from jwql.utils.utils import get_config, filename_parser
 
+
+FILENAME_PARSER_TEST_DATA = [
+
+# Test full path
+('/test/dir/to/the/file/jw90002/jw90002001001_02102_00001_nis_rateints.fits',
+    {'activity': '02',
+     'detector': 'nis',
+     'exposure_id': '00001',
+     'observation': '001',
+     'parallel_seq_id': '1',
+     'program_id': '90002',
+     'suffix': 'rateints',
+     'visit': '001',
+     'visit_group': '02'}),
+
+# Test full stage 1 and 2 filename
+('jw00327001001_02101_00002_nrca1_rate.fits',
+    {'activity': '01',
+     'detector': 'nrca1',
+     'exposure_id': '00002',
+     'observation': '001',
+     'parallel_seq_id': '1',
+     'program_id': '00327',
+     'suffix': 'rate',
+     'visit': '001',
+     'visit_group': '02'}),
+
+# Test root stage 1 and 2 filename
+('jw00327001001_02101_00002_nrca1',
+    {'activity': '01',
+     'detector': 'nrca1',
+     'exposure_id': '00002',
+     'observation': '001',
+     'parallel_seq_id': '1',
+     'program_id': '00327',
+     'visit': '001',
+     'visit_group': '02'}),
+
+# Test full stage 3 filename with target_id
+('jw80600-o009_t001_miri_f1130w_i2d.fits',
+    {'ac_id': 'o009',
+     'instrument': 'miri',
+     'optical_elements': 'f1130w',
+     'program_id': '80600',
+     'suffix': 'i2d',
+     'target_id': 't001'}),
+
+# Test full stage 3 filename with target_id and different ac_id
+('jw80600-c0001_t001_miri_f1130w_i2d.fits',
+    {'ac_id': 'c0001',
+     'instrument': 'miri',
+     'optical_elements': 'f1130w',
+     'program_id': '80600',
+     'suffix': 'i2d',
+     'target_id': 't001'}),
+
+# Test full stage 3 filename with source_id
+('jw80600-o009_s00001_miri_f1130w_i2d.fits',
+    {'ac_id': 'o009',
+     'instrument': 'miri',
+     'optical_elements': 'f1130w',
+     'program_id': '80600',
+     'source_id': 's00001',
+     'suffix': 'i2d'}),
+
+# Test stage 3 filename with target_id and epoch
+('jw80600-o009_t001-epoch1_miri_f1130w_i2d.fits',
+    {'ac_id': 'o009',
+     'instrument': 'miri',
+     'epoch': '1',
+     'optical_elements': 'f1130w',
+     'program_id': '80600',
+     'suffix': 'i2d',
+     'target_id': 't001'}),
+
+# Test stage 3 filename with source_id and epoch
+('jw80600-o009_s00001-epoch1_miri_f1130w_i2d.fits',
+    {'ac_id': 'o009',
+     'instrument': 'miri',
+     'epoch': '1',
+     'optical_elements': 'f1130w',
+     'program_id': '80600',
+     'source_id': 's00001',
+     'suffix': 'i2d'}),
+
+# Test root stage 3 filename with target_id
+('jw80600-o009_t001_miri_f1130w',
+    {'ac_id': 'o009',
+     'instrument': 'miri',
+     'optical_elements': 'f1130w',
+     'program_id': '80600',
+     'target_id': 't001'}),
+
+# Test root stage 3 filename with source_id
+('jw80600-o009_s00001_miri_f1130w',
+    {'ac_id': 'o009',
+     'instrument': 'miri',
+     'optical_elements': 'f1130w',
+     'program_id': '80600',
+     'source_id': 's00001'}),
+
+# Test full time series filename
+('jw00733003001_02101_00002-seg001_nrs1_rate.fits',
+    {'activity': '01',
+    'detector': 'nrs1',
+    'exposure_id': '00002',
+    'observation': '003',
+    'parallel_seq_id': '1',
+    'program_id': '00733',
+    'segment': '001',
+    'suffix': 'rate',
+    'visit': '001',
+    'visit_group': '02'}),
+
+# Test root time series filename
+('jw00733003001_02101_00002-seg001_nrs1',
+    {'activity': '01',
+    'detector': 'nrs1',
+    'exposure_id': '00002',
+    'observation': '003',
+    'parallel_seq_id': '1',
+    'program_id': '00733',
+    'segment': '001',
+    'visit': '001',
+    'visit_group': '02'})
+]
+
+
 @pytest.mark.xfail
 def test_get_config():
-    '''Assert that the ``get_config`` function successfuly creates a
+    """Assert that the ``get_config`` function successfuly creates a
     dictionary.
-    '''
+    """
     settings = get_config()
     assert isinstance(settings, dict)
 
 
-def test_filename_parser_filename():
-    '''Generate a dictionary with parameters from a JWST filename.
+@pytest.mark.parametrize('filename, solution', FILENAME_PARSER_TEST_DATA)
+def test_filename_parser(filename, solution):
+    """Generate a dictionary with parameters from a JWST filename.
     Assert that the dictionary matches what is expected.
-    '''
-    filename = 'jw00327001001_02101_00002_nrca1_rate.fits'
-    filename_dict = filename_parser(filename)
-
-    correct_dict = {'activity': '01',
-                    'detector': 'nrca1',
-                    'exposure_id': '00002',
-                    'observation': '001',
-                    'parallel_seq_id': '1',
-                    'program_id': '00327',
-                    'suffix': 'rate',
-                    'visit': '001',
-                    'visit_group': '02'}
-
-    assert filename_dict == correct_dict
-
-def test_filename_parser_filename_root():
-    '''Generate a dictionary with parameters from a JWST filename.
-    Assert that the dictionary matches what is expected.
-    '''
-    filename = 'jw00327001001_02101_00002_nrca1'
-    filename_dict = filename_parser(filename)
-
-    correct_dict = {'activity': '01',
-                    'detector': 'nrca1',
-                    'exposure_id': '00002',
-                    'observation': '001',
-                    'parallel_seq_id': '1',
-                    'program_id': '00327',
-                    'visit': '001',
-                    'visit_group': '02'}
-
-    assert filename_dict == correct_dict
-
-
-def test_filename_parser_filepath():
-    '''Generate a dictionary with parameters from a JWST filepath
-    (not just the basename). Assert that the dictionary matches what
-    is expected.
-    '''
-    filepath = '/test/dir/to/the/file/jw90002/jw90002001001_02102_00001_nis_rateints.fits'
-    filename_dict = filename_parser(filepath)
-
-    correct_dict = {'activity': '02',
-                    'detector': 'nis',
-                    'exposure_id': '00001',
-                    'observation': '001',
-                    'parallel_seq_id': '1',
-                    'program_id': '90002',
-                    'suffix': 'rateints',
-                    'visit': '001',
-                    'visit_group': '02'}
-
-    assert filename_dict == correct_dict
+
+    Parameters
+    ----------
+    filename : str
+        The filename to test (e.g. ``jw00327001001_02101_00002_nrca1_rate.fits``)
+    solution : dict
+        A dictionary of the expected result
+    """
+
+    assert filename_parser(filename) == solution
 
 
 def test_filename_parser_nonJWST():
-    '''Attempt to generate a file parameter dictionary from a file
+    """Attempt to generate a file parameter dictionary from a file
     that is not formatted in the JWST naming convention. Ensure the
     appropriate error is raised.
-    '''
-    with pytest.raises(ValueError,
-                       match=r'Provided file .+ does not follow JWST naming conventions \(jw<PPPPP><OOO><VVV>_<GGSAA>_<EEEEE>_<detector>_<suffix>\.fits\)'):
+    """
+    with pytest.raises(ValueError):
         filename = 'not_a_jwst_file.fits'
         filename_parser(filename)