Repository: spacetelescope/jwql

Issue:
Determine what to do with non-compliant filenames
With the updated filesystem (#292), I ran `test_utils.test_filename_parser_whole_filesystem` and it kicked back 1578 files that could not be parsed, some common examples include:

```
jw00799003001_gs-acq1_2019154181705_stream.fits
jw00799003001_gs-fg_2019154183426_cal.fits
jw10005-c1001_t001_nircam_f430m-maskrnd-sub320a430r_psfstack.fits
jw10005003001_02102_00001_nrcalong_c1001_psfalign.fits
jw90001001002_02101_00001_nis_o001_crf.fits
jw00729011001_gs-id_1_image_cal.fits
jw00729011001_gs-id_1_stacked_cal.fits
```

We should reach out to DMS folks and see what the dealio is with these types of files.


Discussion:
`*_gs_acq1_*` are FGS guide star acquisition data
`*_gs_fg_*` are FGS fine guide data
`*_gs_id_*` are FGS identification step data
@Johannes-Sahlmann Do you happen to know if those files are indeed in the proper naming convention and will permanently exist in the JWST archive? 
I am confident that they will exist permanently in the archive.
I do not know whether these examples are properly named.
Ok, thanks! 
@shanosborne and I came across some FGS files like this in the filesystem the other day (the `gs-id`, `gs-acq`, and `gs-fg` files). They are products of the GUIDER pipeline and, like Johannes said, we should definitely expect to see files like this in the archive. 

I can ping some FGS people to see if I can determine what the naming structure for them will be.

I'm also happy to ask about the other filetypes, @bourque, if you haven't already?
@laurenmarietta Sounds good, go for it! 

Resolving patch:
diff --git a/jwql/utils/constants.py b/jwql/utils/constants.py
--- a/jwql/utils/constants.py
+++ b/jwql/utils/constants.py
@@ -42,8 +42,22 @@
 JWST_MAST_SERVICES = ['Mast.Jwst.Filtered.{}'.format(value.title()) for value in
                       JWST_INSTRUMENT_NAMES]
 
-FILE_SUFFIX_TYPES = ['uncal', 'cal', 'rateints', 'rate', 'trapsfilled',
-                     'i2d', 'x1d', 's2d', 's3d', 'dark', 'ami', 'crf']
+GUIDER_SUFFIX_TYPES = ['stream', 'stacked_uncal', 'image_uncal', 'stacked_cal', 'image_cal']
+
+GENERIC_SUFFIX_TYPES = ['uncal', 'cal', 'rateints', 'rate', 'trapsfilled', 'i2d',
+                        'x1dints', 'x1d', 's2d', 's3d', 'dark', 'crfints',
+                        'crf', 'ramp', 'fitopt', 'bsubints', 'bsub', 'cat']
+
+TIME_SERIES_SUFFIX_TYPES = ['phot', 'whtlt']
+
+CORONAGRAPHY_SUFFIX_TYPES = ['psfstack', 'psfalign', 'psfsub']
+
+AMI_SUFFIX_TYPES = ['amiavg', 'aminorm', 'ami']
+
+# Concatenate all suffix types (ordered to ensure successful matching)
+FILE_SUFFIX_TYPES = GUIDER_SUFFIX_TYPES + GENERIC_SUFFIX_TYPES + \
+                    TIME_SERIES_SUFFIX_TYPES + CORONAGRAPHY_SUFFIX_TYPES + \
+                    AMI_SUFFIX_TYPES
 
 MONITORS = {
     'fgs': ['Bad Pixel Monitor'],
diff --git a/jwql/utils/utils.py b/jwql/utils/utils.py
--- a/jwql/utils/utils.py
+++ b/jwql/utils/utils.py
@@ -19,6 +19,14 @@
 
     Filename parser modified from Joe Hunkeler:
     https://gist.github.com/jhunkeler/f08783ca2da7bfd1f8e9ee1d207da5ff
+
+    Various documentation related to JWST filename conventions:
+    - https://jwst-docs.stsci.edu/display/JDAT/File+Naming+Conventions+and+Data+Products
+    - https://innerspace.stsci.edu/pages/viewpage.action?pageId=94092600
+    - https://innerspace.stsci.edu/pages/viewpage.action?spaceKey=SCSB&title=JWST+Science+Data+Products
+    - https://jwst-docs.stsci.edu/display/JDAT/Understanding+Associations?q=association%20candidate
+    - https://jwst-pipeline.readthedocs.io/en/stable/jwst/introduction.html#pipeline-step-suffix-definitions
+    - JWST TR JWST-STScI-004800, SM-12
  """
 
 import getpass
@@ -63,7 +71,8 @@ def filename_parser(filename):
     filename = os.path.basename(filename)
     file_root_name = (len(filename.split('.')) < 2)
 
-    # Stage 1 and 2 filenames, e.g. "jw80500012009_01101_00012_nrcalong_uncal.fits"
+    # Stage 1 and 2 filenames
+    # e.g. "jw80500012009_01101_00012_nrcalong_uncal.fits"
     stage_1_and_2 = \
         r"jw" \
         r"(?P<program_id>\d{5})"\
@@ -75,45 +84,64 @@ def filename_parser(filename):
         r"_(?P<exposure_id>\d+)"\
         r"_(?P<detector>((?!_)[\w])+)"
 
-    # Stage 3 filenames with target ID, e.g. "jw80600-o009_t001_miri_f1130w_i2d.fits"
+    # Stage 2c outlier detection filenames
+    # e.g. "jw94015002002_02108_00001_mirimage_o002_crf.fits"
+    stage_2c = \
+        r"jw" \
+        r"(?P<program_id>\d{5})" \
+        r"(?P<observation>\d{3})" \
+        r"(?P<visit>\d{3})" \
+        r"_(?P<visit_group>\d{2})" \
+        r"(?P<parallel_seq_id>\d{1})" \
+        r"(?P<activity>\w{2})" \
+        r"_(?P<exposure_id>\d+)" \
+        r"_(?P<detector>((?!_)[\w])+)"\
+        r"_(?P<ac_id>(o\d{3}|(c|a|r)\d{4}))"
+
+    # Stage 3 filenames with target ID
+    # e.g. "jw80600-o009_t001_miri_f1130w_i2d.fits"
     stage_3_target_id = \
         r"jw" \
         r"(?P<program_id>\d{5})"\
-        r"-(?P<ac_id>(o|c|a|r)\d{3,4})"\
+        r"-(?P<ac_id>(o\d{3}|(c|a|r)\d{4}))"\
         r"_(?P<target_id>(t)\d{3})"\
         r"_(?P<instrument>(nircam|niriss|nirspec|miri|fgs))"\
         r"_(?P<optical_elements>((?!_)[\w-])+)"
 
-    # Stage 3 filenames with source ID, e.g. "jw80600-o009_s00001_miri_f1130w_i2d.fits"
+    # Stage 3 filenames with source ID
+    # e.g. "jw80600-o009_s00001_miri_f1130w_i2d.fits"
     stage_3_source_id = \
         r"jw" \
         r"(?P<program_id>\d{5})"\
-        r"-(?P<ac_id>(o|c|a|r)\d{3,4})"\
+        r"-(?P<ac_id>(o\d{3}|(c|a|r)\d{4}))"\
         r"_(?P<source_id>(s)\d{5})"\
         r"_(?P<instrument>(nircam|niriss|nirspec|miri|fgs))"\
         r"_(?P<optical_elements>((?!_)[\w-])+)"
 
-    # Stage 3 filenames with target ID and epoch, e.g. "jw80600-o009_t001-epoch1_miri_f1130w_i2d.fits"
+    # Stage 3 filenames with target ID and epoch
+    # e.g. "jw80600-o009_t001-epoch1_miri_f1130w_i2d.fits"
     stage_3_target_id_epoch = \
         r"jw" \
         r"(?P<program_id>\d{5})"\
-        r"-(?P<ac_id>(o|c|a|r)\d{3,4})"\
+        r"-(?P<ac_id>(o\d{3}|(c|a|r)\d{4}))"\
         r"_(?P<target_id>(t)\d{3})"\
         r"-epoch(?P<epoch>\d{1})"\
         r"_(?P<instrument>(nircam|niriss|nirspec|miri|fgs))"\
         r"_(?P<optical_elements>((?!_)[\w-])+)"
 
-    # Stage 3 filenames with source ID and epoch, e.g. "jw80600-o009_s00001-epoch1_miri_f1130w_i2d.fits"
+    # Stage 3 filenames with source ID and epoch
+    # e.g. "jw80600-o009_s00001-epoch1_miri_f1130w_i2d.fits"
     stage_3_source_id_epoch = \
         r"jw" \
         r"(?P<program_id>\d{5})"\
-        r"-(?P<ac_id>(o|c|a|r)\d{3,4})"\
+        r"-(?P<ac_id>(o\d{3}|(c|a|r)\d{4}))"\
         r"_(?P<source_id>(s)\d{5})"\
         r"-epoch(?P<epoch>\d{1})"\
         r"_(?P<instrument>(nircam|niriss|nirspec|miri|fgs))"\
         r"_(?P<optical_elements>((?!_)[\w-])+)"
 
-    # Time series filenames, e.g. "jw00733003001_02101_00002-seg001_nrs1_rate.fits"
+    # Time series filenames
+    # e.g. "jw00733003001_02101_00002-seg001_nrs1_rate.fits"
     time_series = \
         r"jw" \
         r"(?P<program_id>\d{5})"\
@@ -126,24 +154,59 @@ def filename_parser(filename):
         r"-seg(?P<segment>\d{3})"\
         r"_(?P<detector>\w+)"
 
+    # Guider filenames
+    # e.g. "jw00729011001_gs-id_1_image_cal.fits" or
+    # "jw00799003001_gs-acq1_2019154181705_stream.fits"
+    guider = \
+        r"jw" \
+        r"(?P<program_id>\d{5})" \
+        r"(?P<observation>\d{3})" \
+        r"(?P<visit>\d{3})" \
+        r"_gs-(?P<guider_mode>(id|acq1|acq2|track|fg))" \
+        r"_((?P<date_time>\d{13})|(?P<guide_star_attempt_id>\d{1}))"
+
     # Build list of filename types
-    filename_types = [stage_1_and_2, stage_3_target_id, stage_3_source_id, stage_3_target_id_epoch, stage_3_source_id_epoch, time_series]
+    filename_types = [
+        stage_1_and_2,
+        stage_2c,
+        stage_3_target_id,
+        stage_3_source_id,
+        stage_3_target_id_epoch,
+        stage_3_source_id_epoch,
+        time_series,
+        guider]
+
+    filename_type_names = [
+        'stage_1_and_2',
+        'stage_2c',
+        'stage_3_target_id',
+        'stage_3_source_id',
+        'stage_3_target_id_epoch',
+        'stage_3_source_id_epoch',
+        'time_series',
+        'guider'
+    ]
 
     # Try to parse the filename
-    for filename_type in filename_types:
+    for filename_type, filename_type_name in zip(filename_types, filename_type_names):
 
         # If full filename, try using suffix
         if not file_root_name:
             filename_type += r"_(?P<suffix>{}).*".format('|'.join(FILE_SUFFIX_TYPES))
+        # If not, make sure the provided regex matches the entire filename root
+        else:
+            filename_type += r"$"
 
         elements = re.compile(filename_type)
         jwst_file = elements.match(filename)
         if jwst_file is not None:
+            name_match = filename_type_name
             break
 
     # Raise error if unable to parse the filename
     try:
         filename_dict = jwst_file.groupdict()
+        filename_dict['filename_type'] = name_match
     except AttributeError:
         jdox_url = 'https://jwst-docs.stsci.edu/display/JDAT/File+Naming+Conventions+and+Data+Products'
         raise ValueError('Provided file {} does not follow JWST naming conventions.  See {} for further information.'.format(filename, jdox_url))


Test patch:
diff --git a/jwql/tests/test_utils.py b/jwql/tests/test_utils.py
--- a/jwql/tests/test_utils.py
+++ b/jwql/tests/test_utils.py
@@ -27,132 +27,215 @@
 
 FILENAME_PARSER_TEST_DATA = [
 
-# Test full path
-('/test/dir/to/the/file/jw90002/jw90002001001_02102_00001_nis_rateints.fits',
-    {'activity': '02',
-     'detector': 'nis',
-     'exposure_id': '00001',
-     'observation': '001',
-     'parallel_seq_id': '1',
-     'program_id': '90002',
-     'suffix': 'rateints',
-     'visit': '001',
-     'visit_group': '02'}),
-
-# Test full stage 1 and 2 filename
-('jw00327001001_02101_00002_nrca1_rate.fits',
-    {'activity': '01',
-     'detector': 'nrca1',
-     'exposure_id': '00002',
-     'observation': '001',
-     'parallel_seq_id': '1',
-     'program_id': '00327',
-     'suffix': 'rate',
-     'visit': '001',
-     'visit_group': '02'}),
-
-# Test root stage 1 and 2 filename
-('jw00327001001_02101_00002_nrca1',
-    {'activity': '01',
-     'detector': 'nrca1',
-     'exposure_id': '00002',
-     'observation': '001',
-     'parallel_seq_id': '1',
-     'program_id': '00327',
-     'visit': '001',
-     'visit_group': '02'}),
-
-# Test full stage 3 filename with target_id
-('jw80600-o009_t001_miri_f1130w_i2d.fits',
-    {'ac_id': 'o009',
-     'instrument': 'miri',
-     'optical_elements': 'f1130w',
-     'program_id': '80600',
-     'suffix': 'i2d',
-     'target_id': 't001'}),
-
-# Test full stage 3 filename with target_id and different ac_id
-('jw80600-c0001_t001_miri_f1130w_i2d.fits',
-    {'ac_id': 'c0001',
-     'instrument': 'miri',
-     'optical_elements': 'f1130w',
-     'program_id': '80600',
-     'suffix': 'i2d',
-     'target_id': 't001'}),
-
-# Test full stage 3 filename with source_id
-('jw80600-o009_s00001_miri_f1130w_i2d.fits',
-    {'ac_id': 'o009',
-     'instrument': 'miri',
-     'optical_elements': 'f1130w',
-     'program_id': '80600',
-     'source_id': 's00001',
-     'suffix': 'i2d'}),
-
-# Test stage 3 filename with target_id and epoch
-('jw80600-o009_t001-epoch1_miri_f1130w_i2d.fits',
-    {'ac_id': 'o009',
-     'instrument': 'miri',
-     'epoch': '1',
-     'optical_elements': 'f1130w',
-     'program_id': '80600',
-     'suffix': 'i2d',
-     'target_id': 't001'}),
-
-# Test stage 3 filename with source_id and epoch
-('jw80600-o009_s00001-epoch1_miri_f1130w_i2d.fits',
-    {'ac_id': 'o009',
-     'instrument': 'miri',
-     'epoch': '1',
-     'optical_elements': 'f1130w',
-     'program_id': '80600',
-     'source_id': 's00001',
-     'suffix': 'i2d'}),
-
-# Test root stage 3 filename with target_id
-('jw80600-o009_t001_miri_f1130w',
-    {'ac_id': 'o009',
-     'instrument': 'miri',
-     'optical_elements': 'f1130w',
-     'program_id': '80600',
-     'target_id': 't001'}),
-
-# Test root stage 3 filename with source_id
-('jw80600-o009_s00001_miri_f1130w',
-    {'ac_id': 'o009',
-     'instrument': 'miri',
-     'optical_elements': 'f1130w',
-     'program_id': '80600',
-     'source_id': 's00001'}),
-
-# Test full time series filename
-('jw00733003001_02101_00002-seg001_nrs1_rate.fits',
-    {'activity': '01',
-    'detector': 'nrs1',
-    'exposure_id': '00002',
-    'observation': '003',
-    'parallel_seq_id': '1',
-    'program_id': '00733',
-    'segment': '001',
-    'suffix': 'rate',
-    'visit': '001',
-    'visit_group': '02'}),
-
-# Test root time series filename
-('jw00733003001_02101_00002-seg001_nrs1',
-    {'activity': '01',
-    'detector': 'nrs1',
-    'exposure_id': '00002',
-    'observation': '003',
-    'parallel_seq_id': '1',
-    'program_id': '00733',
-    'segment': '001',
-    'visit': '001',
-    'visit_group': '02'})
+    # Test full path
+    ('/test/dir/to/the/file/jw90002/jw90002001001_02102_00001_nis_rateints.fits',
+     {'activity': '02',
+      'detector': 'nis',
+      'exposure_id': '00001',
+      'filename_type': 'stage_1_and_2',
+      'observation': '001',
+      'parallel_seq_id': '1',
+      'program_id': '90002',
+      'suffix': 'rateints',
+      'visit': '001',
+      'visit_group': '02'}),
+
+    # Test full stage 1 and 2 filename
+    ('jw00327001001_02101_00002_nrca1_rate.fits',
+     {'activity': '01',
+      'detector': 'nrca1',
+      'exposure_id': '00002',
+      'filename_type': 'stage_1_and_2',
+      'observation': '001',
+      'parallel_seq_id': '1',
+      'program_id': '00327',
+      'suffix': 'rate',
+      'visit': '001',
+      'visit_group': '02'}),
+
+    # Test root stage 1 and 2 filename
+    ('jw00327001001_02101_00002_nrca1',
+     {'activity': '01',
+      'detector': 'nrca1',
+      'exposure_id': '00002',
+      'filename_type': 'stage_1_and_2',
+      'observation': '001',
+      'parallel_seq_id': '1',
+      'program_id': '00327',
+      'visit': '001',
+      'visit_group': '02'}),
+
+    # Test full stage 2c filename
+    ('jw94015002002_02108_00001_mirimage_o002_crf.fits',
+     {'ac_id': 'o002',
+      'activity': '08',
+      'detector': 'mirimage',
+      'exposure_id': '00001',
+      'filename_type': 'stage_2c',
+      'observation': '002',
+      'parallel_seq_id': '1',
+      'program_id': '94015',
+      'suffix': 'crf',
+      'visit': '002',
+      'visit_group': '02'}),
+
+    # Test root stage 2c filename
+    ('jw90001001003_02101_00001_nis_o001',
+     {'ac_id': 'o001',
+      'activity': '01',
+      'detector': 'nis',
+      'exposure_id': '00001',
+      'filename_type': 'stage_2c',
+      'observation': '001',
+      'parallel_seq_id': '1',
+      'program_id': '90001',
+      'visit': '003',
+      'visit_group': '02'}),
+
+    # Test full stage 3 filename with target_id
+    ('jw80600-o009_t001_miri_f1130w_i2d.fits',
+     {'ac_id': 'o009',
+      'filename_type': 'stage_3_target_id',
+      'instrument': 'miri',
+      'optical_elements': 'f1130w',
+      'program_id': '80600',
+      'suffix': 'i2d',
+      'target_id': 't001'}),
+
+    # Test full stage 3 filename with target_id and different ac_id
+    ('jw80600-c0001_t001_miri_f1130w_i2d.fits',
+     {'ac_id': 'c0001',
+      'filename_type': 'stage_3_target_id',
+      'instrument': 'miri',
+      'optical_elements': 'f1130w',
+      'program_id': '80600',
+      'suffix': 'i2d',
+      'target_id': 't001'}),
+
+    # Test full stage 3 filename with source_id
+    ('jw80600-o009_s00001_miri_f1130w_i2d.fits',
+     {'ac_id': 'o009',
+      'filename_type': 'stage_3_source_id',
+      'instrument': 'miri',
+      'optical_elements': 'f1130w',
+      'program_id': '80600',
+      'source_id': 's00001',
+      'suffix': 'i2d'}),
+
+    # Test stage 3 filename with target_id and epoch
+    ('jw80600-o009_t001-epoch1_miri_f1130w_i2d.fits',
+     {'ac_id': 'o009',
+      'filename_type': 'stage_3_target_id_epoch',
+      'instrument': 'miri',
+      'epoch': '1',
+      'optical_elements': 'f1130w',
+      'program_id': '80600',
+      'suffix': 'i2d',
+      'target_id': 't001'}),
+
+    # Test stage 3 filename with source_id and epoch
+    ('jw80600-o009_s00001-epoch1_miri_f1130w_i2d.fits',
+     {'ac_id': 'o009',
+      'filename_type': 'stage_3_source_id_epoch',
+      'instrument': 'miri',
+      'epoch': '1',
+      'optical_elements': 'f1130w',
+      'program_id': '80600',
+      'source_id': 's00001',
+      'suffix': 'i2d'}),
+
+    # Test root stage 3 filename with target_id
+    ('jw80600-o009_t001_miri_f1130w',
+     {'ac_id': 'o009',
+      'filename_type': 'stage_3_target_id',
+      'instrument': 'miri',
+      'optical_elements': 'f1130w',
+      'program_id': '80600',
+      'target_id': 't001'}),
+
+    # Test root stage 3 filename with source_id
+    ('jw80600-o009_s00001_miri_f1130w',
+     {'ac_id': 'o009',
+      'filename_type': 'stage_3_source_id',
+      'instrument': 'miri',
+      'optical_elements': 'f1130w',
+      'program_id': '80600',
+      'source_id': 's00001'}),
+
+    # Test full time series filename
+    ('jw00733003001_02101_00002-seg001_nrs1_rate.fits',
+     {'activity': '01',
+      'detector': 'nrs1',
+      'exposure_id': '00002',
+      'filename_type': 'time_series',
+      'observation': '003',
+      'parallel_seq_id': '1',
+      'program_id': '00733',
+      'segment': '001',
+      'suffix': 'rate',
+      'visit': '001',
+      'visit_group': '02'}),
+
+    # Test root time series filename
+    ('jw00733003001_02101_00002-seg001_nrs1',
+     {'activity': '01',
+      'detector': 'nrs1',
+      'exposure_id': '00002',
+      'filename_type': 'time_series',
+      'observation': '003',
+      'parallel_seq_id': '1',
+      'program_id': '00733',
+      'segment': '001',
+      'visit': '001',
+      'visit_group': '02'}),
+
+    # Test full guider ID filename
+    ('jw00729011001_gs-id_1_image_cal.fits',
+     {'date_time': None,
+      'filename_type': 'guider',
+      'guide_star_attempt_id': '1',
+      'guider_mode': 'id',
+      'observation': '011',
+      'program_id': '00729',
+      'suffix': 'image_cal',
+      'visit': '001'}),
+
+    # Test root guider ID filename
+    ('jw00327001001_gs-id_2',
+     {'date_time': None,
+      'filename_type': 'guider',
+      'guide_star_attempt_id': '2',
+      'guider_mode': 'id',
+      'observation': '001',
+      'program_id': '00327',
+      'visit': '001'}),
+
+    # Test full guider non-ID filename
+    ('jw86600048001_gs-fg_2016018175411_stream.fits',
+     {'date_time': '2016018175411',
+      'filename_type': 'guider',
+      'guide_star_attempt_id': None,
+      'guider_mode': 'fg',
+      'observation': '048',
+      'program_id': '86600',
+      'suffix': 'stream',
+      'visit': '001'}),
+
+    # Test root guider non-ID filename
+    ('jw00729011001_gs-acq2_2019155024808',
+     {'date_time': '2019155024808',
+      'filename_type': 'guider',
+      'guide_star_attempt_id': None,
+      'guider_mode': 'acq2',
+      'observation': '011',
+      'program_id': '00729',
+      'visit': '001'})
+
 ]
 
 
-@pytest.mark.xfail(reason='User must manually supply config file.')
+@pytest.mark.xfail(raises=FileNotFoundError,
+                   reason='User must manually supply config file.')
 def test_get_config():
     """Assert that the ``get_config`` function successfully creates a
     dictionary.
@@ -177,7 +260,7 @@ def test_filename_parser(filename, solution):
     assert filename_parser(filename) == solution
 
 
-@pytest.mark.xfail(reason='Known non-compliant files in filesystem')
+@pytest.mark.xfail(raises=ValueError, reason='Known non-compliant files in filesystem')
 def test_filename_parser_whole_filesystem():
     """Test the filename_parser on all files currently in the filesystem.
     """