Repository: spacetelescope/jwql

Issue:
"original" filetypes are being picked up by the web app
The preview image pages of the web app show `original` as a filetype option, but JWQL doesn't support these filetypes as I believe they are some sort of copy of a file that the archive uses that is not relevant to the instrument teams.  We should fix the code as to ignore these filetypes. 


Discussion:

Resolving patch:
diff --git a/jwql/jwql_monitors/generate_preview_images.py b/jwql/jwql_monitors/generate_preview_images.py
--- a/jwql/jwql_monitors/generate_preview_images.py
+++ b/jwql/jwql_monitors/generate_preview_images.py
@@ -35,7 +35,7 @@
 import numpy as np
 
 from jwql.utils import permissions
-from jwql.utils.constants import NIRCAM_LONGWAVE_DETECTORS, NIRCAM_SHORTWAVE_DETECTORS
+from jwql.utils.constants import IGNORED_SUFFIXES, NIRCAM_LONGWAVE_DETECTORS, NIRCAM_SHORTWAVE_DETECTORS
 from jwql.utils.logging_functions import configure_logging, log_info, log_fail
 from jwql.utils.preview_image import PreviewImage
 from jwql.utils.utils import get_config, filename_parser
@@ -643,7 +643,7 @@ def process_program(program):
     filenames = list(set(filenames))
 
     # Ignore "original" files
-    filenames = [item for item in filenames if '_original.fits' not in item]
+    filenames = [filename for filename in filenames if os.path.splitext(filename.split('_')[-1]) not in IGNORED_SUFFIXES]
 
     # Group together common exposures
     grouped_filenames = group_filenames(filenames)
diff --git a/jwql/utils/constants.py b/jwql/utils/constants.py
--- a/jwql/utils/constants.py
+++ b/jwql/utils/constants.py
@@ -215,6 +215,9 @@
 # Possible suffix types for guider exposures
 GUIDER_SUFFIX_TYPES = ['stream', 'stacked_uncal', 'image_uncal', 'stacked_cal', 'image_cal']
 
+# JWQL should ignore some filetypes in the filesystem.
+IGNORED_SUFFIXES = ['original', 'stream']
+
 # Instrument monitor database tables
 INSTRUMENT_MONITOR_DATABASE_TABLES = {
     'dark_monitor': ['<instrument>_dark_dark_current', '<instrument>_dark_pixel_stats', '<instrument>_dark_query_history'],
diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py
--- a/jwql/website/apps/jwql/data_containers.py
+++ b/jwql/website/apps/jwql/data_containers.py
@@ -47,7 +47,8 @@
 from jwql.instrument_monitors.nirspec_monitors.data_trending import dashboard as nirspec_dash
 from jwql.utils.utils import ensure_dir_exists, filesystem_path, filename_parser, get_config
 from jwql.utils.constants import MONITORS
-from jwql.utils.constants import INSTRUMENT_SERVICE_MATCH, JWST_INSTRUMENT_NAMES_MIXEDCASE, JWST_INSTRUMENT_NAMES_SHORTHAND
+from jwql.utils.constants import IGNORED_SUFFIXES, INSTRUMENT_SERVICE_MATCH, JWST_INSTRUMENT_NAMES_MIXEDCASE, \
+                                 JWST_INSTRUMENT_NAMES_SHORTHAND
 from jwql.utils.preview_image import PreviewImage
 from jwql.utils.credentials import get_mast_token
 
@@ -552,6 +553,9 @@ def get_filenames_by_proposal(proposal):
     proposal_string = '{:05d}'.format(int(proposal))
     filenames = glob.glob(os.path.join(FILESYSTEM_DIR, 'public', 'jw{}'.format(proposal_string), '*/*'))
     filenames.extend(glob.glob(os.path.join(FILESYSTEM_DIR, 'proprietary', 'jw{}'.format(proposal_string), '*/*')))
+
+    # Certain suffixes are always ignored
+    filenames = [filename for filename in filenames if os.path.splitext(filename).split('_')[-1] not in IGNORED_SUFFIXES]
     filenames = sorted([os.path.basename(filename) for filename in filenames])
 
     return filenames
@@ -578,6 +582,9 @@ def get_filenames_by_rootname(rootname):
 
     filenames = glob.glob(os.path.join(FILESYSTEM_DIR, 'public', proposal_dir, observation_dir, '{}*'.format(rootname)))
     filenames.extend(glob.glob(os.path.join(FILESYSTEM_DIR, 'proprietary', proposal_dir, observation_dir, '{}*'.format(rootname))))
+
+    # Certain suffixes are always ignored
+    filenames = [filename for filename in filenames if os.path.splitext(filename).split('_')[-1] not in IGNORED_SUFFIXES]
     filenames = sorted([os.path.basename(filename) for filename in filenames])
 
     return filenames
@@ -678,6 +685,9 @@ def get_image_info(file_root, rewrite):
     observation_dir = file_root[:13]
     filenames = glob.glob(os.path.join(FILESYSTEM_DIR, 'public', proposal_dir, observation_dir, '{}*.fits'.format(file_root)))
     filenames.extend(glob.glob(os.path.join(FILESYSTEM_DIR, 'proprietary', proposal_dir, observation_dir, '{}*.fits'.format(file_root))))
+
+    # Certain suffixes are always ignored
+    filenames = [filename for filename in filenames if os.path.splitext(filename)[0].split('_')[-1] not in IGNORED_SUFFIXES]
     image_info['all_files'] = filenames
 
     for filename in image_info['all_files']:
@@ -767,8 +777,9 @@ def get_preview_images_by_instrument(inst):
     # Parse the results to get the rootnames
     filenames = [result['filename'].split('.')[0] for result in results]
 
-    # Get list of all preview_images
+    # Get list of all preview_images and filter out any with ignored suffixes
     preview_images = glob.glob(os.path.join(PREVIEW_IMAGE_FILESYSTEM, '*', '*.jpg'))
+    preview_images = [item for item in preview_images if os.path.splitext(item).split('_')[-1] not in IGNORED_SUFFIXES]
 
     # Get subset of preview images that match the filenames
     preview_images = [os.path.basename(item) for item in preview_images if
@@ -798,6 +809,7 @@ def get_preview_images_by_proposal(proposal):
     proposal_string = '{:05d}'.format(int(proposal))
     preview_images = glob.glob(os.path.join(PREVIEW_IMAGE_FILESYSTEM, 'jw{}'.format(proposal_string), '*'))
     preview_images = [os.path.basename(preview_image) for preview_image in preview_images]
+    preview_images = [item for item in preview_images if os.path.splitext(item).split('_')[-1] not in IGNORED_SUFFIXES]
 
     return preview_images
 
@@ -825,6 +837,7 @@ def get_preview_images_by_rootname(rootname):
         'jw{}'.format(proposal),
         '{}*'.format(rootname))))
     preview_images = [os.path.basename(preview_image) for preview_image in preview_images]
+    preview_images = [item for item in preview_images if os.path.splitext(item).split('_')[-1] not in IGNORED_SUFFIXES]
 
     return preview_images
 
@@ -934,6 +947,7 @@ def get_thumbnails_all_instruments(parameters):
         results = response[0].json()['data']
 
         inst_filenames = [result['filename'].split('.')[0] for result in results]
+        inst_filenames = [filename for filename in inst_filenames if os.path.splitext(filename).split('_')[-1] not in IGNORED_SUFFIXES]
         filenames.extend(inst_filenames)
 
     # Get list of all thumbnails


Test patch: