Repository: spacetelescope/jwql

Issue:
Add more descriptive information to the image and exposure pages
Currently, the image and exposure pages have very minimal descriptive information for the file(s) shown: only proposal, observation, visit, and detector (for image pages).

It would be helpful to include some more high-level description for context: the exposure type, proposal category, filter/grating, etc.  Essentially, any field that appears in filtering or sorting elsewhere in the application would be nice to have on the image pages.



Discussion:
NIRSpec reports that this is high priority
_This issue is tracked on JIRA as [JWQL-144](https://jira.stsci.edu/browse/JWQL-144)._

Resolving patch:
diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py
--- a/jwql/website/apps/jwql/data_containers.py
+++ b/jwql/website/apps/jwql/data_containers.py
@@ -43,6 +43,7 @@
 from django.conf import settings
 from django.contrib import messages
 from django.core.exceptions import ObjectDoesNotExist
+from django.db.models.query import QuerySet
 import numpy as np
 from operator import itemgetter
 import pandas as pd
@@ -360,6 +361,145 @@ def get_acknowledgements():
     return acknowledgements
 
 
+def get_additional_exposure_info(root_file_infos, image_info):
+    """Create dictionaries of basic exposure information from an exposure's
+    RootFileInfo entry, as well as header information. Originally designed to
+    be used in jwql.website.apps.jwql.views.view_image()
+
+    Parameters
+    ----------
+    root_file_infos : jwql.website.apps.jwql.models.RootFileInfo or django.db.models.query.QuerySet
+        RootFileInfo for a particular file base name, or a QuerySet of RootFileInfos for
+        an exposure base name.
+
+    image_info : : dict
+        A dictionary containing various information for the given
+        ``file_root``.
+
+    Returns
+    -------
+    basic_info : dict
+        Dictionary of information about the file/exposure
+
+    additional_info : dict
+        Dictionary of extra information about the file/exposure
+    """
+    # Get headers from the file so we can pass along info that is common to all
+    # suffixes. The order of possible_suffixes_to_use is itentional, because the
+    # uncal file will not have info on the pipeline version used, and so we would
+    # rather grab information from the rate or cal files.
+    possible_suffixes_to_use = np.array(['rate', 'rateints', 'cal', 'calints', 'uncal'])
+    existing_suffixes = np.array([suffix in image_info['suffixes'] for suffix in possible_suffixes_to_use])
+
+    if isinstance(root_file_infos, QuerySet):
+        root_file_info = root_file_infos[0]
+        filter_value = '/'.join(set([e.filter for e in root_file_infos]))
+        pupil_value = '/'.join(set([e.pupil for e in root_file_infos]))
+        grating_value = '/'.join(set([e.grating for e in root_file_infos]))
+    elif isinstance(root_file_infos, RootFileInfo):
+        root_file_info = root_file_infos
+        filter_value = root_file_info.filter
+        pupil_value = root_file_info.pupil
+        grating_value = root_file_info.grating
+
+    # Initialize dictionary of file info to show at the top of the page, along
+    # with another for info that will be in the collapsible text box.
+    basic_info = {'exp_type': root_file_info.exp_type,
+                  'category': 'N/A',
+                  'visit_status': 'N/A',
+                  'subarray': root_file_info.subarray,
+                  'filter': filter_value
+                  }
+
+    # The order of the elements is important here, in that the webpage displays
+    # them in the order they are here, and we've set this order to try and group
+    # together related keywords.
+    if isinstance(root_file_infos, QuerySet):
+        additional_info = {'READPATT': root_file_info.read_patt,
+                           'TITLE': 'N/A',
+                           'NGROUPS': 'N/A',
+                           'PI_NAME': 'N/A',
+                           'NINTS': 'N/A',
+                           'TARGNAME': 'N/A',
+                           'EXPTIME': 'N/A',
+                           'TARG_RA': 'N/A',
+                           'CAL_VER': 'N/A',
+                           'TARG_DEC': 'N/A',
+                           'CRDS context': 'N/A',
+                           'PA_V3': 'N/A',
+                           'EXPSTART': root_file_info.expstart
+                           }
+    elif isinstance(root_file_infos, RootFileInfo):
+        additional_info = {'READPATT': root_file_info.read_patt,
+                           'TITLE': 'N/A',
+                           'NGROUPS': 'N/A',
+                           'PI_NAME': 'N/A',
+                           'NINTS': 'N/A',
+                           'TARGNAME': 'N/A',
+                           'EXPTIME': 'N/A',
+                           'RA_REF': 'N/A',
+                           'CAL_VER': 'N/A',
+                           'DEC_REF': 'N/A',
+                           'CRDS context': 'N/A',
+                           'ROLL_REF': 'N/A',
+                           'EXPSTART': root_file_info.expstart
+                           }
+
+    # Deal with instrument-specific parameters
+    if root_file_info.instrument == 'NIRSpec':
+        basic_info['grating'] = grating_value
+
+    if root_file_info.instrument in ['NIRCam', 'NIRISS']:
+        basic_info['pupil'] = pupil_value
+
+    # If any of the desired files are present, get the headers and populate the header
+    # info dictionary
+    if any(existing_suffixes):
+        suffix = possible_suffixes_to_use[existing_suffixes][0]
+        filename = f'{root_file_info.root_name}_{suffix}.fits'
+
+        # get_image_info() has already globbed over the directory with the files and
+        # returned the list of existing suffixes, so we shouldn't need to check for
+        # file existence here.
+        file_path = filesystem_path(filename, check_existence=True)
+
+        header = fits.getheader(file_path)
+        header_sci = fits.getheader(file_path, 1)
+
+        basic_info['category'] = header['CATEGORY']
+        basic_info['visit_status'] = header['VISITSTA']
+        additional_info['NGROUPS'] = header['NGROUPS']
+        additional_info['NINTS'] = header['NINTS']
+        additional_info['EXPTIME'] = header['EFFEXPTM']
+        additional_info['TITLE'] = header['TITLE']
+        additional_info['PI_NAME'] = header['PI_NAME']
+        additional_info['TARGNAME'] = header['TARGPROP']
+
+        # For the exposure level (i.e. multiple files) present the target
+        # RA and Dec. For the image level, give RA_REF, DEC_REF, since those
+        # are specific to the detector. Similarly, for the exposure level, show
+        # PA_V3, which applies to all detectors. At the image level, show
+        # ROLL_REF, which is detector-specific.
+        if isinstance(root_file_infos, QuerySet):
+            additional_info['TARG_RA'] = header['TARG_RA']
+            additional_info['TARG_DEC'] = header['TARG_DEC']
+            additional_info['PA_V3'] = header_sci['PA_V3']
+        elif isinstance(root_file_infos, RootFileInfo):
+            additional_info['RA_REF'] = header_sci['RA_REF']
+            additional_info['DEC_REF'] = header_sci['DEC_REF']
+            additional_info['ROLL_REF'] = header_sci['ROLL_REF']
+
+        additional_info['CAL_VER'] = 'N/A'
+        additional_info['CRDS context'] = 'N/A'
+
+        # Pipeline version and CRDS context info are not in uncal files
+        if suffix != 'uncal':
+            additional_info['CAL_VER'] = header['CAL_VER']
+            additional_info['CRDS context'] = header['CRDS_CTX']
+
+    return basic_info, additional_info
+
+
 def get_all_proposals():
     """Return a list of all proposals that exist in the filesystem.
 
diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py
--- a/jwql/website/apps/jwql/views.py
+++ b/jwql/website/apps/jwql/views.py
@@ -51,21 +51,25 @@
 import operator
 import socket
 
+from astropy.time import Time
 from bokeh.layouts import layout
 from bokeh.embed import components
 from django.core.paginator import Paginator
 from django.http import HttpResponse, JsonResponse
 from django.shortcuts import redirect, render
+import numpy as np
 from sqlalchemy import inspect
 
 from jwql.database.database_interface import load_connection
 from jwql.utils import monitor_utils
 from jwql.utils.interactive_preview_image import InteractivePreviewImg
 from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE, URL_DICT, QUERY_CONFIG_TEMPLATE, QueryConfigKeys
-from jwql.utils.utils import filename_parser, get_base_url, get_config, get_rootnames_for_instrument_proposal, query_unformat
+from jwql.utils.utils import filename_parser, filesystem_path, get_base_url, get_config
+from jwql.utils.utils import get_rootnames_for_instrument_proposal, query_unformat
 
 from .data_containers import build_table
 from .data_containers import get_acknowledgements
+from .data_containers import get_additional_exposure_info
 from .data_containers import get_available_suffixes
 from .data_containers import get_anomaly_form
 from .data_containers import get_dashboard_components
@@ -1169,6 +1173,13 @@ def view_exposure(request, inst, group_root):
     root_file_info = RootFileInfo.objects.filter(root_name__startswith=group_root)
     viewed = all([rf.viewed for rf in root_file_info])
 
+    # Convert expstart from MJD to a date
+    expstart_str = Time(root_file_info[0].expstart, format='mjd').to_datetime().strftime('%d %b %Y %H:%M')
+
+    # Create one dict of info to show at the top of the page, and another dict of info
+    # to show in the collapsible text box.
+    basic_info, additional_info = get_additional_exposure_info(root_file_info, image_info)
+
     # Build the context
     context = {'base_url': get_base_url(),
                'group_root_list': group_root_list,
@@ -1182,7 +1193,10 @@ def view_exposure(request, inst, group_root):
                'total_ints': image_info['total_ints'],
                'detectors': sorted(image_info['detectors']),
                'form': form,
-               'marked_viewed': viewed}
+               'marked_viewed': viewed,
+               'expstart_str': expstart_str,
+               'basic_info': basic_info,
+               'additional_info': additional_info}
 
     return render(request, template, context)
 
@@ -1258,6 +1272,13 @@ def view_image(request, inst, file_root):
     # Get our current views RootFileInfo model and send our "viewed/new" information
     root_file_info = RootFileInfo.objects.get(root_name=file_root)
 
+    # Convert expstart from MJD to a date
+    expstart_str = Time(root_file_info.expstart, format='mjd').to_datetime().strftime('%d %b %Y %H:%M')
+
+    # Create one dict of info to show at the top of the page, and another dict of info
+    # to show in the collapsible text box.
+    basic_info, additional_info = get_additional_exposure_info(root_file_info, image_info)
+
     # Build the context
     context = {'base_url': get_base_url(),
                'file_root_list': file_root_list,
@@ -1270,6 +1291,9 @@ def view_image(request, inst, file_root):
                'available_ints': image_info['available_ints'],
                'total_ints': image_info['total_ints'],
                'form': form,
-               'marked_viewed': root_file_info.viewed}
+               'marked_viewed': root_file_info.viewed,
+               'expstart_str': expstart_str,
+               'basic_info': basic_info,
+               'additional_info': additional_info}
 
     return render(request, template, context)


Test patch:
diff --git a/jwql/tests/test_data_containers.py b/jwql/tests/test_data_containers.py
--- a/jwql/tests/test_data_containers.py
+++ b/jwql/tests/test_data_containers.py
@@ -43,8 +43,9 @@
 
 if not ON_GITHUB_ACTIONS:
     from jwql.utils.utils import get_config  # noqa: E402 (module level import not at top of file)
+    from jwql.website.apps.jwql.models import RootFileInfo
 
-
+    
 @pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to django models.')
 def test_build_table():
     tab = data_containers.build_table('filesystem_general')
@@ -127,6 +128,58 @@ def test_get_acknowledgements():
     assert len(acknowledgements) > 0
 
 
+@pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to django models.')
+def test_get_additional_exposure_info():
+    """Tests ``get_additional_exposure_info`` function."""
+    # Test an exposure-level case
+    group_root = 'jw01068002001_02102_00008'
+    image_info = data_containers.get_image_info(group_root)
+    root_file_info = RootFileInfo.objects.filter(root_name__startswith=group_root)
+    basic, additional = data_containers.get_additional_exposure_info(root_file_info, image_info)
+    expected_basic = {'exp_type': 'NRC_IMAGE',
+                      'category': 'COM',
+                      'visit_status': 'SUCCESSFUL',
+                      'subarray': 'SUB320',
+                      'pupil': 'CLEAR'}
+    # We can only test a subset of the keys in additional, since things like the pipeline version,
+    # crds context, etc can change over time.
+    expected_additional = {'READPATT': 'RAPID',
+                           'TITLE': 'NIRCam Subarray-Mode Commissioning, CAR NIRCam-019',
+                           'NGROUPS': 10,
+                           'PI_NAME': 'Hilbert, Bryan',
+                           'NINTS': 10,
+                           'TARGNAME': 'GP2-JMAG14-STAR-OFFSET',
+                           'EXPTIME': 106.904,
+                           'EXPSTART': 59714.6163261875}
+    for key in expected_basic:
+        assert basic[key] == expected_basic[key]
+    for key in expected_additional:
+        assert additional[key] == expected_additional[key]
+
+    # Test an image-level case
+    file_root = 'jw01022016001_03101_00001_nrs1'
+    image_info = data_containers.get_image_info(file_root)
+    root_file_info = RootFileInfo.objects.get(root_name=file_root)
+    basic, additional = data_containers.get_additional_exposure_info(root_file_info, image_info)
+    expected_basic = {'exp_type': 'NRS_IFU',
+                      'category': 'COM',
+                      'visit_status': 'SUCCESSFUL',
+                      'subarray': 'FULL',
+                      'filter': 'F100LP',
+                      'grating': 'G140H'}
+    expected_additional = {'READPATT': 'NRSRAPID',
+                           'TITLE': 'CAR FGS-017 Straylight for Moving Targets (All SIs)',
+                           'NGROUPS': 13,
+                           'PI_NAME': 'Stansberry, John A.',
+                           'NINTS': 2,
+                           'TARGNAME': 'JUPITER',
+                           'EXPTIME': 279.156,
+                           'EXPSTART': 59764.77659749352}
+    assert basic == expected_basic
+    for key in expected_additional:
+        assert additional[key] == expected_additional[key]
+
+
 @pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to central storage.')
 def test_get_all_proposals():
     """Tests the ``get_all_proposals`` function."""
@@ -146,6 +199,7 @@ def test_get_all_proposals():
                            (['uncal', 'rate', 'o001_crf', 'o006_crfints', 'bad'], {'bad'})),
                           (False, ['rate', 'uncal', 'bad', 'o006_crfints', 'o001_crf'],
                            ['uncal', 'rate', 'o001_crf', 'o006_crfints', 'bad'])])
+
 def test_get_available_suffixes(untracked, input_suffixes, expected):
     result = data_containers.get_available_suffixes(
         input_suffixes, return_untracked=untracked)