Repository: spacetelescope/jwql

Issue:
bias monitor crashing with indexing error
Looks like maybe the 'cal_image' value from the database is empty? 

BiasMonitorPlots(instrument)
  File "monitor_bias_bokeh.py", line 248, in __init__
    self.zerothgroup_plots[self.aperture] = ZerothGroupImage(self.db.latest_data).figure
  File "monitor_bias_bokeh.py", line 614, in __init__
    self.create_figure()
  File "monitor_bias_bokeh.py", line 619, in create_figure
    if os.path.isfile(self.data['cal_image'].iloc[0]):
  File "site-packages/pandas/core/indexing.py", line 1103, in __getitem__
    return self._getitem_axis(maybe_callable, axis=axis)
  File "site-packages/pandas/core/indexing.py", line 1656, in _getitem_axis
    self._validate_integer(key, axis)
  File "site-packages/pandas/core/indexing.py", line 1589, in _validate_integer
    raise IndexError("single positional indexer is out-of-bounds")
IndexError: single positional indexer is out-of-bounds


Discussion:

Resolving patch:
diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_bias_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_bias_bokeh.py
--- a/jwql/website/apps/jwql/monitor_pages/monitor_bias_bokeh.py
+++ b/jwql/website/apps/jwql/monitor_pages/monitor_bias_bokeh.py
@@ -101,7 +101,8 @@ def retrieve_trending_data(self, aperture):
                                           self.stats_table.amp3_odd_med,
                                           self.stats_table.amp4_even_med,
                                           self.stats_table.amp4_odd_med,
-                                          self.stats_table.expstart) \
+                                          self.stats_table.expstart,
+                                          self.stats_table.uncal_filename) \
             .filter(self.stats_table.aperture == aperture) \
             .order_by(self.stats_table.expstart) \
             .all()
@@ -113,7 +114,10 @@ def retrieve_trending_data(self, aperture):
                                                                       'amp2_even_med', 'amp2_odd_med',
                                                                       'amp3_even_med', 'amp3_odd_med',
                                                                       'amp4_even_med', 'amp4_odd_med',
-                                                                      'expstart_str'])
+                                                                      'expstart_str', 'uncal_filename'])
+        uncal_basename = [os.path.basename(e) for e in self.trending_data['uncal_filename']]
+        self.trending_data['uncal_filename'] = uncal_basename
+
         # Add a column of expstart values that are datetime objects
         format_data = "%Y-%m-%dT%H:%M:%S.%f"
         datetimes = [datetime.strptime(entry, format_data) for entry in self.trending_data['expstart_str']]
@@ -128,12 +132,13 @@ def retrieve_latest_data(self, aperture):
         aperture : str
             Aperture name (e.g. NRCA1_FULL)
         """
-        subq = (session.query(func.max(self.stats_table.expstart).label("max_created")) \
-        .filter(self.stats_table.aperture == aperture) \
+        subq = (session.query(self.stats_table.aperture, func.max(self.stats_table.expstart).label("max_created")) \
+        .group_by(self.stats_table.aperture)
         .subquery()
         )
 
-        query = (session.query(self.stats_table.uncal_filename,
+        query = (session.query(self.stats_table.aperture,
+                               self.stats_table.uncal_filename,
                                self.stats_table.cal_filename,
                                self.stats_table.cal_image,
                                self.stats_table.expstart,
@@ -142,6 +147,7 @@ def retrieve_latest_data(self, aperture):
                                self.stats_table.counts,
                                self.stats_table.bin_centers,
                                self.stats_table.entry_date)
+                     .filter(self.stats_table.aperture == aperture)
                      .order_by(self.stats_table.entry_date) \
                      .join(subq, self.stats_table.expstart == subq.c.max_created)
                      )
@@ -152,10 +158,10 @@ def retrieve_latest_data(self, aperture):
         # Put the returned data in a dataframe. Include only the most recent entry.
         # The query has already filtered to include only entries using the latest
         # expstart value.
-        self.latest_data = pd.DataFrame(latest_data[-1:], columns=['uncal_filename', 'cal_filename',
-                                                                  'cal_image', 'expstart_str', 'collapsed_rows',
-                                                                  'collapsed_columns', 'counts', 'bin_centers',
-                                                                  'entry_date'])
+        self.latest_data = pd.DataFrame(latest_data[-1:], columns=['aperture', 'uncal_filename', 'cal_filename',
+                                                                   'cal_image', 'expstart_str', 'collapsed_rows',
+                                                                   'collapsed_columns', 'counts', 'bin_centers',
+                                                                   'entry_date'])
         # Add a column of expstart values that are datetime objects
         format_data = "%Y-%m-%dT%H:%M:%S.%f"
         datetimes = [datetime.strptime(entry, format_data) for entry in self.latest_data['expstart_str']]
@@ -535,7 +541,7 @@ def create_amp_plot(self, amp_num, amp_data):
 
         if len(amp_data["expstart"]) > 0:
             plot = figure(title=title_str, tools='pan,box_zoom,reset,wheel_zoom,save',
-                      background_fill_color="#fafafa")
+                          background_fill_color="#fafafa")
             source = ColumnDataSource(amp_data)
             even_col = f'amp{amp_num}_even_med'
             odd_col = f'amp{amp_num}_odd_med'
@@ -561,12 +567,13 @@ def create_amp_plot(self, amp_num, amp_data):
             # interpret the format codes as html tags and crash with errors such as:
             # "Encountered unknown tag 'd'. Jinja was looking for the following tags: 'endblock'.
             # The innermost block that needs to be closed is 'block'"
-            hover_tool = HoverTool(tooltips=[('Even col bias:', f'@{even_col}'),
+            hover_tool = HoverTool(tooltips=[('File:', '@uncal_filename'),
+                                             ('Even col bias:', f'@{even_col}'),
                                              ('Odd col bias:', f'@{odd_col}'),
                                              ('Date:', '@expstart_str')
                                              ]
                                    )
-            hover_tool.formatters = {'@expstart': 'datetime'}
+            #hover_tool.formatters = {'@expstart': 'datetime'}
             plot.tools.append(hover_tool)
             plot.xaxis.axis_label = x_label
             plot.yaxis.axis_label = y_label
@@ -585,7 +592,7 @@ def create_plots(self):
         # Create one plot per amplifier
         for amp_num in range(1, 5):
             cols_to_use = [col for col in self.data.columns if str(amp_num) in col]
-            cols_to_use.append('expstart')
+            cols_to_use.extend(['expstart', 'expstart_str', 'uncal_filename'])
             subframe = self.data[cols_to_use]
             self.plots[amp_num] = self.create_amp_plot(amp_num, subframe)
 
@@ -616,22 +623,28 @@ def __init__(self, data):
     def create_figure(self):
         """Create the Bokeh figure
         """
-        if len(self.data['cal_image']) > 0 and os.path.isfile(self.data['cal_image'].iloc[0]):
-            image = read_png(self.data['cal_image'].iloc[0])
-
-            datestr = self.data['expstart_str'].iloc[0]
-
-            # Display the 32-bit RGBA image
-            ydim, xdim = image.shape
-            dim = max(xdim, ydim)
-            self.figure = figure(title=f'Calibrated Zeroth Group of Most Recent Dark: {datestr}', x_range=(0, xdim), y_range=(0, ydim),
-                                 tools='pan,box_zoom,reset,wheel_zoom,save')
-            self.figure.image_rgba(image=[image], x=0, y=0, dw=xdim, dh=ydim)
-            self.figure.xaxis.visible = False
-            self.figure.yaxis.visible = False
+        if len(self.data['cal_image']) > 0:
+            if os.path.isfile(self.data['cal_image'].iloc[0]):
+                image = read_png(self.data['cal_image'].iloc[0])
+
+                datestr = self.data['expstart_str'].iloc[0]
+
+                # Display the 32-bit RGBA image
+                ydim, xdim = image.shape
+                dim = max(xdim, ydim)
+                self.figure = figure(title=f'Calibrated Zeroth Group of Most Recent Dark: {datestr}', x_range=(0, xdim), y_range=(0, ydim),
+                                     tools='pan,box_zoom,reset,wheel_zoom,save')
+                self.figure.image_rgba(image=[image], x=0, y=0, dw=xdim, dh=ydim)
+                self.figure.xaxis.visible = False
+                self.figure.yaxis.visible = False
+            else:
+                # If the listed file is missing, create an empty plot
+                self.figure = PlaceholderPlot('Calibrated Zeroth Group of Most Recent Dark', '', '').plot
+                self.figure.xaxis.visible = False
+                self.figure.yaxis.visible = False
 
         else:
-            # If the given file is missing, create an empty plot
+            # If no file is given, create an empty plot
             self.figure = PlaceholderPlot('Calibrated Zeroth Group of Most Recent Dark', '', '').plot
             self.figure.xaxis.visible = False
             self.figure.yaxis.visible = False


Test patch: