Repository: spacetelescope/jwql

Issue:
Dark monitor crashing due to reported missing env packages
_Issue [JWQL-138](https://jira.stsci.edu/browse/JWQL-138) was created on JIRA by [Bryan Hilbert](https://jira.stsci.edu/secure/ViewProfile.jspa?name=hilbert):_

On Sept 13, the dark monitor crashed while trying to create png files. The error statement reports:

"RuntimeError: Neither firefox and geckodriver nor a variant of chromium browser and chromedriver are available on system PATH. You can install the former with 'conda install -c conda-forge firefox geckodriver'."

However, I've confirmed that both firefox and geckodriver are present in the jwql-3.9 environment. The environment was just deleted and re-created from scratch last week. Could it be that the web service is working with an old version of the environment? Except that both packages were present in the old version of the environment as well.


Discussion:
I've found a way to get this working, although I still don't understand why what we had been using no longer works. The packages are in the environment, and the correct path to the packages is in the PATH env variable. 

Anyway, when I supply an instance of a web driver in the call to export_png, it works. This suggests that for some reason it isn't able to find or create a web driver instance otherwise?

This works, as tested on the ops server:
```
from bokeh.plotting import figure
from bokeh.io import export_png
import numpy as np
from selenium import webdriver
from bokeh.models import LinearColorMapper
plot = figure(title='test',tools='')
image = np.zeros((200,200))
image[100:105, 100:105] = 1
ny, nx = image.shape
mapper = LinearColorMapper(palette='Viridis256', low=0 ,high=1.1)
imgplot = plot.image(image=[image], x=0, y=0, dw=nx, dh=ny, color_mapper=mapper, level="image")
options = webdriver.FirefoxOptions()
options.add_argument('-headless')
driver = webdriver.Firefox(options=options)
export_png(plot, filename='test.png', webdriver=driver)
driver.quit()
```
Further digging has revealed that this issue is coming from updated versions of selenium. With selenium at 4.9.1, everything works fine (without having to supply a webdriver instance). Using 4.10.0, bokeh's export_png fails. Just before that failure is a statement `The version of chrome cannot be detected. Trying with latest driver version`. And then in versions after 4.10.0, export_png fails without that statement. 

So it seems that in order to keep up with the latest version of selenium, we'll need to use the updated code above when creating png files.

Resolving patch:
diff --git a/jwql/instrument_monitors/common_monitors/dark_monitor.py b/jwql/instrument_monitors/common_monitors/dark_monitor.py
--- a/jwql/instrument_monitors/common_monitors/dark_monitor.py
+++ b/jwql/instrument_monitors/common_monitors/dark_monitor.py
@@ -84,7 +84,6 @@
 from astropy.modeling import models
 from astropy.stats import sigma_clipped_stats
 from astropy.time import Time
-from bokeh.io import export_png
 from bokeh.models import ColorBar, ColumnDataSource, HoverTool, Legend
 from bokeh.models import LinearColorMapper
 from bokeh.plotting import figure
@@ -106,7 +105,7 @@
 from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE, JWST_DATAPRODUCTS, RAPID_READPATTERNS
 from jwql.utils.logging_functions import log_info, log_fail
 from jwql.utils.permissions import set_permissions
-from jwql.utils.utils import copy_files, ensure_dir_exists, get_config, filesystem_path
+from jwql.utils.utils import copy_files, ensure_dir_exists, get_config, filesystem_path, save_png
 
 THRESHOLDS_FILE = os.path.join(os.path.split(__file__)[0], 'dark_monitor_file_thresholds.txt')
 
@@ -276,14 +275,14 @@ def create_mean_slope_figure(self, image, num_files, hotxy=None, deadxy=None, no
 
             # Create figure
             start_time = Time(float(self.query_start), format='mjd').tt.datetime.strftime("%m/%d/%Y")
-            end_time  = Time(float(self.query_end), format='mjd').tt.datetime.strftime("%m/%d/%Y")
+            end_time = Time(float(self.query_end), format='mjd').tt.datetime.strftime("%m/%d/%Y")
 
             self.plot = figure(title=f'{self.aperture}: {num_files} files. {start_time} to {end_time}', tools='')
             #                   tools='pan,box_zoom,reset,wheel_zoom,save')
             self.plot.x_range.range_padding = self.plot.y_range.range_padding = 0
 
             # Create the color mapper that will be used to scale the image
-            mapper = LinearColorMapper(palette='Viridis256', low=(img_med-5*img_dev) ,high=(img_med+5*img_dev))
+            mapper = LinearColorMapper(palette='Viridis256', low=(img_med - (5 * img_dev)), high=(img_med + (5 * img_dev)))
 
             # Plot image and add color bar
             imgplot = self.plot.image(image=[image], x=0, y=0, dw=nx, dh=ny,
@@ -292,13 +291,6 @@ def create_mean_slope_figure(self, image, num_files, hotxy=None, deadxy=None, no
             color_bar = ColorBar(color_mapper=mapper, width=8, title='DN/sec')
             self.plot.add_layout(color_bar, 'right')
 
-            # Add hover tool for all pixel values
-            #hover_tool = HoverTool(tooltips=[('(x, y):', '($x{int}, $y{int})'),
-            #                                 ('value:', '@image')
-            #                                 ],
-            #                       renderers=[imgplot])
-            #self.plot.tools.append(hover_tool)
-
             if (('FULL' in self.aperture) or ('_CEN' in self.aperture)):
 
                 if hotxy is not None:
@@ -338,22 +330,21 @@ def create_mean_slope_figure(self, image, num_files, hotxy=None, deadxy=None, no
                     base_start = Time(float(base_parts[3]), format='mjd').tt.datetime
                     base_end = Time(float(base_parts[5]), format='mjd').tt.datetime
                     base_start_time = base_start.strftime("%m/%d/%Y")
-                    base_end_time  = base_end.strftime("%m/%d/%Y")
+                    base_end_time = base_end.strftime("%m/%d/%Y")
                     legend_title = f'Compared to dark from {base_start_time} to {base_end_time}'
                 else:
                     legend_title = 'Compared to previous mean dark'
                 legend = Legend(items=[hot_legend, dead_legend, noisy_legend],
                                 location="center",
                                 orientation='vertical',
-                                title = legend_title)
+                                title=legend_title)
 
                 self.plot.add_layout(legend, 'below')
 
             # Save the plot in a png
-            export_png(self.plot, filename=output_filename)
+            save_png(self.plot, filename=output_filename)
             set_permissions(output_filename)
 
-
     def get_metadata(self, filename):
         """Collect basic metadata from a fits file
 
@@ -650,7 +641,7 @@ def overplot_bad_pix(self, pix_type, coords, values):
 
         sources[pix_type] = ColumnDataSource(data=dict(pixels_x=coords[0],
                                                        pixels_y=coords[1]
-                                                      )
+                                                       )
                                              )
 
         # Overplot the bad pixel locations
@@ -658,7 +649,7 @@ def overplot_bad_pix(self, pix_type, coords, values):
                                                  source=sources[pix_type], color=colors[pix_type])
 
         # Add to the legend
-        if  numpix > 0:
+        if numpix > 0:
             if numpix <= DARK_MONITOR_MAX_BADPOINTS_TO_PLOT:
                 text = f"{numpix} pix {adjective[pix_type]} than baseline"
             else:
diff --git a/jwql/utils/utils.py b/jwql/utils/utils.py
--- a/jwql/utils/utils.py
+++ b/jwql/utils/utils.py
@@ -45,14 +45,15 @@
 from bokeh.plotting import figure
 import numpy as np
 from PIL import Image
+from selenium import webdriver
 
 from jwql.utils import permissions
 from jwql.utils.constants import FILE_AC_CAR_ID_LEN, FILE_AC_O_ID_LEN, FILE_ACT_LEN, \
-                                 FILE_DATETIME_LEN, FILE_EPOCH_LEN, FILE_GUIDESTAR_ATTMPT_LEN_MIN, \
-                                 FILE_GUIDESTAR_ATTMPT_LEN_MAX, FILE_OBS_LEN, FILE_PARALLEL_SEQ_ID_LEN, \
-                                 FILE_PROG_ID_LEN, FILE_SEG_LEN, FILE_SOURCE_ID_LEN, FILE_SUFFIX_TYPES, \
-                                 FILE_TARG_ID_LEN, FILE_VISIT_GRP_LEN, FILE_VISIT_LEN, FILETYPE_WO_STANDARD_SUFFIX, \
-                                 JWST_INSTRUMENT_NAMES_SHORTHAND
+    FILE_DATETIME_LEN, FILE_EPOCH_LEN, FILE_GUIDESTAR_ATTMPT_LEN_MIN, \
+    FILE_GUIDESTAR_ATTMPT_LEN_MAX, FILE_OBS_LEN, FILE_PARALLEL_SEQ_ID_LEN, \
+    FILE_PROG_ID_LEN, FILE_SEG_LEN, FILE_SOURCE_ID_LEN, FILE_SUFFIX_TYPES, \
+    FILE_TARG_ID_LEN, FILE_VISIT_GRP_LEN, FILE_VISIT_LEN, FILETYPE_WO_STANDARD_SUFFIX, \
+    JWST_INSTRUMENT_NAMES_SHORTHAND
 
 __location__ = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
 
@@ -157,8 +158,7 @@ def create_png_from_fits(filename, outdir):
         plot.ygrid.visible = False
 
         # Create the color mapper that will be used to scale the image
-        #mapper = LogColorMapper(palette='Viridis256', low=(img_med-5*img_dev) ,high=(img_med+5*img_dev))
-        mapper = LogColorMapper(palette='Greys256', low=(img_med-5*img_dev) ,high=(img_med+5*img_dev))
+        mapper = LogColorMapper(palette='Greys256', low=(img_med - (5 * img_dev)), high=(img_med + (5 * img_dev)))
 
         # Plot image
         imgplot = plot.image(image=[image], x=0, y=0, dw=nx, dh=ny,
@@ -169,8 +169,8 @@ def create_png_from_fits(filename, outdir):
         plot.yaxis.visible = False
 
         # Save the plot in a png
-        output_filename = os.path.join(outdir, os.path.basename(filename).replace('fits','png'))
-        export_png(plot, filename=output_filename)
+        output_filename = os.path.join(outdir, os.path.basename(filename).replace('fits', 'png'))
+        save_png(plot, filename=output_filename)
         permissions.set_permissions(output_filename)
         return output_filename
     else:
@@ -749,13 +749,34 @@ def read_png(filename):
 
         # Copy the RGBA image into view, flipping it so it comes right-side up
         # with a lower-left origin
-        view[:,:,:] = np.flipud(np.asarray(rgba_img))
+        view[:, :, :] = np.flipud(np.asarray(rgba_img))
     else:
         view = None
     # Return the 2D version
     return img
 
 
+def save_png(fig, filename=''):
+    """Starting with selenium version 4.10.0, our testing has shown that on the JWQL
+    servers, we need to specify an instance of a web driver when exporting a Bokeh
+    figure as a png. This is a wrapper function that creates the web driver instance
+    and calls Bokeh's export_png function.
+
+    Parameters
+    ----------
+    fig : bokeh.plotting.figure
+        Bokeh figure to be saved as a png
+
+    filename : str
+        Filename to use for the png file
+    """
+    options = webdriver.FirefoxOptions()
+    options.add_argument('-headless')
+    driver = webdriver.Firefox(options=options)
+    export_png(fig, filename=filename, webdriver=driver)
+    driver.quit()
+
+
 def grouper(iterable, chunksize):
     """
     Take a list of items (iterable), and group it into chunks of chunksize, with the
diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py
--- a/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py
+++ b/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py
@@ -22,7 +22,7 @@
 from astropy.stats import sigma_clipped_stats
 from astropy.time import Time
 from bokeh.embed import components, file_html
-from bokeh.io import export_png, show
+from bokeh.io import show
 from bokeh.layouts import layout
 from bokeh.models import ColumnDataSource, DatetimeTickFormatter, HoverTool, Legend, LinearColorMapper, Panel, Tabs, Text, Title
 from bokeh.plotting import figure
@@ -40,7 +40,7 @@
 from jwql.utils.constants import BAD_PIXEL_MONITOR_MAX_POINTS_TO_PLOT, BAD_PIXEL_TYPES, DARKS_BAD_PIXEL_TYPES
 from jwql.utils.constants import DETECTOR_PER_INSTRUMENT, FLATS_BAD_PIXEL_TYPES, JWST_INSTRUMENT_NAMES_MIXEDCASE
 from jwql.utils.permissions import set_permissions
-from jwql.utils.utils import filesystem_path, get_config, read_png
+from jwql.utils.utils import filesystem_path, get_config, read_png, save_png
 
 SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
 OUTPUT_DIR = get_config()['outputs']
@@ -74,8 +74,6 @@ def __init__(self, instrument):
         # Get the relevant database tables
         self.identify_tables()
 
-        #self.detectors = get_unique_values_per_column(self.pixel_table, 'detector')
-
         self.detectors = sorted(DETECTOR_PER_INSTRUMENT[self.instrument])
         if self.instrument == 'miri':
             self.detectors = ['MIRIMAGE']
@@ -128,7 +126,6 @@ def modify_bokeh_saved_html(self):
 
         self._html = "".join(newlines)
 
-
     def run(self):
 
         # Right now, the aperture name in the query history table is used as the title of the
@@ -163,7 +160,7 @@ def run(self):
         # Insert into our html template and save
         template_dir = os.path.join(os.path.dirname(__file__), '../templates')
         template_file = os.path.join(template_dir, 'bad_pixel_monitor_savefile_basic.html')
-        temp_vars = {'inst': self.instrument, 'plot_script': script, 'plot_div':div}
+        temp_vars = {'inst': self.instrument, 'plot_script': script, 'plot_div': div}
         self._html = file_html(tabs, CDN, f'{self.instrument} bad pix monitor', template_file, temp_vars)
 
         # Modify the html such that our Django-related lines are kept in place,
@@ -264,7 +261,6 @@ def __init__(self, pixel_table, instrument, detector):
         for badtype in self.badtypes:
             self.get_trending_data(badtype)
 
-
     def get_most_recent_entry(self):
         """Get all nedded data from the database tables.
         """
@@ -312,8 +308,8 @@ def get_trending_data(self, badpix_type):
         # Query database for all data in the table with a matching detector and bad pixel type
         all_entries_by_type = session.query(self.pixel_table.type, self.pixel_table.detector, func.array_length(self.pixel_table.x_coord, 1),
                                             self.pixel_table.obs_mid_time) \
-                              .filter(and_(self.pixel_table.detector == self.detector,  self.pixel_table.type == badpix_type)) \
-                              .all()
+            .filter(and_(self.pixel_table.detector == self.detector, self.pixel_table.type == badpix_type)) \
+            .all()
 
         # Organize the results
         num_pix = []
@@ -463,7 +459,7 @@ def create_plot(self):
             self.plot = figure(title=title_text, tools=tools,
                                x_axis_label="Pixel Number", y_axis_label="Pixel Number")
         else:
-            self.plot = figure(tools='') #, x_axis_label="Pixel Number", y_axis_label="Pixel Number")
+            self.plot = figure(tools='')
             self.plot.toolbar.logo = None
             self.plot.toolbar_location = None
             self.plot.min_border = 0
@@ -475,18 +471,9 @@ def create_plot(self):
         # Plot image
         if image is not None:
             ny, nx = image.shape
-            #imgplot = self.plot.image(image=[image], x=0, y=0, dw=nx, dh=ny, color_mapper=mapper, level="image")
-
-
-
             # Shift the figure title slightly right in this case to get it
             # to align with the axes
-            #self.plot = figure(title=title, x_range=(0, self._detlen), y_range=(0, self._detlen), width=xdim, height=ydim*,
-            #                   tools='pan,box_zoom,reset,wheel_zoom,save', x_axis_label="Pixel Number", y_axis_label="Pixel Number")
             self.plot.image_rgba(image=[image], x=0, y=0, dw=self._detlen, dh=self._detlen, alpha=0.5)
-
-
-
         else:
             # If the background image is not present, manually set the x and y range
             self.plot.x_range.start = 0
@@ -511,11 +498,10 @@ def create_plot(self):
         legend = Legend(items=[plot_legend],
                         location="center",
                         orientation='vertical',
-                        title = legend_title)
+                        title=legend_title)
 
         self.plot.add_layout(legend, 'below')
 
-
     def overplot_bad_pix(self):
         """Add a scatter plot of potential new bad pixels to the plot
 
@@ -537,7 +523,7 @@ def overplot_bad_pix(self):
             # If there are no new bad pixels, write text within the figure mentioning that
             txt_source = ColumnDataSource(data=dict(x=[self._detlen / 10], y=[self._detlen / 2],
                                           text=[f'No new {self.badpix_type} pixels found']))
-            glyph = Text(x="x", y="y", text="text", angle=0., text_color="navy", text_font_size={'value':'20px'})
+            glyph = Text(x="x", y="y", text="text", angle=0., text_color="navy", text_font_size={'value': '20px'})
             self.plot.add_glyph(txt_source, glyph)
 
             # Insert a fake one, in order to get the plot to be made
@@ -593,7 +579,7 @@ def switch_to_png(self, filename, title):
             Title to add to the Figure
         """
         # Save the figure as a png
-        export_png(self.plot, filename=filename)
+        save_png(self.plot, filename=filename)
         set_permissions(filename)
 
         # Read in the png and insert into a replacement figure
@@ -668,7 +654,7 @@ def create_plot(self):
                                             string_time=string_times,
                                             value=[self.badpix_type] * len(self.num_pix)
                                             )
-                                    )
+                                  )
 
         self.plot = figure(title=f'{self.detector}: New {self.badpix_type} Pixels', tools='pan,box_zoom,reset,wheel_zoom,save',
                            background_fill_color="#fafafa")
@@ -696,7 +682,7 @@ def create_plot(self):
             time_pad = datetime.timedelta(days=1)
         self.plot.x_range.start = min(self.time) - time_pad
         self.plot.x_range.end = max(self.time) + time_pad
-        self.plot.grid.grid_line_color="white"
+        self.plot.grid.grid_line_color = "white"
         self.plot.xaxis.axis_label = 'Date'
         self.plot.yaxis.axis_label = f'Number of {self.badpix_type} pixels'
 


Test patch:
diff --git a/jwql/tests/test_utils.py b/jwql/tests/test_utils.py
--- a/jwql/tests/test_utils.py
+++ b/jwql/tests/test_utils.py
@@ -22,7 +22,11 @@
 from pathlib import Path
 import pytest
 
-from jwql.utils.utils import copy_files, get_config, filename_parser, filesystem_path, _validate_config
+from bokeh.models import LinearColorMapper
+from bokeh.plotting import figure
+import numpy as np
+
+from jwql.utils.utils import copy_files, get_config, filename_parser, filesystem_path, save_png, _validate_config
 
 
 # Determine if tests are being run on Github Actions
@@ -479,6 +483,17 @@ def test_filesystem_path():
     assert check == location
 
 
+def test_save_png():
+    """Test that we can create a png file"""
+    plot = figure(title='test', tools='')
+    image = np.zeros((200, 200))
+    image[100:105, 100:105] = 1
+    ny, nx = image.shape
+    mapper = LinearColorMapper(palette='Viridis256', low=0, high=1.1)
+    imgplot = plot.image(image=[image], x=0, y=0, dw=nx, dh=ny, color_mapper=mapper, level="image")
+    save_png(plot, filename='test.png')
+
+
 @pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to central storage.')
 def test_validate_config():
     """Test that the config validator works."""