Repository: spacetelescope/jwql

Issue:
Jenkins builds still passing despite numerous failed unit tests
It seems that Jenkins builds are passing even though there may be several failed unit tests, for example:

https://ssbjenkins.stsci.edu/blue/organizations/jenkins/STScI%2Fjwql/detail/PR-322/11/pipeline/21/#step-39-log-21

#338 attempted to fix this, but it doesn't seem to be having an effect afterall.  


Discussion:
Is it possible that the Jenkinsfile needs to explicitly state
```
bc0.failedUnstableThresh = 0
bc0.failedFailureThresh = 0
```
rather than removing the lines and not defining them at all?
@laurenmarietta Hmm, good idea, that could be it 
@bourque Just taking a quick look.
`https://ssbjenkins.stsci.edu/blue/organizations/jenkins/STScI%2Fjwql/detail/PR-322/11/pipeline/21/#step-39-log-21`

Seems that build log is no longer available. Looking at one of the latest builds, perhaps:
```
No .xml files found in workspace. Test report ingestion skipped.
```
is causing this? Seems like Jenkins isn't ingesting the test report and therefore doesn't know about the failures?

When the jenkins shared util library is looking for xml files it gets an error since it matches multiple .xml files which test -e doesn't support:
```
+ test -e coverage.xml results.xml

/home/jenkins/workspace/STScI_jwql_PR-365@tmp/durable-4dfab6f1/script.sh: line 1: test: coverage.xml: binary operator expected
```
which is then skipping test report ingestion.

Recommend changing the pytest command in the JenkinsFile to move one of the generated rest reports into a folder. Not sure if the coverage.xml file is in junit format, if not it may cause error being read by jenkins.
@obviousrebel is correct. The multiple .xml files in the repository root after testing and coverage measurement are performed are causing the problem. This is a known limitation. The jenkins_shared_ci_utils docs state:

"After the last test_cmds command is executed, Jenkins examines the build environment for a filename with an .xml extension. If one is found, it is assumed to be a JUnit-compliant test report and is read. "

If the coverage reporter can be told to expect the report in a location other than the repository root, that would be the quickest temporary fix. The CI utils library will ultimately have to receive some logic to be more discriminating of which files represent test reports and are thus ingestible by Jenkins. https://github.com/spacetelescope/jenkins_shared_ci_utils/issues/45

Resolving patch:
diff --git a/jwql/instrument_monitors/common_monitors/dark_monitor.py b/jwql/instrument_monitors/common_monitors/dark_monitor.py
--- a/jwql/instrument_monitors/common_monitors/dark_monitor.py
+++ b/jwql/instrument_monitors/common_monitors/dark_monitor.py
@@ -145,8 +145,6 @@ def mast_query_darks(instrument, aperture, start_date, end_date):
     return query_results
 
 
-@log_fail
-@log_info
 class Dark():
     """Class for executing the dark current monitor.
 
@@ -209,95 +207,8 @@ class Dark():
         If the most recent query search returns more than one entry
     """
 
-    def __init__(self, testing=False):
-
-        logging.info('Begin logging for dark_monitor')
-
-        apertures_to_skip = ['NRCALL_FULL', 'NRCAS_FULL', 'NRCBS_FULL']
-
-        if not testing:
-
-            # Get the output directory
-            self.output_dir = os.path.join(get_config()['outputs'], 'dark_monitor')
-
-            # Read in config file that defines the thresholds for the number
-            # of dark files that must be present in order for the monitor to run
-            limits = ascii.read(THRESHOLDS_FILE)
-
-            # Use the current time as the end time for MAST query
-            self.query_end = Time.now().mjd
-
-            # Loop over all instruments
-            for instrument in ['nircam']:
-                self.instrument = instrument
-
-                # Identify which database tables to use
-                self.identify_tables()
-
-                # Get a list of all possible apertures from pysiaf
-                possible_apertures = list(Siaf(instrument).apernames)
-                possible_apertures = [ap for ap in possible_apertures if ap not in apertures_to_skip]
-
-                for aperture in possible_apertures:
-
-                    logging.info('')
-                    logging.info('Working on aperture {} in {}'.format(aperture, instrument))
-
-                    # Find the appropriate threshold for the number of new files needed
-                    match = aperture == limits['Aperture']
-                    file_count_threshold = limits['Threshold'][match]
-
-                    # Locate the record of the most recent MAST search
-                    self.aperture = aperture
-                    self.query_start = self.most_recent_search()
-                    logging.info('\tQuery times: {} {}'.format(self.query_start, self.query_end))
-
-                    # Query MAST using the aperture and the time of the
-                    # most recent previous search as the starting time
-                    new_entries = mast_query_darks(instrument, aperture, self.query_start, self.query_end)
-                    logging.info('\tAperture: {}, new entries: {}'.format(self.aperture, len(new_entries)))
-
-                    # Check to see if there are enough new files to meet the monitor's signal-to-noise requirements
-                    if len(new_entries) >= file_count_threshold:
-
-                        logging.info('\tSufficient new dark files found for {}, {} to run the dark monitor.'
-                                     .format(self.instrument, self.aperture))
-
-                        # Get full paths to the files
-                        new_filenames = [filesystem_path(file_entry['filename']) for file_entry in new_entries]
-
-                        # Set up directories for the copied data
-                        ensure_dir_exists(os.path.join(self.output_dir, 'data'))
-                        self.data_dir = os.path.join(self.output_dir,
-                                                     'data/{}_{}'.format(self.instrument.lower(),
-                                                                         self.aperture.lower()))
-                        ensure_dir_exists(self.data_dir)
-
-                        # Copy files from filesystem
-                        dark_files, not_copied = copy_files(new_filenames, self.data_dir)
-
-                        # Run the dark monitor
-                        self.run(dark_files)
-                        monitor_run = True
-
-                    else:
-                        logging.info(('\tDark monitor skipped. {} new dark files for {}, {}. {} new files are '
-                                      'required to run dark current monitor.').format(
-                            len(new_entries), instrument, aperture, file_count_threshold[0]))
-                        monitor_run = False
-
-                    # Update the query history
-                    new_entry = {'instrument': instrument,
-                                 'aperture': aperture,
-                                 'start_time_mjd': self.query_start,
-                                 'end_time_mjd': self.query_end,
-                                 'files_found': len(new_entries),
-                                 'run_monitor': monitor_run,
-                                 'entry_date': datetime.datetime.now()}
-                    self.query_table.__table__.insert().execute(new_entry)
-                    logging.info('\tUpdated the query history table')
-
-            logging.info('Dark Monitor completed successfully.')
+    def __init__(self):
+        """Initialize an instance of the ``Dark`` class."""
 
     def add_bad_pix(self, coordinates, pixel_type, files, mean_filename, baseline_filename):
         """Add a set of bad pixels to the bad pixel database table
@@ -575,34 +486,9 @@ def noise_check(self, new_noise_image, baseline_noise_image, threshold=1.5):
 
         return noisy
 
-    def read_baseline_slope_image(self, filename):
-        """Read in a baseline mean slope image and associated standard
-        deviation image from the given fits file
-
-        Parameters
-        ----------
-        filename : str
-            Name of fits file to be read in
-
-        Returns
-        -------
-        mean_image : numpy.ndarray
-            2D mean slope image
-
-        stdev_image : numpy.ndarray
-            2D stdev image
-        """
-
-        try:
-            with fits.open(filename) as hdu:
-                mean_image = hdu['MEAN'].data
-                stdev_image = hdu['STDEV'].data
-            return mean_image, stdev_image
-        except (FileNotFoundError, KeyError) as e:
-            logging.warning('Trying to read {}: {}'.format(filename, e))
-
-    def run(self, file_list):
-        """The main method.  See module docstrings for further details
+    def process(self, file_list):
+        """The main method for processing darks.  See module docstrings
+        for further details.
 
         Parameters
         ----------
@@ -765,6 +651,125 @@ def run(self, file_list):
                              }
             self.stats_table.__table__.insert().execute(dark_db_entry)
 
+    def read_baseline_slope_image(self, filename):
+        """Read in a baseline mean slope image and associated standard
+        deviation image from the given fits file
+
+        Parameters
+        ----------
+        filename : str
+            Name of fits file to be read in
+
+        Returns
+        -------
+        mean_image : numpy.ndarray
+            2D mean slope image
+
+        stdev_image : numpy.ndarray
+            2D stdev image
+        """
+
+        try:
+            with fits.open(filename) as hdu:
+                mean_image = hdu['MEAN'].data
+                stdev_image = hdu['STDEV'].data
+            return mean_image, stdev_image
+        except (FileNotFoundError, KeyError) as e:
+            logging.warning('Trying to read {}: {}'.format(filename, e))
+
+    @log_fail
+    @log_info
+    def run(self):
+        """The main method.  See module docstrings for further
+        details.
+        """
+
+        logging.info('Begin logging for dark_monitor')
+
+        apertures_to_skip = ['NRCALL_FULL', 'NRCAS_FULL', 'NRCBS_FULL']
+
+        # Get the output directory
+        self.output_dir = os.path.join(get_config()['outputs'], 'dark_monitor')
+
+        # Read in config file that defines the thresholds for the number
+        # of dark files that must be present in order for the monitor to run
+        limits = ascii.read(THRESHOLDS_FILE)
+
+        # Use the current time as the end time for MAST query
+        self.query_end = Time.now().mjd
+
+        # Loop over all instruments
+        for instrument in ['nircam']:
+            self.instrument = instrument
+
+            # Identify which database tables to use
+            self.identify_tables()
+
+            # Get a list of all possible apertures from pysiaf
+            possible_apertures = list(Siaf(instrument).apernames)
+            possible_apertures = [ap for ap in possible_apertures if ap not in apertures_to_skip]
+
+            for aperture in possible_apertures:
+
+                logging.info('')
+                logging.info('Working on aperture {} in {}'.format(aperture, instrument))
+
+                # Find the appropriate threshold for the number of new files needed
+                match = aperture == limits['Aperture']
+                file_count_threshold = limits['Threshold'][match]
+
+                # Locate the record of the most recent MAST search
+                self.aperture = aperture
+                self.query_start = self.most_recent_search()
+                logging.info('\tQuery times: {} {}'.format(self.query_start, self.query_end))
+
+                # Query MAST using the aperture and the time of the
+                # most recent previous search as the starting time
+                new_entries = mast_query_darks(instrument, aperture, self.query_start, self.query_end)
+                logging.info('\tAperture: {}, new entries: {}'.format(self.aperture, len(new_entries)))
+
+                # Check to see if there are enough new files to meet the monitor's signal-to-noise requirements
+                if len(new_entries) >= file_count_threshold:
+
+                    logging.info('\tSufficient new dark files found for {}, {} to run the dark monitor.'
+                                 .format(self.instrument, self.aperture))
+
+                    # Get full paths to the files
+                    new_filenames = [filesystem_path(file_entry['filename']) for file_entry in new_entries]
+
+                    # Set up directories for the copied data
+                    ensure_dir_exists(os.path.join(self.output_dir, 'data'))
+                    self.data_dir = os.path.join(self.output_dir,
+                                                 'data/{}_{}'.format(self.instrument.lower(),
+                                                                     self.aperture.lower()))
+                    ensure_dir_exists(self.data_dir)
+
+                    # Copy files from filesystem
+                    dark_files, not_copied = copy_files(new_filenames, self.data_dir)
+
+                    # Run the dark monitor
+                    self.process(dark_files)
+                    monitor_run = True
+
+                else:
+                    logging.info(('\tDark monitor skipped. {} new dark files for {}, {}. {} new files are '
+                                  'required to run dark current monitor.').format(
+                        len(new_entries), instrument, aperture, file_count_threshold[0]))
+                    monitor_run = False
+
+                # Update the query history
+                new_entry = {'instrument': instrument,
+                             'aperture': aperture,
+                             'start_time_mjd': self.query_start,
+                             'end_time_mjd': self.query_end,
+                             'files_found': len(new_entries),
+                             'run_monitor': monitor_run,
+                             'entry_date': datetime.datetime.now()}
+                self.query_table.__table__.insert().execute(new_entry)
+                logging.info('\tUpdated the query history table')
+
+        logging.info('Dark Monitor completed successfully.')
+
     def save_mean_slope_image(self, slope_img, stdev_img, files):
         """Save the mean slope image and associated stdev image to a
         file
@@ -984,5 +989,6 @@ def stats_by_amp(self, image, amps):
     start_time, log_file = initialize_instrument_monitor(module)
 
     monitor = Dark()
+    monitor.run()
 
     update_monitor_table(module, start_time, log_file)


Test patch:
diff --git a/jwql/tests/test_dark_monitor.py b/jwql/tests/test_dark_monitor.py
--- a/jwql/tests/test_dark_monitor.py
+++ b/jwql/tests/test_dark_monitor.py
@@ -29,7 +29,7 @@
 
 def test_find_hot_dead_pixels():
     """Test hot and dead pixel searches"""
-    monitor = dark_monitor.Dark(testing=True)
+    monitor = dark_monitor.Dark()
 
     # Create "baseline" image
     comparison_image = np.zeros((10, 10)) + 1.
@@ -57,7 +57,7 @@ def test_find_hot_dead_pixels():
 def test_get_metadata():
     """Test retrieval of metadata from input file"""
 
-    monitor = dark_monitor.Dark(testing=True)
+    monitor = dark_monitor.Dark()
     filename = os.path.join(get_config()['test_dir'], 'dark_monitor', 'test_image_1.fits')
     monitor.get_metadata(filename)
 
@@ -114,7 +114,7 @@ def test_noise_check():
     baseline[5, 5] = 1.0
     noise_image[5, 5] = 1.25
 
-    monitor = dark_monitor.Dark(testing=True)
+    monitor = dark_monitor.Dark()
     noisy = monitor.noise_check(noise_image, baseline, threshold=1.5)
 
     assert len(noisy[0]) == 2
@@ -125,7 +125,7 @@ def test_noise_check():
 def test_shift_to_full_frame():
     """Test pixel coordinate shifting to be in full frame coords"""
 
-    monitor = dark_monitor.Dark(testing=True)
+    monitor = dark_monitor.Dark()
     monitor.x0 = 512
     monitor.y0 = 512
 
diff --git a/jwql/tests/test_loading_times.py b/jwql/tests/test_loading_times.py
--- a/jwql/tests/test_loading_times.py
+++ b/jwql/tests/test_loading_times.py
@@ -19,6 +19,7 @@
         pytest -s test_loading_times.py
 """
 
+import os
 import pytest
 import time
 import urllib.request
@@ -48,6 +49,8 @@
     urls.append('{}/{}/'.format(instrument, rootname))
 
 
+@pytest.mark.skipif(os.path.expanduser('~') == '/home/jenkins',
+                    reason='Requires access to central storage.')
 @pytest.mark.parametrize('url', urls)
 def test_loading_times(url):
     """Test to see if the given ``url`` returns a webpage sucessfully
diff --git a/jwql/tests/test_permissions.py b/jwql/tests/test_permissions.py
--- a/jwql/tests/test_permissions.py
+++ b/jwql/tests/test_permissions.py
@@ -99,6 +99,8 @@ def test_file(test_dir=TEST_DIRECTORY):
         os.rmdir(test_dir)
 
 
+@pytest.mark.skipif(os.path.expanduser('~') == '/home/jenkins',
+                    reason='Requires access to central storage.')
 def test_file_group(test_file):
     """Create a file with the standard permissions ``('-rw-r--r--')``
     and default group.