Repository: spacetelescope/jwst

Issue:
tso_photometry bug in computation of time stamps
_Issue [JP-1680](https://jira.stsci.edu/browse/JP-1680) was created on JIRA by [Howard Bushouse](https://jira.stsci.edu/secure/ViewProfile.jspa?name=bushouse):_

The tso_photometry step is crashing when processing a set of segmented MIRI imaging TSO data. The input calints products have 0 rows populated in their INT_TIMES table extensions, so the tso_photometry step defaults to computing its own time stamps. But the time stamps are computed based on the total number of integrations in the whole exposure, as opposed to the number of integrations contained in the segment being processed. This leads to a mismatch in the number of time stamps vs. number of photometry results, causing a crash in the astropy table functions.
```java
ERROR RUNNING STEP 'TSOPhotometryStep':
    Inconsistent data column lengths
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/Users/bushouse/JWST/softw/jwst/scripts/strun", line 26, in <module>
    step = Step.from_cmdline(sys.argv[1:])
  File "/Users/bushouse/JWST/softw/jwst/jwst/stpipe/step.py", line 162, in from_cmdline
    return cmdline.step_from_cmdline(args)
  File "/Users/bushouse/JWST/softw/jwst/jwst/stpipe/cmdline.py", line 346, in step_from_cmdline
    step.run(*positional)
  File "/Users/bushouse/JWST/softw/jwst/jwst/stpipe/step.py", line 398, in run
    step_result = self.process(*args)
  File "/Users/bushouse/JWST/softw/jwst/jwst/tso_photometry/tso_photometry_step.py", line 67, in process
    catalog = tso_aperture_photometry(model, xcenter, ycenter,
  File "/Users/bushouse/JWST/softw/jwst/jwst/tso_photometry/tso_photometry.py", line 168, in tso_aperture_photometry
    tbl['aperture_sum'] = aperture_sum
  File "/Users/bushouse/anaconda3/envs/jwstdev/lib/python3.8/site-packages/astropy/table/table.py", line 1635, in __setitem__
    self.add_column(value, name=item, copy=True)
  File "/Users/bushouse/anaconda3/envs/jwstdev/lib/python3.8/site-packages/astropy/table/table.py", line 1916, in add_column
    raise ValueError('Inconsistent data column lengths')
ValueError: Inconsistent data column lengths
```

Discussion:

Patch:
diff --git a/CHANGES.rst b/CHANGES.rst
index 8bb5ab16c..c55d24748 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -13,6 +13,11 @@ blendmeta
 
 - Do not close files that were not opened by blendmodels [#5299]
 
+cube_build
+----------
+
+- If every wavelength plane of the IFU cube contains 0 data, cube_build is skipped [#5294]
+
 flat_field
 ----------
 
@@ -49,6 +54,12 @@ pipeline
 - Include the per-slit failure traceback in any RuntimeError raised in Spec2Pipeline. [#5315]
 
 
+tso_photometry
+--------------
+
+- Fix a bug in the computation of integration time stamps when the INT_TIMES
+  table is not available. [#5318]
+
 0.17.0 (2020-08-28)
 ===================
 
diff --git a/jwst/cube_build/cube_build_step.py b/jwst/cube_build/cube_build_step.py
index 83fb161a8..145bc11f9 100755
--- a/jwst/cube_build/cube_build_step.py
+++ b/jwst/cube_build/cube_build_step.py
@@ -294,6 +294,7 @@ class CubeBuildStep (Step):
         # ModelContainer of ifucubes
         cube_container = datamodels.ModelContainer()
 
+        status_cube = 0
         for i in range(num_cubes):
             icube = str(i + 1)
             list_par1 = cube_pars[icube]['par1']
@@ -332,6 +333,7 @@ class CubeBuildStep (Step):
 # If single = True: map each file to output grid and return single mapped file
 # to output grid
 # This option is used for background matching and outlier rejection
+            status = 0
             if self.single:
                 self.output_file = None
                 cube_container = thiscube.build_ifucube_single()
@@ -340,14 +342,25 @@ class CubeBuildStep (Step):
 
 # Else standard IFU cube building
             else:
-                result = thiscube.build_ifucube()
+                cube_result  = thiscube.build_ifucube()
+                result, status = cube_result
                 cube_container.append(result)
+
             if self.debug_file is not None:
                 self.debug_file.close()
+
+            # check if cube_build failed
+            # **************************
+            if status == 1:
+                status_cube = 1
+
         for cube in cube_container:
             footprint = cube.meta.wcs.footprint(axis_type="spatial")
             update_s_region_keyword(cube, footprint)
 
+        if status_cube ==1:
+            self.skip = True
+
         return cube_container
 # ******************************************************************************
 
diff --git a/jwst/cube_build/ifu_cube.py b/jwst/cube_build/ifu_cube.py
index 11cc22ca7..59a75f46e 100644
--- a/jwst/cube_build/ifu_cube.py
+++ b/jwst/cube_build/ifu_cube.py
@@ -779,12 +779,12 @@ class IFUCubeData():
         self.set_final_dq_flags()
         t1 = time.time()
         log.info("Time to find Cube Flux = %.1f s" % (t1 - t0,))
-
-        ifucube_model = self.setup_final_ifucube_model(0)
+        # result consist of ifu_cube and status
+        result  = self.setup_final_ifucube_model(0)
 # _______________________________________________________________________
 # shove Flux and iflux in the  final IFU cube
 
-        return ifucube_model
+        return result
 # ********************************************************************************
 
     def build_ifucube_single(self):
@@ -797,7 +797,6 @@ class IFUCubeData():
         system
 
         """
-
         # loop over input models
         single_ifucube_container = datamodels.ModelContainer()
         n = len(self.input_models)
@@ -853,14 +852,15 @@ class IFUCubeData():
 # shove Flux and iflux in the  final ifucube
             self.find_spaxel_flux()
 
-            #now determine Cube Spaxel flux
-
-            ifucube_model = self.setup_final_ifucube_model(j)
-
+            # determine Cube Spaxel flux
+            status = 0
+            result = self.setup_final_ifucube_model(j)
+            ifucube_model, status = result
             t1 = time.time()
             log.debug("Time to Create Single ifucube = %.1f s" % (t1 - t0,))
-# _______________________________________________________________________
             single_ifucube_container.append(ifucube_model)
+            if status !=0:
+                log.debug("Possible problem with single ifu cube, no valid data in cube" )
 
         return single_ifucube_container
 # **************************************************************************
@@ -2048,6 +2048,7 @@ class IFUCubeData():
 
         """
 
+        status = 0
         # loop over the wavelength planes to confirm each plane has some data
         # for initial or final planes that do not have any data - eliminated them
         # from the IFUcube
@@ -2088,9 +2089,16 @@ class IFUCubeData():
                     break;
                 k = k -1
 
-            if (remove_start+remove_final) > 0:
+            remove_total = remove_start + remove_final
+            if remove_total >= self.naxis3:
+                log.error('All the wavelength planes have zero data, check input data')
+                # status of 1 sets up failure of IFU cube building
+                # the input to cube_build  is returned instead of an zero filled ifucube
+                status = 1
+
+            if (remove_total > 0 and remove_total < self.naxis3) :
                 log.info('Number of wavelength planes removed with no data: %i',
-                         remove_start+remove_final)
+                         remove_total)
 
                 temp_flux = temp_flux[remove_start:self.naxis3-remove_final,:,:]
                 temp_wmap = temp_wmap[remove_start:self.naxis3-remove_final,:,:]
@@ -2304,8 +2312,12 @@ class IFUCubeData():
                                                (0, naxis3 - 1))
 
         ifucube_model.meta.cal_step.cube_build = 'COMPLETE'
+        # problem with cube_build - contains only 0 data
+        if status == 1:
+            ifucube_model.meta.cal_step.cube_build = 'SKIPPED'
 
-        return ifucube_model
+        result = (ifucube_model,status)
+        return result
 # ********************************************************************************
 
     def blend_output_metadata(self, IFUCube):
diff --git a/jwst/tso_photometry/tso_photometry.py b/jwst/tso_photometry/tso_photometry.py
index b5090d6c5..45977c812 100644
--- a/jwst/tso_photometry/tso_photometry.py
+++ b/jwst/tso_photometry/tso_photometry.py
@@ -115,13 +115,14 @@ def tso_aperture_photometry(datamodel, xcenter, ycenter, radius, radius_inner,
     # initialize the output table
     tbl = QTable(meta=meta)
 
+    # check for the INT_TIMES table extension
     if hasattr(datamodel, 'int_times') and datamodel.int_times is not None:
         nrows = len(datamodel.int_times)
     else:
         nrows = 0
-    if nrows == 0:
-        log.warning("There is no INT_TIMES table in the input file.")
+        log.warning("The INT_TIMES table in the input file is missing or empty.")
 
+    # load the INT_TIMES table data
     if nrows > 0:
         shape = datamodel.data.shape
         if len(shape) == 2:
@@ -131,8 +132,7 @@ def tso_aperture_photometry(datamodel, xcenter, ycenter, radius, radius_inner,
         int_start = datamodel.meta.exposure.integration_start
         if int_start is None:
             int_start = 1
-            log.warning("INTSTART not found; assuming a value of %d",
-                        int_start)
+            log.warning(f"INTSTART not found; assuming a value of {int_start}")
 
         # Columns of integration numbers & times of integration from the
         # INT_TIMES table.
@@ -146,23 +146,23 @@ def tso_aperture_photometry(datamodel, xcenter, ycenter, radius, radius_inner,
             del int_num, mid_utc
             nrows = 0                   # flag as bad
         else:
-            log.debug("Times are from the INT_TIMES table.")
+            log.debug("Times are from the INT_TIMES table")
             time_arr = mid_utc[offset: offset + num_integ]
             int_times = Time(time_arr, format='mjd', scale='utc')
 
+    # compute integration time stamps on the fly
     if nrows == 0:
-        log.debug("Times were computed from EXPSTART and TGROUP.")
-
-        dt = (datamodel.meta.exposure.group_time *
-              (datamodel.meta.exposure.ngroups + 1))
-        dt_arr = (np.arange(1, 1 + datamodel.meta.exposure.nints) *
-                  dt - (dt / 2.))
+        log.debug("Times computed from EXPSTART and EFFINTTM")
+        dt = datamodel.meta.exposure.integration_time
+        n_dt = (datamodel.meta.exposure.integration_end -
+                datamodel.meta.exposure.integration_start + 1)
+        dt_arr = (np.arange(1, 1 + n_dt) * dt - (dt / 2.))
         int_dt = TimeDelta(dt_arr, format='sec')
         int_times = (Time(datamodel.meta.exposure.start_time, format='mjd') +
                      int_dt)
 
+    # populate table columns
     tbl['MJD'] = int_times.mjd
-
     tbl['aperture_sum'] = aperture_sum
     tbl['aperture_sum_err'] = aperture_sum_err
 
diff --git a/jwst/tso_photometry/tso_photometry_step.py b/jwst/tso_photometry/tso_photometry_step.py
index 7c77010b5..2f701cef4 100755
--- a/jwst/tso_photometry/tso_photometry_step.py
+++ b/jwst/tso_photometry/tso_photometry_step.py
@@ -60,6 +60,8 @@ class TSOPhotometryStep(Step):
             self.log.debug(f'radius = {radius}')
             self.log.debug(f'radius_inner = {radius_inner}')
             self.log.debug(f'radius_outer = {radius_outer}')
+            self.log.debug(f'xcenter = {xcenter}')
+            self.log.debug(f'ycenter = {ycenter}')
 
             # Compute the aperture photometry
             catalog = tso_aperture_photometry(model, xcenter, ycenter,


Test patch:
diff --git a/jwst/tso_photometry/tests/test_tso_photometry.py b/jwst/tso_photometry/tests/test_tso_photometry.py
index e3e0be318..d894abd61 100644
--- a/jwst/tso_photometry/tests/test_tso_photometry.py
+++ b/jwst/tso_photometry/tests/test_tso_photometry.py
@@ -56,11 +56,12 @@ def set_meta(datamodel, sub64p=False):
     """Assign some metadata"""
 
     datamodel.meta.exposure.nints = datamodel.data.shape[0]
-    datamodel.meta.exposure.ngroups = 8
-    datamodel.meta.exposure.group_time = 10.7 * datamodel.meta.exposure.ngroups
+    datamodel.meta.exposure.integration_time = 10.7
     datamodel.meta.exposure.start_time = 58704.62847
 
     datamodel.meta.exposure.integration_start = 5
+    datamodel.meta.exposure.integration_end = (
+        datamodel.meta.exposure.integration_start + datamodel.meta.exposure.nints - 1)
 
     datamodel.meta.instrument.name = 'NIRCAM'
     datamodel.meta.instrument.detector = 'NRCA2'
@@ -229,7 +230,7 @@ def test_tso_phot_4():
                                       radius + 1., radius_inner,
                                       radius_outer)
 
-    int_times = np.array([58704.63293, 58704.641845, 58704.65076,
-                          58704.65968, 58704.6686, 58704.677512,
-                          58704.686428])
+    int_times = np.array([58704.62853, 58704.628655, 58704.62878,
+                          58704.62890, 58704.6290, 58704.6291511,
+                          58704.629275])
     assert np.allclose(catalog['MJD'], int_times, rtol=1.e-8)