Repository: spacetelescope/jwst

Issue:
Invert the default state of association merging
# Issue
The generator, by default, merges level2 associations into a single assocation file with multiple products. The intent was to reduce file count/clutter since there would otherwise be one association-per-exposure. However, SDP has opted not to go this route. To make usage more consistent between what will be found in the archive to what users would get, make the default to not merge associations.

Discussion:

Patch:
diff --git a/CHANGES.rst b/CHANGES.rst
index 5b3130372..de4c64837 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -12,6 +12,8 @@ associations
 
 - MIRI MRS dedicated background exposures are now listed as science observations in
   a new association. [#3542]
+  
+- Generate will no longer merge Level2 associations by default [#3631]
 
 datamodels
 ----------
@@ -33,6 +35,9 @@ pipeline
 - ``calwebb_spec2`` was changed to allow processing of exposures
   with ``EXP_TYPE=NRS_LAMP.`` [#3603]
 
+- ``calwebb_tso3`` was changed to allow processing of exposures
+  with ``EXP_TYPE=MIR_IMAGE.`` [#3633]
+
 
 0.13.3 (2019-06-04)
 ===================
diff --git a/jwst/associations/association_io.py b/jwst/associations/association_io.py
index 8c0f259a0..43a739ea8 100644
--- a/jwst/associations/association_io.py
+++ b/jwst/associations/association_io.py
@@ -124,7 +124,7 @@ class yaml():
         except AttributeError:
             pass
         try:
-            asn = yaml_lib.load(serialized)
+            asn = yaml_lib.safe_load(serialized)
         except Exception as err:
             logger.debug('Error unserializing: "{}"'.format(err))
             raise AssociationNotValidError(
diff --git a/jwst/associations/main.py b/jwst/associations/main.py
index 1310cdab3..1b37febf1 100644
--- a/jwst/associations/main.py
+++ b/jwst/associations/main.py
@@ -176,8 +176,12 @@ class Main():
             help='Version of the generator.'
         )
         parser.add_argument(
-            '--no-merge', action='store_true',
-            help='Do not merge Level2 associations into one'
+            '--merge', action='store_true',
+            help='Merge associations into single associations with multiple products'
+        )
+        parser.add_argument(
+            '--no-merge', action=DeprecateNoMerge,
+            help='Deprecated: Default is to not merge. See "--merge".'
         )
 
         parsed = parser.parse_args(args=args)
@@ -265,7 +269,7 @@ class Main():
 
         # Do a grand merging. This is done particularly for
         # Level2 associations.
-        if not parsed.no_merge:
+        if parsed.merge:
             try:
                 self.associations = self.rules.Utility.merge_asns(self.associations)
             except AttributeError:
@@ -335,6 +339,16 @@ class Main():
 # #########
 # Utilities
 # #########
+class DeprecateNoMerge(argparse.Action):
+    """Deprecate the `--no-merge` option"""
+    def __init__(self, option_strings, dest, nargs=None, **kwargs):
+        logger.warning('The "--no-merge" option is deprecated in favor of the "--merge" option.')
+        super(DeprecateNoMerge, self).__init__(option_strings, dest, const=True, nargs=0, **kwargs)
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        setattr(namespace, self.dest, values)
+
+
 def constrain_on_candidates(candidates):
     """Create a constraint based on a list of candidates
 
diff --git a/jwst/pipeline/calwebb_tso3.py b/jwst/pipeline/calwebb_tso3.py
index 8d60d133d..9b38f15c1 100644
--- a/jwst/pipeline/calwebb_tso3.py
+++ b/jwst/pipeline/calwebb_tso3.py
@@ -37,7 +37,7 @@ class Tso3Pipeline(Pipeline):
                  'extract_1d': extract_1d_step.Extract1dStep,
                  'white_light': white_light_step.WhiteLightStep
                  }
-    image_exptypes = ['NRC_TSIMAGE']
+    image_exptypes = ['NRC_TSIMAGE', 'MIR_IMAGE']
     reference_file_types = ['gain', 'readnoise']
 
     def process(self, input):
@@ -58,10 +58,15 @@ class Tso3Pipeline(Pipeline):
         self.asn_id = input_models.meta.asn_table.asn_id
 
         input_exptype = None
+        input_tsovisit = None
         # Input may consist of multiple exposures, so loop over each of them
         for cube in input_models:
             if input_exptype is None:
                 input_exptype = cube.meta.exposure.type
+
+            if input_tsovisit is None:
+                input_tsovisit = cube.meta.visit.tsovisit
+
             # Convert CubeModel into ModelContainer of 2-D DataModels
             input_2dmodels = datamodels.ModelContainer()
             for i in range(cube.data.shape[0]):
@@ -105,7 +110,8 @@ class Tso3Pipeline(Pipeline):
         # Create final photometry results as a single output
         # regardless of how many members there may be...
         phot_result_list = []
-        if input_exptype in self.image_exptypes:
+
+        if input_exptype in self.image_exptypes and input_tsovisit:
             # Create name for extracted photometry (Level 3) product
             phot_tab_suffix = 'phot'
 
diff --git a/jwst/tso_photometry/tso_photometry_step.py b/jwst/tso_photometry/tso_photometry_step.py
index 1d993d0cd..8076ee183 100755
--- a/jwst/tso_photometry/tso_photometry_step.py
+++ b/jwst/tso_photometry/tso_photometry_step.py
@@ -40,8 +40,12 @@ class TSOPhotometryStep(Step):
                 self.log.warning('the tso_photometry step will be skipped.')
                 return None
 
+            pupil_name = 'ANY'
+            if model.meta.instrument.pupil is not None:
+                pupil_name = model.meta.instrument.pupil
+
             (radius, radius_inner, radius_outer) = get_ref_data(
-                        tsophot_filename, pupil=model.meta.instrument.pupil)
+                        tsophot_filename, pupil=pupil_name)
             self.log.debug('Using reference file {}'.format(tsophot_filename))
             self.log.debug('radius = {}'.format(radius))
             self.log.debug('radius_inner = {}'.format(radius_inner))


Test patch:
diff --git a/jwst/associations/tests/test_level2_candidates.py b/jwst/associations/tests/test_level2_candidates.py
index 43ed70b69..ec9ebbfdf 100644
--- a/jwst/associations/tests/test_level2_candidates.py
+++ b/jwst/associations/tests/test_level2_candidates.py
@@ -21,12 +21,12 @@ from ..main import Main
         # Basic observation ACIDs
         (
             ['-i', 'o001'],
-            1
+            10
         ),
         # Whole program
         (
             [],
-            7
+            70
         ),
         # Discovered only
         (
@@ -36,7 +36,7 @@ from ..main import Main
         # Candidates only
         (
             ['--all-candidates'],
-            7
+            70
         ),
     ]
 )
diff --git a/jwst/associations/tests/test_wfss.py b/jwst/associations/tests/test_wfss.py
index 08a18ddc3..48018d54c 100644
--- a/jwst/associations/tests/test_wfss.py
+++ b/jwst/associations/tests/test_wfss.py
@@ -25,7 +25,7 @@ def test_niriss_wfss():
     asns = results.associations
 
     # Need 4 associations: image2, spec2, image3, spec3
-    assert len(asns) == 4
+    assert len(asns) == 12
     asn_types = [
         asn['asn_type']
         for asn in asns
diff --git a/jwst/tests_nightly/general/associations/test_generate.py b/jwst/tests_nightly/general/associations/test_generate.py
index 1f84165d5..f617a9d4f 100644
--- a/jwst/tests_nightly/general/associations/test_generate.py
+++ b/jwst/tests_nightly/general/associations/test_generate.py
@@ -12,7 +12,7 @@ def test_generate(full_pool_rules):
     """Run a full sized pool using all rules"""
     pool, rules, pool_fname = full_pool_rules
     asns = generate(pool, rules)
-    assert len(asns) == 96
+    assert len(asns) == 97
     for asn in asns:
         asn_name, asn_store = asn.dump()
         asn_table = load_asn(asn_store)
diff --git a/jwst/tests_nightly/general/associations/test_main.py b/jwst/tests_nightly/general/associations/test_main.py
index d4ec947a4..db28198c0 100644
--- a/jwst/tests_nightly/general/associations/test_main.py
+++ b/jwst/tests_nightly/general/associations/test_main.py
@@ -22,12 +22,12 @@ def test_script(full_pool_rules):
         'discover_Asn_IFU', 'candidate_Asn_Lv2WFSC',
         'candidate_Asn_Lv2ImageNonScience', 'discover_Asn_SpectralTarget',
         'candidate_Asn_Lv2ImageTSO', 'discover_Asn_SpectralSource',
-        'discover_Asn_Image', 'candidate_Asn_Lv2FGS'
+        'discover_Asn_Image', 'candidate_Asn_Lv2FGS', 'candidate_Asn_Lv3SpecAux'
     }
 
     generated = Main([pool_fname, '--dry-run'])
     asns = generated.associations
-    assert len(asns) == 442
+    assert len(asns) == 939
     assert len(generated.orphaned) == 61
     found_rules = set(
         asn['asn_rule']
@@ -42,9 +42,9 @@ def test_asn_candidates(full_pool_rules):
     pool, rules, pool_fname = full_pool_rules
 
     generated = Main([pool_fname, '--dry-run', '-i', 'o001'])
-    assert len(generated.associations) == 3
+    assert len(generated.associations) == 12
     generated = Main([pool_fname, '--dry-run', '-i', 'o001', 'o002'])
-    assert len(generated.associations) == 6
+    assert len(generated.associations) == 24
 
 
 @pytest.mark.slow