Repository: spacetelescope/jwst

Issue:
Level 2 associations and overwriting of products
# Issue
For Level2, there is a situation where different association candidates can be produced, resulting in the possibility that the '_cal' product will be replaced with a lesser association. 

The particular case is when there is a background association. In initial DP processing, there will not be an issue since the background candidate will always complete after the initial observation candidate. However, when reprocessing, both associations will exist, and if the observation candidate is processed after the background candidate, the final '_cal' file will be the bad, non-background-subtracted, result.

The generator-specific solution is to check for this case before writing out the final associations, and remove the "lesser" association candidate, when using the `--all-candidates` flag.

Discussion:
stscieisenhamer: Also resolves #3602/JP-767

Patch:
diff --git a/CHANGES.rst b/CHANGES.rst
index 1b6d6ffac..5b3130372 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -68,6 +68,8 @@ associations
 
 - Orders the elements in Level3 naming in alphabetical order [#3614]
 
+- Ensured that higher-order candidates only exist for Level2 associations [#3629]
+
 combine_1d
 ----------
 
diff --git a/jwst/associations/lib/diff.py b/jwst/associations/lib/diff.py
index eab73ffb8..007ecd027 100644
--- a/jwst/associations/lib/diff.py
+++ b/jwst/associations/lib/diff.py
@@ -8,6 +8,7 @@ import re
 from ..load_asn import load_asn
 
 logger = logging.getLogger(__name__)
+logger.addHandler(logging.NullHandler())
 
 
 def compare_asn_files(left_paths, right_paths):
@@ -73,12 +74,14 @@ def compare_asn_lists(left_asns, right_asns):
     """
 
     # Ensure that product names are unique
-    left_product_names = no_duplicate_products(left_asns)
-    right_product_names = no_duplicate_products(right_asns)
+    left_product_names, left_duplicates = get_product_names(left_asns)
+    right_product_names, right_duplicates = get_product_names(right_asns)
+    assert not left_duplicates, f'Left associations have duplicate products {left_duplicates} '
+    assert not right_duplicates, f'Right associations have duplicate products {left_duplicates} '
 
     # Ensure that the product name lists are the same.
     name_diff = left_product_names ^ right_product_names
-    assert not len(name_diff), (
+    assert not name_diff, (
         'Associations do not share a common set of products: {}'
         ''.format(name_diff)
     )
@@ -168,10 +171,15 @@ def _compare_asns(left, right):
             - key 'exptype` for each member
     """
 
-    # Metadata
+    # Assert that the same result type is the same.
     assert left['asn_type'] == right['asn_type'], \
         'Type mismatch {} != {}'.format(left['asn_type'], right['asn_type'])
 
+    # Assert that the level of association candidate is the same.
+    # Cannot guarantee value, but that the 'a'/'c'/'o' levels are similar.
+    assert left['asn_id'][0] == right['asn_id'][0], \
+        f"Candidate level mismatch left '{left['asn_id'][0]}' != right '{right['asn_id'][0]}'"
+
     # Membership
     return compare_membership(left, right)
 
@@ -288,8 +296,8 @@ def separate_products(asn):
     return separated
 
 
-def no_duplicate_products(asns):
-    """Ensure there are no duplicate product names in a list of associations
+def get_product_names(asns):
+    """Return product names from associations and flag duplicates
 
     Parameters
     ----------
@@ -297,14 +305,8 @@ def no_duplicate_products(asns):
 
     Returns
     -------
-    product_names: set(str[, ...])
-        The set of product names.
-
-    Raises
-    ------
-    AssertionError
-        If there are differences. The message will contain
-        all the differences.
+    product_names, duplicates: set(str[, ...]), [str[,...]]
+        2-tuple consisting of the set of product names and the list of duplicates.
     """
     product_names = [
         asn['products'][0]['name']
@@ -316,9 +318,9 @@ def no_duplicate_products(asns):
         for name, count in Counter(product_names).items()
         if count > 1
     ]
-    assert not len(dups), (
-        'Associations have the following product name duplication'
-        '{}'.format(dups)
-    )
+    if dups:
+        logger.debug(
+            'Duplicate product names: %s', dups
+        )
 
-    return set(product_names)
+    return set(product_names), dups
diff --git a/jwst/associations/lib/rules_level2_base.py b/jwst/associations/lib/rules_level2_base.py
index 6878b2b51..6fcdf9d35 100644
--- a/jwst/associations/lib/rules_level2_base.py
+++ b/jwst/associations/lib/rules_level2_base.py
@@ -1,4 +1,5 @@
 """Base classes which define the Level2 Associations"""
+from collections import defaultdict
 import copy
 import logging
 from os.path import (
@@ -18,6 +19,7 @@ from jwst.associations.lib.constraint import (
     Constraint,
     SimpleConstraint,
 )
+from jwst.associations.lib.diff import get_product_names
 from jwst.associations.lib.dms_base import (
     CORON_EXP_TYPES,
     DMSAttrConstraint,
@@ -456,6 +458,98 @@ class DMSLevel2bBase(DMSBaseMixin, Association):
 class Utility():
     """Utility functions that understand DMS Level 3 associations"""
 
+    @staticmethod
+    @RegistryMarker.callback('finalize')
+    def finalize(associations):
+        """Check validity and duplications in an association list
+
+        Parameters
+        ----------
+        associations:[association[, ...]]
+            List of associations
+
+        Returns
+        -------
+        finalized_associations : [association[, ...]]
+            The validated list of associations
+        """
+        finalized_asns = []
+        lv2_asns = []
+        for asn in associations:
+            if isinstance(asn, DMSLevel2bBase):
+                finalized = asn.finalize()
+                if finalized is not None:
+                    lv2_asns.extend(finalized)
+            else:
+                finalized_asns.append(asn)
+        lv2_asns = Utility.prune_duplicate_products(lv2_asns)
+
+        return finalized_asns + lv2_asns
+
+    @staticmethod
+    def merge_asns(associations):
+        """merge level2 associations
+
+        Parameters
+        ----------
+        associations : [asn(, ...)]
+            Associations to search for merging.
+
+        Returns
+        -------
+        associatons : [association(, ...)]
+            List of associations, some of which may be merged.
+        """
+        others = []
+        lv2_asns = []
+        for asn in associations:
+            if isinstance(asn, DMSLevel2bBase):
+                lv2_asns.append(asn)
+            else:
+                others.append(asn)
+
+        lv2_asns = Utility._merge_asns(lv2_asns)
+
+        return others + lv2_asns
+
+    @staticmethod
+    def prune_duplicate_products(asns):
+        """Remove duplicate products in favor of higher level versions
+
+        For Level 2 associations, since the products are always just the input
+        exposures, different candidates can be created for each exposure. Remove
+        those associations of lesser candidates.
+
+        The assumption is that there is only one product per association, before merging
+
+        Parameters
+        ----------
+        asns: [Association[,...]]
+            Associations to prune
+
+        Returns
+        pruned: [Association[,...]]
+            Pruned list of associations
+        """
+        product_names, dups = get_product_names(asns)
+        if not dups:
+            return asns
+
+        pruned = copy.copy(asns)
+        to_prune = defaultdict(list)
+        for asn in asns:
+            product_name = asn['products'][0]['name']
+            if product_name in dups:
+                    to_prune[product_name].append(asn)
+
+        for product_name, asns_to_prune in to_prune.items():
+            asns_to_prune = Utility.sort_by_candidate(asns_to_prune)
+            for asn in asns_to_prune[1:]:
+                pruned.remove(asn)
+
+        return pruned
+
+
     @staticmethod
     def rename_to_level2a(level1b_name, use_integrations=False):
         """Rename a Level 1b Exposure to another level
@@ -500,57 +594,28 @@ class Utility():
         return Utility_Level3.resequence(*args, **kwargs)
 
     @staticmethod
-    @RegistryMarker.callback('finalize')
-    def finalize(associations):
-        """Check validity and duplications in an association list
+    def sort_by_candidate(asns):
+        """Sort associations by candidate
 
         Parameters
         ----------
-        associations:[association[, ...]]
+        asns: [Association[,...]]
             List of associations
 
         Returns
         -------
-        finalized_associations : [association[, ...]]
-            The validated list of associations
-        """
-        finalized_asns = []
-        lv2_asns = []
-        for asn in associations:
-            if isinstance(asn, DMSLevel2bBase):
-                finalized = asn.finalize()
-                if finalized is not None:
-                    lv2_asns.extend(finalized)
-            else:
-                finalized_asns.append(asn)
+        sorted_by_candidate: [Associations[,...]]
+            New list of the associations sorted.
 
-        return finalized_asns + lv2_asns
-
-    @staticmethod
-    def merge_asns(associations):
-        """merge level2 associations
-
-        Parameters
-        ----------
-        associations : [asn(, ...)]
-            Associations to search for merging.
+        Notes
+        -----
+        The current definition of candidates allows strictly lexigraphical
+        sorting:
+        aXXXX > cXXXX > oXXX
 
-        Returns
-        -------
-        associatons : [association(, ...)]
-            List of associations, some of which may be merged.
+        If this changes, a comparision function will need be implemented
         """
-        others = []
-        lv2_asns = []
-        for asn in associations:
-            if isinstance(asn, DMSLevel2bBase):
-                lv2_asns.append(asn)
-            else:
-                others.append(asn)
-
-        lv2_asns = Utility._merge_asns(lv2_asns)
-
-        return others + lv2_asns
+        return sorted(asns, key=lambda asn: asn['asn_id'])
 
     @staticmethod
     def _merge_asns(asns):


Test patch:
diff --git a/jwst/associations/lib/tests/test_diff.py b/jwst/associations/lib/tests/test_diff.py
index 8e88e6a8b..2235ddb77 100644
--- a/jwst/associations/lib/tests/test_diff.py
+++ b/jwst/associations/lib/tests/test_diff.py
@@ -6,6 +6,7 @@ import jwst.associations.lib.diff as asn_diff
 
 standard_asn = {
     'asn_type': 'test',
+    'asn_id': 'c1001',
     'products': [
         {
             'name': 'product_a',
@@ -33,6 +34,7 @@ standard_asn = {
 
 badexptype_asn = {
     'asn_type': 'test',
+    'asn_id': 'c1001',
     'products': [
         {
             'name': 'product_a',
@@ -60,6 +62,7 @@ badexptype_asn = {
 
 dup_product_asn = {
     'asn_type': 'test',
+    'asn_id': 'c1001',
     'products': [
         {
             'name': 'product_a',
@@ -87,6 +90,7 @@ dup_product_asn = {
 
 disjoint_product_asn = {
     'asn_type': 'test',
+    'asn_id': 'c1001',
     'products': [
         {
             'name': 'product_a',
@@ -114,6 +118,7 @@ disjoint_product_asn = {
 
 badtype_asn = {
     'asn_type': 'badtype',
+    'asn_id': 'c1001',
     'products': [
         {
             'name': 'product_a',
@@ -141,6 +146,35 @@ badtype_asn = {
 
 badmember_asn = {
     'asn_type': 'test',
+    'asn_id': 'c1001',
+    'products': [
+        {
+            'name': 'product_a',
+            'members': [
+                {'expname': 'member_a_a', 'exptype': 'science'},
+                {'expname': 'member_a_a', 'exptype': 'science'}
+            ]
+        },
+        {
+            'name': 'product_b',
+            'members': [
+                {'expname': 'member_b_a', 'exptype': 'science'},
+                {'expname': 'member_b_b', 'exptype': 'science'}
+            ]
+        },
+        {
+            'name': 'product_d',
+            'members': [
+                {'expname': 'member_d_a', 'exptype': 'science'},
+                {'expname': 'member_d_b', 'exptype': 'science'}
+            ]
+        },
+    ]
+}
+
+badcandidate_asn = {
+    'asn_type': 'test',
+    'asn_id': 'o001',
     'products': [
         {
             'name': 'product_a',
@@ -174,7 +208,8 @@ badmember_asn = {
         disjoint_product_asn,
         badtype_asn,
         badmember_asn,
-        badexptype_asn
+        badexptype_asn,
+        badcandidate_asn,
     ]
 )
 def test_fails(mismatched, standard=standard_asn):
diff --git a/jwst/associations/tests/test_level2_background.py b/jwst/associations/tests/test_level2_background.py
index 34f1eccd5..ab7ee4195 100644
--- a/jwst/associations/tests/test_level2_background.py
+++ b/jwst/associations/tests/test_level2_background.py
@@ -46,7 +46,7 @@ def test_nrs_fixedslit_nod_chop():
     asns = generate(pool, registry_level2_only(
         global_constraints=constraint_all_candidates)
     )
-    assert len(asns) == 14
+    assert len(asns) == 8
     for asn in asns:
         assert asn['asn_rule'] in ['Asn_Lv2NRSFSS', 'Asn_Lv2SpecSpecial']
         if asn['asn_rule'] == 'Asn_Lv2SpecSpecial':
diff --git a/jwst/tests_nightly/general/associations/test_standards.py b/jwst/tests_nightly/general/associations/test_standards.py
index 4c8e5cac8..de8252483 100644
--- a/jwst/tests_nightly/general/associations/test_standards.py
+++ b/jwst/tests_nightly/general/associations/test_standards.py
@@ -65,28 +65,26 @@ standards = [
     MakePars('pool_004_wfs'),
     MakePars('pool_005_spec_niriss'),
     MakePars('pool_006_spec_nirspec'),
-    MakePars('pool_007_spec_miri', xfail='Not updated since PR #3399'),
-    MakePars('pool_009_spec_miri_lv2bkg', xfail='Issue #3041'),
-    MakePars('pool_010_spec_nirspec_lv2bkg', xfail='Issue #3041'),
-    MakePars('pool_011_spec_miri_lv2bkg_lrs', xfail='Issue #3041'),
+    MakePars('pool_007_spec_miri'),
+    MakePars('pool_009_spec_miri_lv2bkg'),
+    MakePars('pool_010_spec_nirspec_lv2bkg'),
+    MakePars('pool_011_spec_miri_lv2bkg_lrs'),
     MakePars('pool_013_coron_nircam'),
     MakePars('pool_014_ami_niriss'),
-    MakePars('pool_015_spec_nirspec_lv2bkg_reversed', main_args=LV2_ONLY_ARGS,
-             xfail='Issue #3041'),
-    MakePars('pool_016_spec_nirspec_lv2bkg_double', main_args=LV2_ONLY_ARGS,
-             xfail='Issue #3041'),
+    MakePars('pool_015_spec_nirspec_lv2bkg_reversed', main_args=LV2_ONLY_ARGS),
+    MakePars('pool_016_spec_nirspec_lv2bkg_double', main_args=LV2_ONLY_ARGS),
     MakePars('pool_017_spec_nirspec_lv2imprint'),
-    MakePars('pool_018_all_exptypes', main_args=LV2_ONLY_ARGS, xfail='Issue #3041'),
+    MakePars('pool_018_all_exptypes', main_args=LV2_ONLY_ARGS),
     MakePars('pool_019_niriss_wfss'),
     MakePars('pool_020_00009_image_miri'),
     MakePars('pool_021_tso'),
     MakePars('pool_022_tso_noflag'),
     MakePars('pool_023_nirspec_msa_3nod', main_args=LV2_ONLY_ARGS),
     MakePars('pool_024_nirspec_fss_nods'),
-    MakePars('pool_025_nirspec_fss_nod_chop', xfail='Issue #3041'),
+    MakePars('pool_025_nirspec_fss_nod_chop'),
     MakePars('pool_026_mir_image_tso'),
-    MakePars('pool_027_nirspec_ifu_nods', xfail='Issue #3041'),
-    MakePars('pool_028_mir_lrsfs_nods', xfail='Not updated since PR #3399'),
+    MakePars('pool_027_nirspec_ifu_nods'),
+    MakePars('pool_028_mir_lrsfs_nods'),
 ]