Repository: spacetelescope/jwst

Issue:
calwebb_ami3 failing in blendmodels
Issue [JP-778](https://jira.stsci.edu/browse/JP-778) was created by Howard Bushouse:

After recent updates to blendmodels and ami_analyze (not sure if they're connected) the calwebb_ami3 regression test is failing when attempting to run blendmodels after completing the ami_normalize step.

E.g. [https://plwishmaster.stsci.edu:8081/job/RT/job/JWST/274/testReport/jwst.tests_nightly.general.niriss.test_niriss_steps_single/TestAMIPipeline/_stable_deps__test_ami_pipeline/]

Discussion:

Patch:
diff --git a/CHANGES.rst b/CHANGES.rst
index 5b3130372..1b6d6ffac 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -68,8 +68,6 @@ associations
 
 - Orders the elements in Level3 naming in alphabetical order [#3614]
 
-- Ensured that higher-order candidates only exist for Level2 associations [#3629]
-
 combine_1d
 ----------
 
diff --git a/docs/jwst/model_blender/blender_rules.rst b/docs/jwst/model_blender/blender_rules.rst
index f62239909..dba0f53d5 100644
--- a/docs/jwst/model_blender/blender_rules.rst
+++ b/docs/jwst/model_blender/blender_rules.rst
@@ -30,7 +30,9 @@ includes
                       'mintime': mintime,
                       'maxtime': maxtime,
                       'mindate': mindate,
-                      'maxdate': maxdate}
+                      'maxdate': maxdate,
+		      'mindatetime': mindatetime,
+		      'maxdatetime': maxdatetime}
 
 The rules that should be referenced in the model schema definition are the
 keys defined for `jwst.model_blender.blender_rules.blender_funcs` listed
diff --git a/jwst/associations/lib/diff.py b/jwst/associations/lib/diff.py
index 007ecd027..eab73ffb8 100644
--- a/jwst/associations/lib/diff.py
+++ b/jwst/associations/lib/diff.py
@@ -8,7 +8,6 @@ import re
 from ..load_asn import load_asn
 
 logger = logging.getLogger(__name__)
-logger.addHandler(logging.NullHandler())
 
 
 def compare_asn_files(left_paths, right_paths):
@@ -74,14 +73,12 @@ def compare_asn_lists(left_asns, right_asns):
     """
 
     # Ensure that product names are unique
-    left_product_names, left_duplicates = get_product_names(left_asns)
-    right_product_names, right_duplicates = get_product_names(right_asns)
-    assert not left_duplicates, f'Left associations have duplicate products {left_duplicates} '
-    assert not right_duplicates, f'Right associations have duplicate products {left_duplicates} '
+    left_product_names = no_duplicate_products(left_asns)
+    right_product_names = no_duplicate_products(right_asns)
 
     # Ensure that the product name lists are the same.
     name_diff = left_product_names ^ right_product_names
-    assert not name_diff, (
+    assert not len(name_diff), (
         'Associations do not share a common set of products: {}'
         ''.format(name_diff)
     )
@@ -171,15 +168,10 @@ def _compare_asns(left, right):
             - key 'exptype` for each member
     """
 
-    # Assert that the same result type is the same.
+    # Metadata
     assert left['asn_type'] == right['asn_type'], \
         'Type mismatch {} != {}'.format(left['asn_type'], right['asn_type'])
 
-    # Assert that the level of association candidate is the same.
-    # Cannot guarantee value, but that the 'a'/'c'/'o' levels are similar.
-    assert left['asn_id'][0] == right['asn_id'][0], \
-        f"Candidate level mismatch left '{left['asn_id'][0]}' != right '{right['asn_id'][0]}'"
-
     # Membership
     return compare_membership(left, right)
 
@@ -296,8 +288,8 @@ def separate_products(asn):
     return separated
 
 
-def get_product_names(asns):
-    """Return product names from associations and flag duplicates
+def no_duplicate_products(asns):
+    """Ensure there are no duplicate product names in a list of associations
 
     Parameters
     ----------
@@ -305,8 +297,14 @@ def get_product_names(asns):
 
     Returns
     -------
-    product_names, duplicates: set(str[, ...]), [str[,...]]
-        2-tuple consisting of the set of product names and the list of duplicates.
+    product_names: set(str[, ...])
+        The set of product names.
+
+    Raises
+    ------
+    AssertionError
+        If there are differences. The message will contain
+        all the differences.
     """
     product_names = [
         asn['products'][0]['name']
@@ -318,9 +316,9 @@ def get_product_names(asns):
         for name, count in Counter(product_names).items()
         if count > 1
     ]
-    if dups:
-        logger.debug(
-            'Duplicate product names: %s', dups
-        )
+    assert not len(dups), (
+        'Associations have the following product name duplication'
+        '{}'.format(dups)
+    )
 
-    return set(product_names), dups
+    return set(product_names)
diff --git a/jwst/associations/lib/rules_level2_base.py b/jwst/associations/lib/rules_level2_base.py
index 6fcdf9d35..6878b2b51 100644
--- a/jwst/associations/lib/rules_level2_base.py
+++ b/jwst/associations/lib/rules_level2_base.py
@@ -1,5 +1,4 @@
 """Base classes which define the Level2 Associations"""
-from collections import defaultdict
 import copy
 import logging
 from os.path import (
@@ -19,7 +18,6 @@ from jwst.associations.lib.constraint import (
     Constraint,
     SimpleConstraint,
 )
-from jwst.associations.lib.diff import get_product_names
 from jwst.associations.lib.dms_base import (
     CORON_EXP_TYPES,
     DMSAttrConstraint,
@@ -458,98 +456,6 @@ class DMSLevel2bBase(DMSBaseMixin, Association):
 class Utility():
     """Utility functions that understand DMS Level 3 associations"""
 
-    @staticmethod
-    @RegistryMarker.callback('finalize')
-    def finalize(associations):
-        """Check validity and duplications in an association list
-
-        Parameters
-        ----------
-        associations:[association[, ...]]
-            List of associations
-
-        Returns
-        -------
-        finalized_associations : [association[, ...]]
-            The validated list of associations
-        """
-        finalized_asns = []
-        lv2_asns = []
-        for asn in associations:
-            if isinstance(asn, DMSLevel2bBase):
-                finalized = asn.finalize()
-                if finalized is not None:
-                    lv2_asns.extend(finalized)
-            else:
-                finalized_asns.append(asn)
-        lv2_asns = Utility.prune_duplicate_products(lv2_asns)
-
-        return finalized_asns + lv2_asns
-
-    @staticmethod
-    def merge_asns(associations):
-        """merge level2 associations
-
-        Parameters
-        ----------
-        associations : [asn(, ...)]
-            Associations to search for merging.
-
-        Returns
-        -------
-        associatons : [association(, ...)]
-            List of associations, some of which may be merged.
-        """
-        others = []
-        lv2_asns = []
-        for asn in associations:
-            if isinstance(asn, DMSLevel2bBase):
-                lv2_asns.append(asn)
-            else:
-                others.append(asn)
-
-        lv2_asns = Utility._merge_asns(lv2_asns)
-
-        return others + lv2_asns
-
-    @staticmethod
-    def prune_duplicate_products(asns):
-        """Remove duplicate products in favor of higher level versions
-
-        For Level 2 associations, since the products are always just the input
-        exposures, different candidates can be created for each exposure. Remove
-        those associations of lesser candidates.
-
-        The assumption is that there is only one product per association, before merging
-
-        Parameters
-        ----------
-        asns: [Association[,...]]
-            Associations to prune
-
-        Returns
-        pruned: [Association[,...]]
-            Pruned list of associations
-        """
-        product_names, dups = get_product_names(asns)
-        if not dups:
-            return asns
-
-        pruned = copy.copy(asns)
-        to_prune = defaultdict(list)
-        for asn in asns:
-            product_name = asn['products'][0]['name']
-            if product_name in dups:
-                    to_prune[product_name].append(asn)
-
-        for product_name, asns_to_prune in to_prune.items():
-            asns_to_prune = Utility.sort_by_candidate(asns_to_prune)
-            for asn in asns_to_prune[1:]:
-                pruned.remove(asn)
-
-        return pruned
-
-
     @staticmethod
     def rename_to_level2a(level1b_name, use_integrations=False):
         """Rename a Level 1b Exposure to another level
@@ -594,28 +500,57 @@ class Utility():
         return Utility_Level3.resequence(*args, **kwargs)
 
     @staticmethod
-    def sort_by_candidate(asns):
-        """Sort associations by candidate
+    @RegistryMarker.callback('finalize')
+    def finalize(associations):
+        """Check validity and duplications in an association list
 
         Parameters
         ----------
-        asns: [Association[,...]]
+        associations:[association[, ...]]
             List of associations
 
         Returns
         -------
-        sorted_by_candidate: [Associations[,...]]
-            New list of the associations sorted.
+        finalized_associations : [association[, ...]]
+            The validated list of associations
+        """
+        finalized_asns = []
+        lv2_asns = []
+        for asn in associations:
+            if isinstance(asn, DMSLevel2bBase):
+                finalized = asn.finalize()
+                if finalized is not None:
+                    lv2_asns.extend(finalized)
+            else:
+                finalized_asns.append(asn)
 
-        Notes
-        -----
-        The current definition of candidates allows strictly lexigraphical
-        sorting:
-        aXXXX > cXXXX > oXXX
+        return finalized_asns + lv2_asns
+
+    @staticmethod
+    def merge_asns(associations):
+        """merge level2 associations
 
-        If this changes, a comparision function will need be implemented
+        Parameters
+        ----------
+        associations : [asn(, ...)]
+            Associations to search for merging.
+
+        Returns
+        -------
+        associatons : [association(, ...)]
+            List of associations, some of which may be merged.
         """
-        return sorted(asns, key=lambda asn: asn['asn_id'])
+        others = []
+        lv2_asns = []
+        for asn in associations:
+            if isinstance(asn, DMSLevel2bBase):
+                lv2_asns.append(asn)
+            else:
+                others.append(asn)
+
+        lv2_asns = Utility._merge_asns(lv2_asns)
+
+        return others + lv2_asns
 
     @staticmethod
     def _merge_asns(asns):
diff --git a/jwst/datamodels/schemas/core.schema.yaml b/jwst/datamodels/schemas/core.schema.yaml
index df1e07575..26516930a 100644
--- a/jwst/datamodels/schemas/core.schema.yaml
+++ b/jwst/datamodels/schemas/core.schema.yaml
@@ -4,10 +4,8 @@ properties:
     type: object
     properties:
       date:
-        anyOf:
-          - $ref: http://stsci.edu/schemas/asdf/time/time-1.0.0
-          - type: string
         title: "[yyyy-mm-ddThh:mm:ss.ss] UTC date file created"
+        type: string
         fits_keyword: DATE
         blend_table: True
       origin:
@@ -138,10 +136,8 @@ properties:
         type: object
         properties:
           date:
-            anyOf:
-              - $ref: http://stsci.edu/schemas/asdf/time/time-1.0.0
-              - type: string
             title: "[yyyy-mm-dd] UTC date at start of exposure"
+            type: string
             fits_keyword: DATE-OBS
             blend_table: True
             blend_rule: mindate
@@ -156,13 +152,15 @@ properties:
             title: "Date-time start of data acquisition"
             fits_keyword: DATE-BEG
             blend_table: True
-            blend_rule: mindate
+            blend_rule: mindatetime
           date_end:
             type: string
             title: "[yyyy-mm-dd] UTC date at end of exposure"
             fits_keyword: DATE-END
             blend_table: True
-            blend_rule: maxdate
+            # Despite what the title might lead you to believe, this keyword
+            # actually contains a full datetime string.
+            blend_rule: maxdatetime
           time_end:
             title: "[hh:mm:ss.sss] UTC time at end of exposure"
             type: string
diff --git a/jwst/model_blender/blendrules.py b/jwst/model_blender/blendrules.py
index 4ecf18028..ed701c01b 100644
--- a/jwst/model_blender/blendrules.py
+++ b/jwst/model_blender/blendrules.py
@@ -84,14 +84,24 @@ def last(items):
     return None
 
 def mindate(items):
-    """Return the minimum time from a list of time strings."""
-    time_list = Time(items)
-    return time_list.min()
+    """Return the minimum date from a list of date strings in yyyy-mm-dd format."""
+    time_list = Time(items, format="iso", in_subfmt="date", out_subfmt="date")
+    return str(time_list.min())
 
 def maxdate(items):
-    """Return the maximum time from a list of time strings."""
-    time_list = Time(items)
-    return time_list.max()
+    """Return the maximum date from a list of date strings in yyyy-mm-dd format."""
+    time_list = Time(items, format="iso", in_subfmt="date", out_subfmt="date")
+    return str(time_list.max())
+
+def mindatetime(items):
+    """Return the minimum datetime from a list of datetime strings in ISO-8601 format."""
+    time_list = Time(items, format="isot")
+    return str(time_list.min())
+
+def maxdatetime(items):
+    """Return the maximum datetime from a list of datetime strings in ISO-8601 format."""
+    time_list = Time(items, format="isot")
+    return str(time_list.max())
 
 def mintime(items):
     times = [_isotime(time_str) for time_str in items]
@@ -123,7 +133,9 @@ blender_funcs = {'first': first,
                  'mintime': mintime,
                  'maxtime': maxtime,
                  'mindate': mindate,
-                 'maxdate': maxdate}
+                 'maxdate': maxdate,
+                 'mindatetime': mindatetime,
+                 'maxdatetime': maxdatetime}
 
 
 # Classes for managing keyword rules


Test patch:
diff --git a/jwst/associations/lib/tests/test_diff.py b/jwst/associations/lib/tests/test_diff.py
index 2235ddb77..8e88e6a8b 100644
--- a/jwst/associations/lib/tests/test_diff.py
+++ b/jwst/associations/lib/tests/test_diff.py
@@ -6,7 +6,6 @@ import jwst.associations.lib.diff as asn_diff
 
 standard_asn = {
     'asn_type': 'test',
-    'asn_id': 'c1001',
     'products': [
         {
             'name': 'product_a',
@@ -34,7 +33,6 @@ standard_asn = {
 
 badexptype_asn = {
     'asn_type': 'test',
-    'asn_id': 'c1001',
     'products': [
         {
             'name': 'product_a',
@@ -62,7 +60,6 @@ badexptype_asn = {
 
 dup_product_asn = {
     'asn_type': 'test',
-    'asn_id': 'c1001',
     'products': [
         {
             'name': 'product_a',
@@ -90,7 +87,6 @@ dup_product_asn = {
 
 disjoint_product_asn = {
     'asn_type': 'test',
-    'asn_id': 'c1001',
     'products': [
         {
             'name': 'product_a',
@@ -118,7 +114,6 @@ disjoint_product_asn = {
 
 badtype_asn = {
     'asn_type': 'badtype',
-    'asn_id': 'c1001',
     'products': [
         {
             'name': 'product_a',
@@ -146,35 +141,6 @@ badtype_asn = {
 
 badmember_asn = {
     'asn_type': 'test',
-    'asn_id': 'c1001',
-    'products': [
-        {
-            'name': 'product_a',
-            'members': [
-                {'expname': 'member_a_a', 'exptype': 'science'},
-                {'expname': 'member_a_a', 'exptype': 'science'}
-            ]
-        },
-        {
-            'name': 'product_b',
-            'members': [
-                {'expname': 'member_b_a', 'exptype': 'science'},
-                {'expname': 'member_b_b', 'exptype': 'science'}
-            ]
-        },
-        {
-            'name': 'product_d',
-            'members': [
-                {'expname': 'member_d_a', 'exptype': 'science'},
-                {'expname': 'member_d_b', 'exptype': 'science'}
-            ]
-        },
-    ]
-}
-
-badcandidate_asn = {
-    'asn_type': 'test',
-    'asn_id': 'o001',
     'products': [
         {
             'name': 'product_a',
@@ -208,8 +174,7 @@ badcandidate_asn = {
         disjoint_product_asn,
         badtype_asn,
         badmember_asn,
-        badexptype_asn,
-        badcandidate_asn,
+        badexptype_asn
     ]
 )
 def test_fails(mismatched, standard=standard_asn):
diff --git a/jwst/associations/tests/test_level2_background.py b/jwst/associations/tests/test_level2_background.py
index ab7ee4195..34f1eccd5 100644
--- a/jwst/associations/tests/test_level2_background.py
+++ b/jwst/associations/tests/test_level2_background.py
@@ -46,7 +46,7 @@ def test_nrs_fixedslit_nod_chop():
     asns = generate(pool, registry_level2_only(
         global_constraints=constraint_all_candidates)
     )
-    assert len(asns) == 8
+    assert len(asns) == 14
     for asn in asns:
         assert asn['asn_rule'] in ['Asn_Lv2NRSFSS', 'Asn_Lv2SpecSpecial']
         if asn['asn_rule'] == 'Asn_Lv2SpecSpecial':
diff --git a/jwst/model_blender/tests/test_blend.py b/jwst/model_blender/tests/test_blend.py
index 927f023aa..0a78292d0 100644
--- a/jwst/model_blender/tests/test_blend.py
+++ b/jwst/model_blender/tests/test_blend.py
@@ -12,8 +12,9 @@ start_times = [57877.00359994354, 57877.0168373584, 57877.03126958496]
 exp_times = [107.3676, 107.3676, 107.3676]
 end_times = [57877.0048426241, 57877.01808003896, 57877.03251226551]
 filenames = ['image1_cal.fits', 'image2_cal.fits', 'image3_cal.fits']
-dates = ['2017-11-30T13:52:20.367', '2017-11-11T15:14:29.176',
-         '2017-11-11T15:15:06.118']
+datetimes = ['2017-11-30T13:52:20.367', '2017-11-11T15:14:29.176',
+             '2017-11-11T15:15:06.118']
+dates = ['2017-11-30', '2017-11-11', '2017-12-10']
 instrument_names = ['NIRCAM'] * 3
 
 
@@ -28,13 +29,17 @@ def setup():
                     'meta.exposure.end_time': end_times,
                     'meta.filename': filenames,
                     'meta.instrument.name': instrument_names,
-                    'meta.date': dates}
+                    'meta.date': datetimes,
+                    'meta.observation.date': dates,
+                    'meta.observation.date_beg': datetimes}
     OUTPUT_VALUES = {'meta.exposure.start_time': start_times[0],
                     'meta.exposure.exposure_time': np.sum(exp_times),
                     'meta.exposure.end_time': end_times[-1],
                     'meta.filename': filenames[0],
                     'meta.instrument.name': instrument_names[0],
-                    'meta.date': dates[0]}
+                    'meta.date': datetimes[0],
+                    'meta.observation.date': dates[1],
+                    'meta.observation.date_beg': datetimes[1]}
 
     for i, tfile in enumerate(TMP_FILES):
         for attr in INPUT_VALUES:
diff --git a/jwst/tests_nightly/general/associations/test_standards.py b/jwst/tests_nightly/general/associations/test_standards.py
index de8252483..4c8e5cac8 100644
--- a/jwst/tests_nightly/general/associations/test_standards.py
+++ b/jwst/tests_nightly/general/associations/test_standards.py
@@ -65,26 +65,28 @@ standards = [
     MakePars('pool_004_wfs'),
     MakePars('pool_005_spec_niriss'),
     MakePars('pool_006_spec_nirspec'),
-    MakePars('pool_007_spec_miri'),
-    MakePars('pool_009_spec_miri_lv2bkg'),
-    MakePars('pool_010_spec_nirspec_lv2bkg'),
-    MakePars('pool_011_spec_miri_lv2bkg_lrs'),
+    MakePars('pool_007_spec_miri', xfail='Not updated since PR #3399'),
+    MakePars('pool_009_spec_miri_lv2bkg', xfail='Issue #3041'),
+    MakePars('pool_010_spec_nirspec_lv2bkg', xfail='Issue #3041'),
+    MakePars('pool_011_spec_miri_lv2bkg_lrs', xfail='Issue #3041'),
     MakePars('pool_013_coron_nircam'),
     MakePars('pool_014_ami_niriss'),
-    MakePars('pool_015_spec_nirspec_lv2bkg_reversed', main_args=LV2_ONLY_ARGS),
-    MakePars('pool_016_spec_nirspec_lv2bkg_double', main_args=LV2_ONLY_ARGS),
+    MakePars('pool_015_spec_nirspec_lv2bkg_reversed', main_args=LV2_ONLY_ARGS,
+             xfail='Issue #3041'),
+    MakePars('pool_016_spec_nirspec_lv2bkg_double', main_args=LV2_ONLY_ARGS,
+             xfail='Issue #3041'),
     MakePars('pool_017_spec_nirspec_lv2imprint'),
-    MakePars('pool_018_all_exptypes', main_args=LV2_ONLY_ARGS),
+    MakePars('pool_018_all_exptypes', main_args=LV2_ONLY_ARGS, xfail='Issue #3041'),
     MakePars('pool_019_niriss_wfss'),
     MakePars('pool_020_00009_image_miri'),
     MakePars('pool_021_tso'),
     MakePars('pool_022_tso_noflag'),
     MakePars('pool_023_nirspec_msa_3nod', main_args=LV2_ONLY_ARGS),
     MakePars('pool_024_nirspec_fss_nods'),
-    MakePars('pool_025_nirspec_fss_nod_chop'),
+    MakePars('pool_025_nirspec_fss_nod_chop', xfail='Issue #3041'),
     MakePars('pool_026_mir_image_tso'),
-    MakePars('pool_027_nirspec_ifu_nods'),
-    MakePars('pool_028_mir_lrsfs_nods'),
+    MakePars('pool_027_nirspec_ifu_nods', xfail='Issue #3041'),
+    MakePars('pool_028_mir_lrsfs_nods', xfail='Not updated since PR #3399'),
 ]