Repository: spacetelescope/stdatamodels (language: python)

Issue:
Remove stcal as dependency
https://jira.stsci.edu/browse/JP-3110

*Moving the datamodels from jwst to stdatamodels required adding stcal as a dependency for stdatamodels. This should not be necessary as this code is only needed for the datamodels in jwst.*


Discussion / hints:

Fix patch:
diff --git a/CHANGES.rst b/CHANGES.rst
index 2f6aabca..1092bf03 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -1,7 +1,12 @@
 1.3.0 (unreleased)
 ==================
 
-- 
+Other
+-----
+
+- Move the ``dqflags`` and related code from ``stcal`` to this package
+  so that the ``stcal`` dependency can be dropped. [#134]
+
 
 1.2.0 (2023-03-02)
 ==================
diff --git a/pyproject.toml b/pyproject.toml
index 08450c2a..76574fb3 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -16,9 +16,8 @@ dependencies = [
     "asdf>=2.14.1",
     "asdf-astropy>=0.3.0",
     "psutil>=5.7.2",
-    "numpy>=1.16",
+    "numpy>=1.18",
     "astropy>=5.0.4",
-    "stcal>=1.3.2",
 ]
 dynamic = [
     "version",
@@ -52,6 +51,7 @@ test = [
     "pytest-doctestplus",
     "pytest-openfiles>=0.5.0",
     "crds>=11.16.14",
+    "scipy>=1.5",
 ]
 docs = [
     "sphinx",
diff --git a/src/stdatamodels/basic_utils.py b/src/stdatamodels/basic_utils.py
new file mode 100644
index 00000000..3d2abd99
--- /dev/null
+++ b/src/stdatamodels/basic_utils.py
@@ -0,0 +1,44 @@
+import re
+
+
+__all__ = ["multiple_replace"]
+
+
+def multiple_replace(string, rep_dict):
+    """Single-pass replacement of multiple substrings
+
+    Similar to `str.replace`, except that a dictionary of replacements
+    can be specified.
+
+    The replacements are done in a single-pass. This means that a previous
+    replacement will not be replaced by a subsequent match.
+
+    Parameters
+    ----------
+    string: str
+        The source string to have replacements done on it.
+
+    rep_dict: dict
+        The replacements were key is the input substring and
+        value is the replacement
+
+    Returns
+    -------
+    replaced: str
+        New string with the replacements done
+
+    Examples
+    --------
+    Basic example that also demonstrates the single-pass nature.
+    If the replacements where chained, the result would have been
+    'lamb lamb'
+
+    >>> multiple_replace('button mutton', {'but': 'mut', 'mutton': 'lamb'})
+    'mutton lamb'
+
+    """
+    pattern = re.compile(
+        "|".join([re.escape(k) for k in sorted(rep_dict, key=len, reverse=True)]),
+        flags=re.DOTALL
+    )
+    return pattern.sub(lambda x: rep_dict[x.group(0)], string)
diff --git a/src/stdatamodels/dqflags.py b/src/stdatamodels/dqflags.py
new file mode 100644
index 00000000..8f036e32
--- /dev/null
+++ b/src/stdatamodels/dqflags.py
@@ -0,0 +1,108 @@
+"""
+Implementation
+--------------
+
+The flags are implemented as "bit flags": Each flag is assigned a bit position
+in a byte, or multi-byte word, of memory. If that bit is set, the flag assigned
+to that bit is interpreted as being set or active.
+
+The data structure that stores bit flags is just the standard Python `int`,
+which provides 32 bits. Bits of an integer are most easily referred to using
+the formula `2**bit_number` where `bit_number` is the 0-index bit of interest.
+"""
+from astropy.nddata.bitmask import interpret_bit_flags as ap_interpret_bit_flags
+from stdatamodels.basic_utils import multiple_replace
+
+
+def interpret_bit_flags(bit_flags, flip_bits=None, mnemonic_map=None):
+    """Converts input bit flags to a single integer value (bit mask) or `None`.
+
+    Wraps `astropy.nddate.bitmask.interpret_bit_flags`, allowing the
+    bit mnemonics to be used in place of integers.
+
+    Parameters
+    ----------
+    bit_flags : int, str, list, None
+        See `astropy.nddate.bitmask.interpret_bit_flags`.
+        Also allows strings using Roman mnemonics
+
+    flip_bits : bool, None
+        See `astropy.nddata.bitmask.interpret_bit_flags`.
+
+    mnemonic_map : {str: int[,...]}
+        Dictionary associating the mnemonic string to an integer value
+        representing the set bit for that mnemonic.
+
+    Returns
+    -------
+    bitmask : int or None
+        Returns an integer bit mask formed from the input bit value or `None`
+        if input ``bit_flags`` parameter is `None` or an empty string.
+        If input string value was prepended with '~' (or ``flip_bits`` was set
+        to `True`), then returned value will have its bits flipped
+        (inverse mask).
+    """
+    if mnemonic_map is None:
+        raise TypeError("`mnemonic_map` is a required argument")
+    bit_flags_dm = bit_flags
+    if isinstance(bit_flags, str):
+        dm_flags = {
+            key: str(val)
+            for key, val in mnemonic_map.items()
+        }
+        bit_flags_dm = multiple_replace(bit_flags, dm_flags)
+
+    return ap_interpret_bit_flags(bit_flags_dm, flip_bits=flip_bits)
+
+
+def dqflags_to_mnemonics(dqflags, mnemonic_map):
+    """Interpret value as bit flags and return the mnemonics
+
+    Parameters
+    ----------
+    dqflags : int-like
+        The value to interpret as DQ flags
+
+    mnemonic_map: {str: int[,...]}
+        Dictionary associating the mnemonic string to an integer value
+        representing the set bit for that mnemonic.
+
+    Returns
+    -------
+    mnemonics : {str[,...]}
+        Set of mnemonics represented by the set bit flags
+
+    Examples
+    --------
+    >>> pixel = {'GOOD':             0,      # No bits set, all is good
+    ...          'DO_NOT_USE':       2**0,   # Bad pixel. Do not use
+    ...          'SATURATED':        2**1,   # Pixel saturated during exposure
+    ...          'JUMP_DET':         2**2,   # Jump detected during exposure
+    ...          }
+
+    >>> group = {'GOOD':       pixel['GOOD'],
+    ...          'DO_NOT_USE': pixel['DO_NOT_USE'],
+    ...          'SATURATED':  pixel['SATURATED'],
+    ...          }
+
+    >>> dqflags_to_mnemonics(1, pixel)
+    {'DO_NOT_USE'}
+
+    >>> dqflags_to_mnemonics(7, pixel)             #doctest: +SKIP
+    {'JUMP_DET', 'DO_NOT_USE', 'SATURATED'}
+
+    >>> dqflags_to_mnemonics(7, pixel) == {'JUMP_DET', 'DO_NOT_USE', 'SATURATED'}
+    True
+
+    >>> dqflags_to_mnemonics(1, mnemonic_map=pixel)
+    {'DO_NOT_USE'}
+
+    >>> dqflags_to_mnemonics(1, mnemonic_map=group)
+    {'DO_NOT_USE'}
+    """
+    mnemonics = {
+        mnemonic
+        for mnemonic, value in mnemonic_map.items()
+        if (dqflags & value)
+    }
+    return mnemonics
diff --git a/src/stdatamodels/dynamicdq.py b/src/stdatamodels/dynamicdq.py
new file mode 100644
index 00000000..131109f2
--- /dev/null
+++ b/src/stdatamodels/dynamicdq.py
@@ -0,0 +1,51 @@
+import numpy as np
+
+import logging
+log = logging.getLogger(__name__)
+log.setLevel(logging.DEBUG)
+log.addHandler(logging.NullHandler())
+
+
+def dynamic_mask(input_model, mnemonic_map):
+    """
+    Return a mask model given a mask with dynamic DQ flags.
+
+    Dynamic flags define what each plane refers to using the DQ_DEF extension.
+
+    Parameters
+    ----------
+    input_model : ``MaskModel``
+        An instance of a Mask model defined in jwst or romancal.
+    mnemonic_map : dict
+
+    Returns
+    -------
+    dqmask : ndarray
+        A Numpy array
+    """
+
+    dq_table = input_model.dq_def
+    # Get the DQ array and the flag definitions
+    if (dq_table is not None and
+        not np.isscalar(dq_table) and
+        len(dq_table.shape) and
+            len(dq_table)):
+        #
+        # Make an empty mask
+        dqmask = np.zeros(input_model.dq.shape, dtype=input_model.dq.dtype)
+        for record in dq_table:
+            bitplane = record['VALUE']
+            dqname = record['NAME'].strip()
+            try:
+                standard_bitvalue = mnemonic_map[dqname]
+            except KeyError:
+                log.warning('Keyword %s does not correspond to an existing '
+                            'DQ mnemonic, so will be ignored' % (dqname))
+                continue
+            just_this_bit = np.bitwise_and(input_model.dq, bitplane)
+            pixels = np.where(just_this_bit != 0)
+            dqmask[pixels] = np.bitwise_or(dqmask[pixels], standard_bitvalue)
+    else:
+        dqmask = input_model.dq
+
+    return dqmask
diff --git a/src/stdatamodels/jwst/datamodels/dark.py b/src/stdatamodels/jwst/datamodels/dark.py
index 35b4e5d9..b82e2aff 100644
--- a/src/stdatamodels/jwst/datamodels/dark.py
+++ b/src/stdatamodels/jwst/datamodels/dark.py
@@ -1,4 +1,4 @@
-from stcal.dynamicdq import dynamic_mask
+from stdatamodels.dynamicdq import dynamic_mask
 from .dqflags import pixel
 from .reference import ReferenceFileModel
 
diff --git a/src/stdatamodels/jwst/datamodels/darkMIRI.py b/src/stdatamodels/jwst/datamodels/darkMIRI.py
index ca855e9f..c172c2d5 100644
--- a/src/stdatamodels/jwst/datamodels/darkMIRI.py
+++ b/src/stdatamodels/jwst/datamodels/darkMIRI.py
@@ -1,4 +1,4 @@
-from stcal.dynamicdq import dynamic_mask
+from stdatamodels.dynamicdq import dynamic_mask
 from .dqflags import pixel
 from .reference import ReferenceFileModel
 
diff --git a/src/stdatamodels/jwst/datamodels/dqflags.py b/src/stdatamodels/jwst/datamodels/dqflags.py
index a8d2bf55..f9785a26 100644
--- a/src/stdatamodels/jwst/datamodels/dqflags.py
+++ b/src/stdatamodels/jwst/datamodels/dqflags.py
@@ -19,8 +19,8 @@
 
 # These imports are here for backwards compatibility
 from astropy.nddata.bitmask import interpret_bit_flags as ap_interpret_bit_flags
-from stcal.dqflags import interpret_bit_flags, dqflags_to_mnemonics
-from stcal.basic_utils import multiple_replace
+from stdatamodels.dqflags import interpret_bit_flags, dqflags_to_mnemonics
+from stdatamodels.basic_utils import multiple_replace
 
 # Pixel-specific flags
 pixel = {'GOOD':             0,      # No bits set, all is good
diff --git a/src/stdatamodels/jwst/datamodels/flat.py b/src/stdatamodels/jwst/datamodels/flat.py
index 8c266306..67546483 100644
--- a/src/stdatamodels/jwst/datamodels/flat.py
+++ b/src/stdatamodels/jwst/datamodels/flat.py
@@ -1,4 +1,4 @@
-from stcal.dynamicdq import dynamic_mask
+from stdatamodels.dynamicdq import dynamic_mask
 from .dqflags import pixel
 from .reference import ReferenceFileModel
 
diff --git a/src/stdatamodels/jwst/datamodels/fringe.py b/src/stdatamodels/jwst/datamodels/fringe.py
index ad59a11a..870ae798 100644
--- a/src/stdatamodels/jwst/datamodels/fringe.py
+++ b/src/stdatamodels/jwst/datamodels/fringe.py
@@ -1,4 +1,4 @@
-from stcal.dynamicdq import dynamic_mask
+from stdatamodels.dynamicdq import dynamic_mask
 from .dqflags import pixel
 from .reference import ReferenceFileModel
 
diff --git a/src/stdatamodels/jwst/datamodels/lastframe.py b/src/stdatamodels/jwst/datamodels/lastframe.py
index b7e07850..2298c0c6 100644
--- a/src/stdatamodels/jwst/datamodels/lastframe.py
+++ b/src/stdatamodels/jwst/datamodels/lastframe.py
@@ -1,4 +1,4 @@
-from stcal.dynamicdq import dynamic_mask
+from stdatamodels.dynamicdq import dynamic_mask
 from .dqflags import pixel
 from .reference import ReferenceFileModel
 
diff --git a/src/stdatamodels/jwst/datamodels/linearity.py b/src/stdatamodels/jwst/datamodels/linearity.py
index 9c29ca62..a67c28eb 100644
--- a/src/stdatamodels/jwst/datamodels/linearity.py
+++ b/src/stdatamodels/jwst/datamodels/linearity.py
@@ -1,4 +1,4 @@
-from stcal.dynamicdq import dynamic_mask
+from stdatamodels.dynamicdq import dynamic_mask
 from .dqflags import pixel
 from .reference import ReferenceFileModel
 
diff --git a/src/stdatamodels/jwst/datamodels/mask.py b/src/stdatamodels/jwst/datamodels/mask.py
index f0fe2b02..488083a6 100644
--- a/src/stdatamodels/jwst/datamodels/mask.py
+++ b/src/stdatamodels/jwst/datamodels/mask.py
@@ -1,5 +1,5 @@
 from .reference import ReferenceFileModel
-from stcal.dynamicdq import dynamic_mask
+from stdatamodels.dynamicdq import dynamic_mask
 from .dqflags import pixel
 
 __all__ = ['MaskModel']
diff --git a/src/stdatamodels/jwst/datamodels/nirspec_flat.py b/src/stdatamodels/jwst/datamodels/nirspec_flat.py
index f9d2c33f..8833e8cc 100644
--- a/src/stdatamodels/jwst/datamodels/nirspec_flat.py
+++ b/src/stdatamodels/jwst/datamodels/nirspec_flat.py
@@ -1,4 +1,4 @@
-from stcal.dynamicdq import dynamic_mask
+from stdatamodels.dynamicdq import dynamic_mask
 from .dqflags import pixel
 from .reference import ReferenceFileModel
 
diff --git a/src/stdatamodels/jwst/datamodels/persat.py b/src/stdatamodels/jwst/datamodels/persat.py
index 4c21fdee..c35a4168 100644
--- a/src/stdatamodels/jwst/datamodels/persat.py
+++ b/src/stdatamodels/jwst/datamodels/persat.py
@@ -1,4 +1,4 @@
-from stcal.dynamicdq import dynamic_mask
+from stdatamodels.dynamicdq import dynamic_mask
 from .dqflags import pixel
 from .reference import ReferenceFileModel
 
diff --git a/src/stdatamodels/jwst/datamodels/photom.py b/src/stdatamodels/jwst/datamodels/photom.py
index 59279baf..8645004a 100644
--- a/src/stdatamodels/jwst/datamodels/photom.py
+++ b/src/stdatamodels/jwst/datamodels/photom.py
@@ -1,4 +1,4 @@
-from stcal.dynamicdq import dynamic_mask
+from stdatamodels.dynamicdq import dynamic_mask
 from .dqflags import pixel
 from .reference import ReferenceFileModel
 
diff --git a/src/stdatamodels/jwst/datamodels/reference.py b/src/stdatamodels/jwst/datamodels/reference.py
index 1862181b..c1fa08a3 100644
--- a/src/stdatamodels/jwst/datamodels/reference.py
+++ b/src/stdatamodels/jwst/datamodels/reference.py
@@ -1,7 +1,7 @@
 import warnings
 
 from stdatamodels.validate import ValidationWarning
-from stcal.dynamicdq import dynamic_mask
+from stdatamodels.dynamicdq import dynamic_mask
 
 from .model_base import JwstDataModel
 from .dqflags import pixel
diff --git a/src/stdatamodels/jwst/datamodels/reset.py b/src/stdatamodels/jwst/datamodels/reset.py
index 21177df1..349b13e4 100644
--- a/src/stdatamodels/jwst/datamodels/reset.py
+++ b/src/stdatamodels/jwst/datamodels/reset.py
@@ -1,5 +1,5 @@
 
-from stcal.dynamicdq import dynamic_mask
+from stdatamodels.dynamicdq import dynamic_mask
 from .dqflags import pixel
 from .reference import ReferenceFileModel
 
diff --git a/src/stdatamodels/jwst/datamodels/saturation.py b/src/stdatamodels/jwst/datamodels/saturation.py
index 646ca380..984b4335 100644
--- a/src/stdatamodels/jwst/datamodels/saturation.py
+++ b/src/stdatamodels/jwst/datamodels/saturation.py
@@ -1,4 +1,4 @@
-from stcal.dynamicdq import dynamic_mask
+from stdatamodels.dynamicdq import dynamic_mask
 from .dqflags import pixel
 from .reference import ReferenceFileModel
 
diff --git a/src/stdatamodels/jwst/datamodels/superbias.py b/src/stdatamodels/jwst/datamodels/superbias.py
index d1a29a8e..05def370 100644
--- a/src/stdatamodels/jwst/datamodels/superbias.py
+++ b/src/stdatamodels/jwst/datamodels/superbias.py
@@ -1,4 +1,4 @@
-from stcal.dynamicdq import dynamic_mask
+from stdatamodels.dynamicdq import dynamic_mask
 from .dqflags import pixel
 from .reference import ReferenceFileModel
 
diff --git a/src/stdatamodels/jwst/datamodels/trapdensity.py b/src/stdatamodels/jwst/datamodels/trapdensity.py
index 6e592613..cab866ed 100644
--- a/src/stdatamodels/jwst/datamodels/trapdensity.py
+++ b/src/stdatamodels/jwst/datamodels/trapdensity.py
@@ -1,4 +1,4 @@
-from stcal.dynamicdq import dynamic_mask
+from stdatamodels.dynamicdq import dynamic_mask
 from .dqflags import pixel
 from .reference import ReferenceFileModel
 
diff --git a/src/stdatamodels/jwst/datamodels/wfssbkg.py b/src/stdatamodels/jwst/datamodels/wfssbkg.py
index e974877f..579dc934 100644
--- a/src/stdatamodels/jwst/datamodels/wfssbkg.py
+++ b/src/stdatamodels/jwst/datamodels/wfssbkg.py
@@ -1,4 +1,4 @@
-from stcal.dynamicdq import dynamic_mask
+from stdatamodels.dynamicdq import dynamic_mask
 from .dqflags import pixel
 from .reference import ReferenceFileModel
 


Test patch:
diff --git a/tests/test_dq.py b/tests/test_dq.py
new file mode 100644
index 00000000..0c220131
--- /dev/null
+++ b/tests/test_dq.py
@@ -0,0 +1,42 @@
+from stdatamodels import dqflags
+
+PIXEL = {'GOOD':             0,      # No bits set, all is good
+         'DO_NOT_USE':       2**0,   # Bad pixel. Do not use.
+         'SATURATED':        2**1,   # Pixel saturated during exposure
+         'JUMP_DET':         2**2,   # Jump detected during exposure
+         'DROPOUT':          2**3,   # Data lost in transmission
+         'OUTLIER':          2**4,   # Flagged by outlier detection (was RESERVED_1)
+         'PERSISTENCE':      2**5,   # High persistence (was RESERVED_2)
+         'AD_FLOOR':         2**6,   # Below A/D floor (0 DN, was RESERVED_3)
+         'RESERVED_4':       2**7,   #
+         'UNRELIABLE_ERROR': 2**8,   # Uncertainty exceeds quoted error
+         'NON_SCIENCE':      2**9,   # Pixel not on science portion of detector
+         'DEAD':             2**10,  # Dead pixel
+         'HOT':              2**11,  # Hot pixel
+         'WARM':             2**12,  # Warm pixel
+         'LOW_QE':           2**13,  # Low quantum efficiency
+         'RC':               2**14,  # RC pixel
+         'TELEGRAPH':        2**15,  # Telegraph pixel
+         'NONLINEAR':        2**16,  # Pixel highly nonlinear
+         'BAD_REF_PIXEL':    2**17,  # Reference pixel cannot be used
+         'NO_FLAT_FIELD':    2**18,  # Flat field cannot be measured
+         'NO_GAIN_VALUE':    2**19,  # Gain cannot be measured
+         'NO_LIN_CORR':      2**20,  # Linearity correction not available
+         'NO_SAT_CHECK':     2**21,  # Saturation check not available
+         'UNRELIABLE_BIAS':  2**22,  # Bias variance large
+         'UNRELIABLE_DARK':  2**23,  # Dark variance large
+         'UNRELIABLE_SLOPE': 2**24,  # Slope variance large (i.e., noisy pixel)
+         'UNRELIABLE_FLAT':  2**25,  # Flat variance large
+         'OPEN':             2**26,  # Open pixel (counts move to adjacent pixels)
+         'ADJ_OPEN':         2**27,  # Adjacent to open pixel
+         'UNRELIABLE_RESET': 2**28,  # Sensitive to reset anomaly
+         'MSA_FAILED_OPEN':  2**29,  # Pixel sees light from failed-open shutter
+         'OTHER_BAD_PIXEL':  2**30,  # A catch-all flag
+         'REFERENCE_PIXEL':  2**31,  # Pixel is a reference pixel
+         }
+
+
+def test_dqflags():
+    assert dqflags.dqflags_to_mnemonics(1, PIXEL) == {'DO_NOT_USE'}
+    assert dqflags.dqflags_to_mnemonics(7, PIXEL) == {'JUMP_DET', 'DO_NOT_USE', 'SATURATED'}
+    assert dqflags.interpret_bit_flags('DO_NOT_USE + WARM', mnemonic_map=PIXEL) == 4097