Repository: theupdateframework/python-tuf (python)

Issue:
Metadata API: add `_Delegator.get_delegate_signing_status()`
`_Delegator.verify_delegate` provides a binary answer for whether a delegated role has enough valid signatures to meet the threshold of keys defined by its delegator or not. 

While this is sufficient for client validation, TUF applications might also be interested in more granular information, e.g. when implementing a distributed signing feature for TUF metadata, so they can tell users how many signatures by what keys are still missing. 

Interesting information would be:
- set of valid signatures
- set of invalid signatures (this would require validating all signatures; `verify_threshold` only verifies until threshold is met)
- optional: eligible keys, used keys, remaining keys (maybe out of scope)


Given that threshold verification is such a crucial part of the Metadata API and can be done wrong so easily, adding a new public function --  e.g. `_Delegator.get_delegate_signing_status()` -- to Metadata API seems like a good idea.





Pull request description:
Metadata API: add get_verification_result method
The method returns detailed information about signature verification of a delegated role metadata.

Its implementation is taken from the verify_delegate method and slightly updated. verify_delegate now is a thin wrapper on top of get_verification_result.

fixes #2449



Patch:
diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml
index 47a367b9..e7dd6cc6 100644
--- a/.github/workflows/cd.yml
+++ b/.github/workflows/cd.yml
@@ -18,7 +18,7 @@ jobs:
     needs: test
     steps:
       - name: Checkout release tag
-        uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
+        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0
         with:
           ref: ${{ github.event.workflow_run.head_branch }}
 
diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
index bb37806d..f3fef3a3 100644
--- a/.github/workflows/codeql-analysis.yml
+++ b/.github/workflows/codeql-analysis.yml
@@ -22,12 +22,12 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
+      uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0
 
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@6a28655e3dcb49cb0840ea372fd6d17733edd8a4 # v2.21.8
+      uses: github/codeql-action/init@00e563ead9f72a8461b24876bee2d0c2e8bd2ee8 # v2.21.5
       with:
         languages: 'python'
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@6a28655e3dcb49cb0840ea372fd6d17733edd8a4 # v2.21.8
+      uses: github/codeql-action/analyze@00e563ead9f72a8461b24876bee2d0c2e8bd2ee8 # v2.21.5
diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml
index 78521288..d58301c7 100644
--- a/.github/workflows/dependency-review.yml
+++ b/.github/workflows/dependency-review.yml
@@ -15,6 +15,6 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: 'Checkout Repository'
-        uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
+        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0
       - name: 'Dependency Review'
         uses: actions/dependency-review-action@6c5ccdad469c9f8a2996bfecaec55a631a347034 # v3.1.0
diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml
index 620a0e99..f93e1ce9 100644
--- a/.github/workflows/scorecards.yml
+++ b/.github/workflows/scorecards.yml
@@ -21,7 +21,7 @@ jobs:
 
     steps:
       - name: "Checkout code"
-        uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
+        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0
 
       - name: "Run analysis"
         uses: ossf/scorecard-action@08b4669551908b1024bb425080c797723083c031 # v2.2.0
@@ -34,6 +34,6 @@ jobs:
           publish_results: true
 
       - name: "Upload to code-scanning dashboard"
-        uses: github/codeql-action/upload-sarif@6a28655e3dcb49cb0840ea372fd6d17733edd8a4 # v2.21.8
+        uses: github/codeql-action/upload-sarif@00e563ead9f72a8461b24876bee2d0c2e8bd2ee8 # v2.21.5
         with:
           sarif_file: results.sarif
diff --git a/.github/workflows/specification-version-check.yml b/.github/workflows/specification-version-check.yml
index 7cb257f9..713a5a20 100644
--- a/.github/workflows/specification-version-check.yml
+++ b/.github/workflows/specification-version-check.yml
@@ -14,7 +14,7 @@ jobs:
     outputs:
       version: ${{ steps.get-version.outputs.version }}
     steps:
-      - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
+      - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0
       - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0
         with:
           python-version: "3.x"
diff --git a/pyproject.toml b/pyproject.toml
index bb2c2b2f..89497d69 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -8,7 +8,7 @@ name = "tuf"
 description = "A secure updater framework for Python"
 readme = "README.md"
 license = { text = "MIT OR Apache-2.0" }
-requires-python = ">=3.8"
+requires-python = ">=3.7"
 authors = [
   { email = "theupdateframework@googlegroups.com" },
 ]
@@ -33,6 +33,7 @@ classifiers = [
   "Operating System :: POSIX :: Linux",
   "Programming Language :: Python",
   "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3.7",
   "Programming Language :: Python :: 3.8",
   "Programming Language :: Python :: 3.9",
   "Programming Language :: Python :: 3.10",
diff --git a/requirements/lint.txt b/requirements/lint.txt
index 5046a9ad..77c1ab8b 100644
--- a/requirements/lint.txt
+++ b/requirements/lint.txt
@@ -6,9 +6,9 @@
 # Lint tools
 # (We are not so interested in the specific versions of the tools: the versions
 # are pinned to prevent unexpected linting failures when tools update)
-black==23.9.1
+black==23.7.0
 isort==5.12.0
-pylint==2.17.7
+pylint==2.17.5
 mypy==1.5.1
 bandit==1.7.5
 pydocstyle==6.3.0
diff --git a/requirements/main.txt b/requirements/main.txt
index e1d3346d..1e638aad 100644
--- a/requirements/main.txt
+++ b/requirements/main.txt
@@ -6,5 +6,40 @@
 # 'pinned.txt' is updated on GitHub with Dependabot, which
 # triggers CI/CD builds to automatically test against updated dependencies.
 #
+#
+# NOTE: 'pip-compile' only adds dependencies relevant for the Python version,
+# in which it is executed. Moreover, it does not add environment markers of
+# transitive dependencies.
+# The official recommendation for cross-environment usage of pip-compile tends
+# towards separate requirements files for each environment (see
+# jazzband/pip-tools#651), this seem like an overkill for tuf, where we only
+# have a few conditional dependencies, i.e. dependencies that are required on
+# Python < 3 only.
+#
+#
+# Below instructions can be used to re-generate 'pinned.txt', e.g.
+# if:
+# - requirements are added or removed from this file
+# - Python version support is changed
+# - CI/CD build breaks due to updates (e.g. transitive dependency conflicts)
+#
+# 1. Use this script to create a pinned requirements file for each Python
+#    version
+# ```
+# for v in 3.7 3.8 3.9 3.10 3.11; do
+#   mkvirtualenv tuf-env-${v} -p python${v};
+#   python3 -m pip install pip-tools;
+#   pip-compile --no-header -o requirements-${v}.txt main.txt;
+#   deactivate;
+#   rmvirtualenv tuf-env-${v};
+# done;
+#
+# ```
+# 2. Use this command to merge per-version files
+#    `sort -o pinned.txt -u requirements-?.?.txt`
+# 2. Manually add environment markers to pinned.txt
+# 3. Use this command to remove per-version files
+#    `rm requirements-?.?.txt`
+#
 securesystemslib[crypto, pynacl]
 requests
diff --git a/requirements/pinned.txt b/requirements/pinned.txt
index 8cd23857..87306a61 100644
--- a/requirements/pinned.txt
+++ b/requirements/pinned.txt
@@ -1,10 +1,10 @@
 certifi==2023.7.22        # via requests
-cffi==1.16.0              # via cryptography, pynacl
-charset-normalizer==3.3.0 # via requests
-cryptography==41.0.4      # via securesystemslib
+cffi==1.15.1              # via cryptography, pynacl
+charset-normalizer==3.2.0 # via requests
+cryptography==41.0.3      # via securesystemslib
 idna==3.4                 # via requests
 pycparser==2.21           # via cffi
 pynacl==1.5.0             # via securesystemslib
 requests==2.31.0
 securesystemslib[crypto,pynacl]==0.29.0
-urllib3==2.0.6           # via requests
+urllib3==2.0.4           # via requests
diff --git a/tuf/api/metadata.py b/tuf/api/metadata.py
index 07b349b6..205678a3 100644
--- a/tuf/api/metadata.py
+++ b/tuf/api/metadata.py
@@ -33,6 +33,7 @@
 import io
 import logging
 import tempfile
+from dataclasses import dataclass
 from datetime import datetime
 from typing import (
     IO,
@@ -43,6 +44,7 @@
     Iterator,
     List,
     Optional,
+    Set,
     Tuple,
     Type,
     TypeVar,
@@ -646,6 +648,37 @@ def to_dict(self) -> Dict[str, Any]:
         }
 
 
+@dataclass
+class VerificationResult:
+    """Signature verification result for delegated role metadata.
+
+    Attributes:
+        verified: True, if threshold of signatures is met.
+        signed: Set of delegated keyids, which validly signed.
+        unsigned: Set of delegated keyids, which did not validly sign.
+
+    """
+
+    verified: bool
+    signed: Set[str]
+    unsigned: Set[str]
+
+    def __bool__(self) -> bool:
+        return self.verified
+
+    def union(self, other: "VerificationResult") -> "VerificationResult":
+        """Combine two verification results.
+
+        Can be used to verify, if root metadata is signed by the threshold of
+        keys of previous root and the threshold of keys of itself.
+        """
+        return VerificationResult(
+            self.verified and other.verified,
+            self.signed | other.signed,
+            self.unsigned | other.unsigned,
+        )
+
+
 class _DelegatorMixin(metaclass=abc.ABCMeta):
     """Class that implements verify_delegate() for Root and Targets"""
 
@@ -665,17 +698,16 @@ def get_key(self, keyid: str) -> Key:
         """
         raise NotImplementedError
 
-    def verify_delegate(
+    def get_verification_result(
         self,
         delegated_role: str,
         payload: bytes,
         signatures: Dict[str, Signature],
-    ) -> None:
-        """Verify signature threshold for delegated role.
+    ) -> VerificationResult:
+        """Return signature threshold verification result for delegated role.
 
-        Verify that there are enough valid ``signatures`` over ``payload``, to
-        meet the threshold of keys for ``delegated_role``, as defined by the
-        delegator (``self``).
+        NOTE: Unlike `verify_delegate()` this method does not raise, if the
+        role metadata is not fully verified.
 
         Args:
             delegated_role: Name of the delegated role to verify
@@ -683,36 +715,68 @@ def verify_delegate(
             signatures: Signatures over payload bytes
 
         Raises:
-            UnsignedMetadataError: ``delegated_role`` was not signed with
-                required threshold of keys for ``role_name``.
             ValueError: no delegation was found for ``delegated_role``.
         """
         role = self.get_delegated_role(delegated_role)
 
-        # verify that delegated_metadata is signed by threshold of unique keys
-        signing_keys = set()
+        signed = set()
+        unsigned = set()
+
         for keyid in role.keyids:
             try:
                 key = self.get_key(keyid)
             except ValueError:
+                unsigned.add(keyid)
                 logger.info("No key for keyid %s", keyid)
                 continue
 
             if keyid not in signatures:
+                unsigned.add(keyid)
                 logger.info("No signature for keyid %s", keyid)
                 continue
 
             sig = signatures[keyid]
             try:
                 key.verify_signature(sig, payload)
-                signing_keys.add(keyid)
+                signed.add(keyid)
             except sslib_exceptions.UnverifiedSignatureError:
+                unsigned.add(keyid)
                 logger.info("Key %s failed to verify %s", keyid, delegated_role)
 
-        if len(signing_keys) < role.threshold:
+        return VerificationResult(
+            len(signed) >= role.threshold, signed, unsigned
+        )
+
+    def verify_delegate(
+        self,
+        delegated_role: str,
+        payload: bytes,
+        signatures: Dict[str, Signature],
+    ) -> None:
+        """Verify signature threshold for delegated role.
+
+        Verify that there are enough valid ``signatures`` over ``payload``, to
+        meet the threshold of keys for ``delegated_role``, as defined by the
+        delegator (``self``).
+
+        Args:
+            delegated_role: Name of the delegated role to verify
+            payload: Signed payload bytes for the delegated role
+            signatures: Signatures over payload bytes
+
+        Raises:
+            UnsignedMetadataError: ``delegated_role`` was not signed with
+                required threshold of keys for ``role_name``.
+            ValueError: no delegation was found for ``delegated_role``.
+        """
+        result = self.get_verification_result(
+            delegated_role, payload, signatures
+        )
+        if not result:
+            role = self.get_delegated_role(delegated_role)
             raise UnsignedMetadataError(
-                f"{delegated_role} was signed by {len(signing_keys)}/"
-                f"{role.threshold} keys",
+                f"{delegated_role} was signed by {len(result.signed)}/"
+                f"{role.threshold} keys"
             )
 
 


Test patch:
diff --git a/.github/workflows/_test.yml b/.github/workflows/_test.yml
index 9d4c7464..4113a0f8 100644
--- a/.github/workflows/_test.yml
+++ b/.github/workflows/_test.yml
@@ -12,7 +12,7 @@ jobs:
 
     steps:
       - name: Checkout TUF
-        uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
+        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0
 
       - name: Set up Python 3.x
         uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0
@@ -37,7 +37,7 @@ jobs:
       # (sslib main) on Linux/Python3.x only.
       matrix:
         toxenv: [py]
-        python-version: ["3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
         os: [ubuntu-latest, macos-latest, windows-latest]
         include:
           - python-version: 3.x
@@ -47,13 +47,16 @@ jobs:
 
     env:
       # Set TOXENV env var to tell tox which testenv (see tox.ini) to use
+      # NOTE: The Python 2.7 runner has two Python versions on the path (see
+      # setup-python below), so we tell tox explicitly to use the 'py27'
+      # testenv. For all other runners the toxenv configured above suffices.
       TOXENV: ${{ matrix.toxenv }}
 
     runs-on: ${{ matrix.os }}
 
     steps:
       - name: Checkout TUF
-        uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
+        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0
 
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0
diff --git a/requirements/test.txt b/requirements/test.txt
index 08f8db8a..e9ed7feb 100644
--- a/requirements/test.txt
+++ b/requirements/test.txt
@@ -4,4 +4,4 @@
 -r pinned.txt
 
 # coverage measurement
-coverage==7.3.1
+coverage==7.2.7
diff --git a/tests/test_api.py b/tests/test_api.py
index adada930..517ff5bd 100755
--- a/tests/test_api.py
+++ b/tests/test_api.py
@@ -47,6 +47,7 @@
     TargetFile,
     Targets,
     Timestamp,
+    VerificationResult,
 )
 from tuf.api.serialization import DeserializationError, SerializationError
 from tuf.api.serialization.json import JSONSerializer
@@ -470,6 +471,96 @@ def test_signed_verify_delegate(self) -> None:
             Snapshot.type, snapshot_md.signed_bytes, snapshot_md.signatures
         )
 
+    def test_signed_get_verification_result(self) -> None:
+        # Setup: Load test metadata and keys
+        root_path = os.path.join(self.repo_dir, "metadata", "root.json")
+        root = Metadata[Root].from_file(root_path)
+        initial_root_keyids = root.signed.roles[Root.type].keyids
+        self.assertEqual(len(initial_root_keyids), 1)
+        key1_id = initial_root_keyids[0]
+        key2 = self.keystore[Timestamp.type]
+        key2_id = key2["keyid"]
+        key3_id = "123456789abcdefg"
+        key4 = self.keystore[Snapshot.type]
+        key4_id = key4["keyid"]
+
+        # Test: 1 authorized key, 1 valid signature
+        result = root.signed.get_verification_result(
+            Root.type, root.signed_bytes, root.signatures
+        )
+        self.assertTrue(result.verified)
+        self.assertEqual(result.signed, {key1_id})
+        self.assertEqual(result.unsigned, set())
+
+        # Test: 2 authorized keys, 1 invalid signature
+        # Adding a key, i.e. metadata change, invalidates existing signature
+        root.signed.add_key(
+            SSlibKey.from_securesystemslib_key(key2),
+            Root.type,
+        )
+        result = root.signed.get_verification_result(
+            Root.type, root.signed_bytes, root.signatures
+        )
+        self.assertFalse(result.verified)
+        self.assertEqual(result.signed, set())
+        self.assertEqual(result.unsigned, {key1_id, key2_id})
+
+        # Test: 3 authorized keys, 1 invalid signature, 1 key missing key data
+        # Adding a keyid w/o key, fails verification the same as no signature
+        # or an invalid signature for that key
+        root.signed.roles[Root.type].keyids.append(key3_id)
+        result = root.signed.get_verification_result(
+            Root.type, root.signed_bytes, root.signatures
+        )
+        self.assertFalse(result.verified)
+        self.assertEqual(result.signed, set())
+        self.assertEqual(result.unsigned, {key1_id, key2_id, key3_id})
+
+        # Test: 3 authorized keys, 1 valid signature, 1 invalid signature, 1
+        # key missing key data
+        root.sign(SSlibSigner(key2), append=True)
+        result = root.signed.get_verification_result(
+            Root.type, root.signed_bytes, root.signatures
+        )
+        self.assertTrue(result.verified)
+        self.assertEqual(result.signed, {key2_id})
+        self.assertEqual(result.unsigned, {key1_id, key3_id})
+
+        # Test: 3 authorized keys, 1 valid signature, 1 invalid signature, 1
+        # key missing key data, 1 ignored unrelated signature
+        root.sign(SSlibSigner(key4), append=True)
+        self.assertEqual(
+            set(root.signatures.keys()), {key1_id, key2_id, key4_id}
+        )
+        self.assertTrue(result.verified)
+        self.assertEqual(result.signed, {key2_id})
+        self.assertEqual(result.unsigned, {key1_id, key3_id})
+
+        # See test_signed_verify_delegate for more related tests ...
+
+    def test_signed_verification_result_union(self) -> None:
+        # Test all possible "unions" (AND) of "verified" field
+        data = [
+            (True, True, True),
+            (True, False, False),
+            (False, True, False),
+            (False, False, False),
+        ]
+
+        for a_part, b_part, ab_part in data:
+            self.assertEqual(
+                VerificationResult(a_part, set(), set()).union(
+                    VerificationResult(b_part, set(), set())
+                ),
+                VerificationResult(ab_part, set(), set()),
+            )
+
+        # Test exemplary union (|) of "signed" and "unsigned" fields
+        a = VerificationResult(True, {"1"}, {"2"})
+        b = VerificationResult(True, {"3"}, {"4"})
+        ab = VerificationResult(True, {"1", "3"}, {"2", "4"})
+        self.assertEqual(a.union(b), ab)
+
     def test_key_class(self) -> None:
         # Test if from_securesystemslib_key removes the private key from keyval
         # of a securesystemslib key dictionary.
diff --git a/tests/test_metadata_eq_.py b/tests/test_metadata_eq_.py
index dcadb444..c8de6147 100644
--- a/tests/test_metadata_eq_.py
+++ b/tests/test_metadata_eq_.py
@@ -113,7 +113,8 @@ def test_md_eq_signatures_reversed_order(self) -> None:
         md.signatures = {"a": Signature("a", "a"), "b": Signature("b", "b")}
         md_2 = copy.deepcopy(md)
         # Reverse signatures order in md_2.
-        md_2.signatures = dict(reversed(md_2.signatures.items()))
+        # In python3.7 we need to cast to a list and then reverse.
+        md_2.signatures = dict(reversed(list(md_2.signatures.items())))
         # Assert that both objects are not the same because of signatures order.
         self.assertNotEqual(md, md_2)
 
@@ -167,8 +168,9 @@ def test_delegations_eq_roles_reversed_order(self) -> None:
 
         # Create a second delegations obj with reversed roles order
         delegations_2 = copy.deepcopy(delegations)
+        # In python3.7 we need to cast to a list and then reverse.
         assert isinstance(delegations.roles, dict)
-        delegations_2.roles = dict(reversed(delegations.roles.items()))
+        delegations_2.roles = dict(reversed(list(delegations.roles.items())))
 
         # Both objects are not the equal because of delegated roles order.
         self.assertNotEqual(delegations, delegations_2)