Repository: spdx/tools-python

Issue:
Implement External Package Reference validation
part of #307

This is probably the most involved part of the validation process, earning it [an own annex](https://spdx.github.io/spdx-spec/v2.3-RC1/external-repository-identifiers/) in the specification.

Also add tests for this

Comments:

Patch:
diff --git a/src/spdx/model/package.py b/src/spdx/model/package.py
index bbeff65..0a62170 100644
--- a/src/spdx/model/package.py
+++ b/src/spdx/model/package.py
@@ -11,7 +11,7 @@
 from dataclasses import field
 from datetime import datetime
 from enum import Enum, auto
-from typing import Optional, Union, List
+from typing import Optional, Union, List, Dict
 
 from spdx.model.actor import Actor
 from spdx.model.checksum import Checksum
@@ -54,6 +54,13 @@ class ExternalPackageRefCategory(Enum):
     OTHER = auto()
 
 
+CATEGORY_TO_EXTERNAL_PACKAGE_REF_TYPES: Dict[ExternalPackageRefCategory, List[str]] = {
+    ExternalPackageRefCategory.SECURITY : ["cpe22Type", "cpe23Type", "advisory", "fix", "url", "swid"],
+    ExternalPackageRefCategory.PACKAGE_MANAGER : ["maven-central", "npm", "nuget", "bower", "purl"],
+    ExternalPackageRefCategory.PERSISTENT_ID : ["swh", "gitoid"]
+}
+
+
 @dataclass_with_properties
 class ExternalPackageRef:
     category: ExternalPackageRefCategory
diff --git a/src/spdx/validation/external_package_ref_validator.py b/src/spdx/validation/external_package_ref_validator.py
index 10ff0ee..7be94f6 100644
--- a/src/spdx/validation/external_package_ref_validator.py
+++ b/src/spdx/validation/external_package_ref_validator.py
@@ -8,11 +8,34 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import re
+from typing import List, Dict
 
-from typing import List
+from spdx.model.package import ExternalPackageRef, ExternalPackageRefCategory, CATEGORY_TO_EXTERNAL_PACKAGE_REF_TYPES
+from spdx.validation.uri_validators import validate_url, validate_uri
+from spdx.validation.validation_message import ValidationMessage, ValidationContext, SpdxElementType
 
-from spdx.model.package import ExternalPackageRef
-from spdx.validation.validation_message import ValidationMessage
+CPE22TYPE_REGEX = r'^c[pP][eE]:/[AHOaho]?(:[A-Za-z0-9._\-~%]*){0,6}$'
+CPE23TYPE_REGEX = r'^cpe:2\.3:[aho\*\-](:(((\?*|\*?)([a-zA-Z0-9\-\._]|(\\[\\\*\?!"#$$%&\'\(\)\+,\/:;<=>@\[\]\^`\{\|}~]))+(\?*|\*?))|[\*\-])){5}(:(([a-zA-Z]{2,3}(-([a-zA-Z]{2}|[0-9]{3}))?)|[\*\-]))(:(((\?*|\*?)([a-zA-Z0-9\-\._]|(\\[\\\*\?!"#$$%&\'\(\)\+,\/:;<=>@\[\]\^`\{\|}~]))+(\?*|\*?))|[\*\-])){4}$'
+MAVEN_CENTRAL_REGEX = r'^[^:]+:[^:]+(:[^:]+)?$'
+NPM_REGEX = r'^[^@]+@[^@]+$'
+NUGET_REGEX = r'^[^/]+/[^/]+$'
+BOWER_REGEX = r'^[^#]+#[^#]+$'
+PURL_REGEX = r'^pkg:.+(\/.+)?\/.+(@.+)?(\?.+)?(#.+)?$'
+SWH_REGEX = r'^swh:1:(snp|rel|rev|dir|cnt):[0-9a-fA-F]{40}$'
+GITOID_REGEX = r'^gitoid:(blob|tree|commit|tag):(sha1:[0-9a-fA-F]{40}|sha256:[0-9a-fA-F]{64})$'
+
+TYPE_TO_REGEX: Dict[str, str] = {
+    "cpe22Type": CPE22TYPE_REGEX,
+    "cpe23Type": CPE23TYPE_REGEX,
+    "maven-central": MAVEN_CENTRAL_REGEX,
+    "npm": NPM_REGEX,
+    "nuget": NUGET_REGEX,
+    "bower": BOWER_REGEX,
+    "purl": PURL_REGEX,
+    "swh": SWH_REGEX,
+    "gitoid": GITOID_REGEX
+}
 
 
 def validate_external_package_refs(external_package_refs: List[ExternalPackageRef], parent_id: str) -> List[
@@ -25,5 +48,47 @@ def validate_external_package_refs(external_package_refs: List[ExternalPackageRe
 
 
 def validate_external_package_ref(external_package_ref: ExternalPackageRef, parent_id: str) -> List[ValidationMessage]:
-    # TODO: https://github.com/spdx/tools-python/issues/373
+    context = ValidationContext(parent_id=parent_id, element_type=SpdxElementType.EXTERNAL_PACKAGE_REF,
+                                full_element=external_package_ref)
+
+    category = external_package_ref.category
+    locator = external_package_ref.locator
+    reference_type = external_package_ref.reference_type
+
+    if category == ExternalPackageRefCategory.OTHER:
+        if " " in locator:
+            return [ValidationMessage(
+                f"externalPackageRef locator in category OTHER must contain no spaces, but is: {locator}",
+                context)]
+        return []
+
+    if reference_type not in CATEGORY_TO_EXTERNAL_PACKAGE_REF_TYPES[category]:
+        return [ValidationMessage(
+            f"externalPackageRef type in category {category.name} must be one of {CATEGORY_TO_EXTERNAL_PACKAGE_REF_TYPES[category]}, but is: {reference_type}",
+            context)]
+
+    if reference_type in ["advisory", "fix", "url"]:
+        if validate_url(locator):
+            return [ValidationMessage(
+                f'externalPackageRef locator of type "{reference_type}" must be a valid URL, but is: {locator}',
+                context)]
+        return []
+
+    if reference_type == "swid":
+        if validate_uri(locator) or not locator.startswith("swid"):
+            return [ValidationMessage(
+                f'externalPackageRef locator of type "swid" must be a valid URI with scheme swid, but is: {locator}',
+                context)]
+        return []
+
+    return validate_against_regex(locator, reference_type, context)
+
+
+def validate_against_regex(string_to_validate: str, reference_type: str, context: ValidationContext) -> List[
+    ValidationMessage]:
+    regex = TYPE_TO_REGEX[reference_type]
+    if not re.match(regex, string_to_validate):
+        return [ValidationMessage(
+            f'externalPackageRef locator of type "{reference_type}" must conform with the regex {regex}, but is: {string_to_validate}',
+            context)]
     return []


Test patch:
diff --git a/tests/spdx/validation/test_external_package_ref_validator.py b/tests/spdx/validation/test_external_package_ref_validator.py
index ee0c386..f0085c8 100644
--- a/tests/spdx/validation/test_external_package_ref_validator.py
+++ b/tests/spdx/validation/test_external_package_ref_validator.py
@@ -13,25 +13,137 @@ from typing import List
 
 import pytest
 
-from spdx.validation.external_package_ref_validator import validate_external_package_ref
+from spdx.model.package import ExternalPackageRef, ExternalPackageRefCategory
+from spdx.validation.external_package_ref_validator import validate_external_package_ref, CPE22TYPE_REGEX, \
+    CPE23TYPE_REGEX, MAVEN_CENTRAL_REGEX, NPM_REGEX, NUGET_REGEX, BOWER_REGEX, PURL_REGEX, SWH_REGEX, GITOID_REGEX
 from spdx.validation.validation_message import ValidationMessage, ValidationContext, SpdxElementType
-from tests.spdx.fixtures import external_package_ref_fixture
 
 
-def test_valid_external_package_ref():
-    external_package_ref = external_package_ref_fixture()
+@pytest.mark.parametrize("category, reference_type, locator",
+                         [(ExternalPackageRefCategory.SECURITY, "cpe22Type",
+                           "cpe:/o:canonical:ubuntu_linux:10.04:-:lts"),
+                          (ExternalPackageRefCategory.SECURITY, "cpe23Type",
+                           "cpe:2.3:o:canonical:ubuntu_linux:10.04:-:lts:*:*:*:*:*"),
+                          (ExternalPackageRefCategory.SECURITY, "advisory",
+                           "https://nvd.nist.gov/vuln/detail/CVE-2020-28498"),
+                          (ExternalPackageRefCategory.SECURITY, "fix",
+                           "https://github.com/indutny/elliptic/commit/441b7428"),
+                          (ExternalPackageRefCategory.SECURITY, "url",
+                           "https://github.com/christianlundkvist/blog/blob/master/2020_05_26_secp256k1_twist_attacks/secp256k1_twist_attacks.md"),
+                          (ExternalPackageRefCategory.SECURITY, "swid", "swid:2df9de35-0aff-4a86-ace6-f7dddd1ade4c"),
+                          (ExternalPackageRefCategory.PACKAGE_MANAGER, "maven-central",
+                           "org.apache.tomcat:tomcat:9.0.0.M4"),
+                          (ExternalPackageRefCategory.PACKAGE_MANAGER, "npm", "http-server@0.3.0"),
+                          (ExternalPackageRefCategory.PACKAGE_MANAGER, "nuget", "Microsoft.AspNet.MVC/5.0.0"),
+                          (ExternalPackageRefCategory.PACKAGE_MANAGER, "bower", "modernizr#2.6.2"),
+                          (ExternalPackageRefCategory.PACKAGE_MANAGER, "purl",
+                           "pkg:docker/debian@sha256:2f04d3d33b6027bb74ecc81397abe780649ec89f1a2af18d7022737d0482cefe"),
+                          (ExternalPackageRefCategory.PACKAGE_MANAGER, "purl",
+                           "pkg:bitbucket/birkenfeld/pygments-main@244fd47e07d1014f0aed9c"),
+                          (ExternalPackageRefCategory.PACKAGE_MANAGER, "purl",
+                           "pkg:deb/debian/curl@7.50.3-1?arch=i386&distro=jessie"),
+                          (ExternalPackageRefCategory.PACKAGE_MANAGER, "purl",
+                           "pkg:docker/customer/dockerimage@sha256:244fd47e07d1004f0aed9c?repository_url=gcr.io"),
+                          (ExternalPackageRefCategory.PACKAGE_MANAGER, "purl",
+                           "pkg:gem/jruby-launcher@1.1.2?platform=java"),
+                          (ExternalPackageRefCategory.PACKAGE_MANAGER, "purl", "pkg:gem/ruby-advisory-db-check@0.12.4"),
+                          (ExternalPackageRefCategory.PACKAGE_MANAGER, "purl",
+                           "pkg:github/package-url/purl-spec@244fd47e07d1004f0aed9c"),
+                          (ExternalPackageRefCategory.PACKAGE_MANAGER, "purl",
+                           "pkg:golang/google.golang.org/genproto#googleapis/api/annotations"),
+                          (ExternalPackageRefCategory.PACKAGE_MANAGER, "purl",
+                           "pkg:maven/org.apache.xmlgraphics/batik-anim@1.9.1?repository_url=repo.spring.io%2Frelease"),
+                          (ExternalPackageRefCategory.PACKAGE_MANAGER, "purl", "pkg:npm/%40angular/animation@12.3.1"),
+                          (ExternalPackageRefCategory.PACKAGE_MANAGER, "purl",
+                           "pkg:nuget/EnterpriseLibrary.Common@6.0.1304"),
+                          (ExternalPackageRefCategory.PACKAGE_MANAGER, "purl",
+                           "pkg:rpm/fedora/curl@7.50.3-1.fc25?arch=i386&distro=fedora-25"),
+                          (ExternalPackageRefCategory.PERSISTENT_ID, "swh",
+                           "swh:1:cnt:94a9ed024d3859793618152ea559a168bbcbb5e2"),
+                          (ExternalPackageRefCategory.PERSISTENT_ID, "swh",
+                           "swh:1:dir:d198bc9d7a6bcf6db04f476d29314f157507d505"),
+                          (ExternalPackageRefCategory.PERSISTENT_ID, "swh",
+                           "swh:1:rev:309cf2674ee7a0749978cf8265ab91a60aea0f7d"),
+                          (ExternalPackageRefCategory.PERSISTENT_ID, "swh",
+                           "swh:1:rel:22ece559cc7cc2364edc5e5593d63ae8bd229f9f"),
+                          (ExternalPackageRefCategory.PERSISTENT_ID, "swh",
+                           "swh:1:snp:c7c108084bc0bf3d81436bf980b46e98bd338453"),
+                          (ExternalPackageRefCategory.PERSISTENT_ID, "gitoid",
+                           "gitoid:blob:sha1:261eeb9e9f8b2b4b0d119366dda99c6fd7d35c64"),
+                          (ExternalPackageRefCategory.PERSISTENT_ID, "gitoid",
+                           "gitoid:blob:sha256:3557f7eb43c621c71483743d4b37059bb80933e7f71277c0c3b3846159d1f61c"),
+                          (ExternalPackageRefCategory.OTHER, "some idstring", "#//string-withOUT!Spaces\\?")
+                          ])
+def test_valid_external_package_ref(category, reference_type, locator):
+    external_package_ref = ExternalPackageRef(category, reference_type, locator, "externalPackageRef comment")
     validation_messages: List[ValidationMessage] = validate_external_package_ref(external_package_ref, "parent_id")
 
     assert validation_messages == []
 
 
-@pytest.mark.parametrize("external_package_ref, expected_message",
-                         [(external_package_ref_fixture(),
-                           "TBD"),
+@pytest.mark.parametrize("category, reference_type, locator, expected_message",
+                         [(
+                          ExternalPackageRefCategory.SECURITY, "cpe22Typo", "cpe:/o:canonical:ubuntu_linux:10.04:-:lts",
+                          "externalPackageRef type in category SECURITY must be one of ['cpe22Type', 'cpe23Type', 'advisory', 'fix', 'url', 'swid'], but is: cpe22Typo"),
+                          (ExternalPackageRefCategory.PACKAGE_MANAGER, "nugat",
+                           "cpe:/o:canonical:ubuntu_linux:10.04:-:lts",
+                           "externalPackageRef type in category PACKAGE_MANAGER must be one of ['maven-central', 'npm', 'nuget', 'bower', 'purl'], but is: nugat"),
+                          (ExternalPackageRefCategory.PERSISTENT_ID, "git-oid",
+                           "cpe:/o:canonical:ubuntu_linux:10.04:-:lts",
+                           "externalPackageRef type in category PERSISTENT_ID must be one of ['swh', 'gitoid'], but is: git-oid")
+                          ])
+def test_invalid_external_package_ref_types(category, reference_type, locator, expected_message):
+    external_package_ref = ExternalPackageRef(category, reference_type, locator, "externalPackageRef comment")
+    parent_id = "SPDXRef-Package"
+    validation_messages: List[ValidationMessage] = validate_external_package_ref(external_package_ref, parent_id)
+
+    expected = ValidationMessage(expected_message,
+                                 ValidationContext(parent_id=parent_id,
+                                                   element_type=SpdxElementType.EXTERNAL_PACKAGE_REF,
+                                                   full_element=external_package_ref))
+
+    assert validation_messages == [expected]
+
+
+@pytest.mark.parametrize("category, reference_type, locator, expected_message",
+                         [(ExternalPackageRefCategory.SECURITY, "cpe22Type", "cpe:o:canonical:ubuntu_linux:10.04:-:lts",
+                           f'externalPackageRef locator of type "cpe22Type" must conform with the regex {CPE22TYPE_REGEX}, but is: cpe:o:canonical:ubuntu_linux:10.04:-:lts'),
+                          (ExternalPackageRefCategory.SECURITY, "cpe23Type",
+                           "cpe:2.3:/o:canonical:ubuntu_linux:10.04:-:lts:*:*:*:*:*",
+                           f'externalPackageRef locator of type "cpe23Type" must conform with the regex {CPE23TYPE_REGEX}, but is: cpe:2.3:/o:canonical:ubuntu_linux:10.04:-:lts:*:*:*:*:*'),
+                          (ExternalPackageRefCategory.SECURITY, "advisory", "http://locatorurl",
+                           f'externalPackageRef locator of type "advisory" must be a valid URL, but is: http://locatorurl'),
+                          (ExternalPackageRefCategory.SECURITY, "fix", "http://fixurl",
+                           f'externalPackageRef locator of type "fix" must be a valid URL, but is: http://fixurl'),
+                          (ExternalPackageRefCategory.SECURITY, "url", "http://url",
+                           f'externalPackageRef locator of type "url" must be a valid URL, but is: http://url'),
+                          (ExternalPackageRefCategory.SECURITY, "swid", "2df9de35-0aff-4a86-ace6-f7dddd1ade4c",
+                           f'externalPackageRef locator of type "swid" must be a valid URI with scheme swid, but is: 2df9de35-0aff-4a86-ace6-f7dddd1ade4c'),
+                          (ExternalPackageRefCategory.PACKAGE_MANAGER, "maven-central",
+                           "org.apache.tomcat:tomcat:tomcat:9.0.0.M4",
+                           f'externalPackageRef locator of type "maven-central" must conform with the regex {MAVEN_CENTRAL_REGEX}, but is: org.apache.tomcat:tomcat:tomcat:9.0.0.M4'),
+                          (ExternalPackageRefCategory.PACKAGE_MANAGER, "npm", "http-server:0.3.0",
+                           f'externalPackageRef locator of type "npm" must conform with the regex {NPM_REGEX}, but is: http-server:0.3.0'),
+                          (ExternalPackageRefCategory.PACKAGE_MANAGER, "nuget", "Microsoft.AspNet.MVC@5.0.0",
+                           f'externalPackageRef locator of type "nuget" must conform with the regex {NUGET_REGEX}, but is: Microsoft.AspNet.MVC@5.0.0'),
+                          (ExternalPackageRefCategory.PACKAGE_MANAGER, "bower", "modernizr:2.6.2",
+                           f'externalPackageRef locator of type "bower" must conform with the regex {BOWER_REGEX}, but is: modernizr:2.6.2'),
+                          (ExternalPackageRefCategory.PACKAGE_MANAGER, "purl", "pkg:npm@12.3.1",
+                           f'externalPackageRef locator of type "purl" must conform with the regex {PURL_REGEX}, but is: pkg:npm@12.3.1'),
+                          (ExternalPackageRefCategory.PERSISTENT_ID, "swh",
+                           "swh:cnt:94a9ed024d3859793618152ea559a168bbcbb5e2",
+                           f'externalPackageRef locator of type "swh" must conform with the regex {SWH_REGEX}, but is: swh:cnt:94a9ed024d3859793618152ea559a168bbcbb5e2'),
+                          (ExternalPackageRefCategory.PERSISTENT_ID, "gitoid",
+                           "gitoid:blob:sha1:3557f7eb43c621c71483743d4b37059bb80933e7f71277c0c3b3846159d1f61c",
+                           f'externalPackageRef locator of type "gitoid" must conform with the regex {GITOID_REGEX}, but is: gitoid:blob:sha1:3557f7eb43c621c71483743d4b37059bb80933e7f71277c0c3b3846159d1f61c'),
+                          (ExternalPackageRefCategory.PERSISTENT_ID, "gitoid",
+                           "gitoid:blob:sha256:261eeb9e9f8b2b4b0d119366dda99c6fd7d35c64",
+                           f'externalPackageRef locator of type "gitoid" must conform with the regex {GITOID_REGEX}, but is: gitoid:blob:sha256:261eeb9e9f8b2b4b0d119366dda99c6fd7d35c64'),
+                          (ExternalPackageRefCategory.OTHER, "id string", "locator string",
+                           "externalPackageRef locator in category OTHER must contain no spaces, but is: locator string"),
                           ])
-@pytest.mark.skip(
-    "add tests once external package ref validation is implemented: https://github.com/spdx/tools-python/issues/373")
-def test_invalid_external_package_ref(external_package_ref, expected_message):
+def test_invalid_external_package_ref_locators(category, reference_type, locator, expected_message):
+    external_package_ref = ExternalPackageRef(category, reference_type, locator, "externalPackageRef comment")
     parent_id = "SPDXRef-Package"
     validation_messages: List[ValidationMessage] = validate_external_package_ref(external_package_ref, parent_id)