Repository: pantsbuild/pants

Issue:
[jvm] erroneous BouncyCastle transitive dependencies resolved
**Describe the bug**
When trying to resolve dependencies of the Bouncy Castle jars (using `./pants coursier-resolve ...`), transitivie dependencies are rendered into the lock file with the `version` and `packaging` fields using swapped values.

Let's consider a `BUILD` file like the following:

```
GROUP_ID="org.bouncycastle"
VERSION="1.70"

jvm_artifact(
    name="bcprov",
    group=GROUP_ID,
    artifact="bcprov-jdk15on",
    version=VERSION,
    packages=[
        'org.bouncycastle.*',
        'org.bouncycastle.asn1.*',
        'org.bouncycastle.asn1.anssi.**',
        'org.bouncycastle.asn1.bc.**',
        'org.bouncycastle.asn1.cryptopro.**',
        'org.bouncycastle.asn1.edec.**',
        'org.bouncycastle.asn1.gm.**',
        'org.bouncycastle.asn1.gnu.**',
        'org.bouncycastle.asn1.iana.**',
        'org.bouncycastle.asn1.isara.**',
        'org.bouncycastle.asn1.iso.**',
        'org.bouncycastle.asn1.kisa.**',
        'org.bouncycastle.asn1.microsoft.**',
        'org.bouncycastle.asn1.misc.**',
        'org.bouncycastle.asn1.mozilla.**',
        'org.bouncycastle.asn1.nist.**',
        'org.bouncycastle.asn1.nsri.**',
        'org.bouncycastle.asn1.ntt.**',
        'org.bouncycastle.asn1.ocsp.**',
        'org.bouncycastle.asn1.oiw.**',
        'org.bouncycastle.asn1.pkcs.**',
        'org.bouncycastle.asn1.rosstandart.**',
        'org.bouncycastle.asn1.sec.**',
        'org.bouncycastle.asn1.teletrust.**',
        'org.bouncycastle.asn1.ua.**',
        'org.bouncycastle.asn1.util.**',
        'org.bouncycastle.asn1.x500.**',
        'org.bouncycastle.asn1.x509.**',
        'org.bouncycastle.asn1.x9.**',
        'org.bouncycastle.crypto.**',
        'org.bouncycastle.i18n.**',
        'org.bouncycastle.iana.**',
        'org.bouncycastle.jcajce.**',
        'org.bouncycastle.jce.**',
        'org.bouncycastle.math.**',
        'org.bouncycastle.pqc.**',
        'org.bouncycastle.util.**',
        'org.bouncycastle.x509.**',
    ],
    compatible_resolves=["jvm_default"],
)

jvm_artifact(
    name="bcutil",
    group=GROUP_ID,
    artifact="bcutil-jdk15on",
    version=VERSION,
    packages=[
        'org.bouncycastle.asn1.bsi.**',
        'org.bouncycastle.asn1.cmc.**',
        'org.bouncycastle.asn1.cmp.**',
        'org.bouncycastle.asn1.cms.**',
        'org.bouncycastle.asn1.cms.ecc.**',
        'org.bouncycastle.asn1.crmf.**',
        'org.bouncycastle.asn1.dvcs.**',
        'org.bouncycastle.asn1.eac.**',
        'org.bouncycastle.asn1.esf.**',
        'org.bouncycastle.asn1.ess.**',
        'org.bouncycastle.asn1.est.**',
        'org.bouncycastle.asn1.icao.**',
        'org.bouncycastle.asn1.isismtt.**',
        'org.bouncycastle.asn1.smime.**',
        'org.bouncycastle.asn1.tsp.**',
        'org.bouncycastle.oer.**',
    ],
    compatible_resolves=["jvm_default"],
)
```

Then, run `./pants coursier-resolve --names=jvm_default`. Check the lockfile generated and look for the `bcprov-jdk15on` artifact. It would show something like:

```
[
    {
        "coord": {
            "group": "org.bouncycastle",
            "artifact": "bcprov-jdk15on",
            "version": "1.70",
            "packaging": "jar"
        },
        "directDependencies": [],
        "dependencies": [],
        "file_name": "org.bouncycastle_bcprov-jdk15on_1.70.jar",
        "file_digest": {
            "fingerprint": "8f3c20e3e2d565d26f33e8d4857a37d0d7f8ac39b62a7026496fcab1bdac30d4",
            "serialized_bytes_length": 5867298
        },
        "remote_url": null,
        "pants_address": null
    },
    {
        "coord": {
            "group": "org.bouncycastle",
            "artifact": "bcprov-jdk15on",
            "version": "jar",
            "packaging": "1.70"
        },
        "directDependencies": [],
        "dependencies": [],
        "file_name": "org.bouncycastle_bcprov-jdk15on_jar_1.70.jar",
        "file_digest": {
            "fingerprint": "8f3c20e3e2d565d26f33e8d4857a37d0d7f8ac39b62a7026496fcab1bdac30d4",
            "serialized_bytes_length": 5867298
        },
        "remote_url": null,
        "pants_address": null
    },
    {
        "coord": {
            "group": "org.bouncycastle",
            "artifact": "bcutil-jdk15on",
            "version": "1.70",
            "packaging": "jar"
        },
        "directDependencies": [
            {
                "group": "org.bouncycastle",
                "artifact": "bcprov-jdk15on",
                "version": "jar",
                "packaging": "1.70"
            }
        ],
        "dependencies": [
            {
                "group": "org.bouncycastle",
                "artifact": "bcprov-jdk15on",
                "version": "jar",
                "packaging": "1.70"
            }
        ],
        "file_name": "org.bouncycastle_bcutil-jdk15on_1.70.jar",
        "file_digest": {
            "fingerprint": "52dc5551b0257666526c5095424567fed7dc7b00d2b1ba7bd52298411112b1d0",
            "serialized_bytes_length": 482530
        },
        "remote_url": null,
        "pants_address": null
    },
    ...
]
```

First thing I noticed is that the item for `bcprov-...` is duplicated, then the fact that in the second occurrence, it does have the values for the `version` and `packaging` fields swapped. This also happens in the transitive dependencies of `bcutil-...` (and also the `bcutil` item in the lockfile is duplicated with the swapped values in the second instance, just showing here only one for brevity):

```
"version": "jar",
"packaging": "1.70"
```

**Pants version**
2.9.0rc1

**OS**
MacOS

**Additional info**
I'm evaluating Pants 2 with a middle-sized repository with a fair amount of 3rd party JVM dependencies and Bouncy Castle is the only one in which I observed this behavior. Not sure if this is a Coursier error, the code of the `Coordinates` parsing looks correct and the public POM files for the Bouncy Castle seem correct to me.



Discussion:
Seems the issue is in coursier itself, not pants.

To reproduce: run coursier from the command line:

```bash
cs fetch --json-output-file=deps.json "org.bouncycastle:bcutil-jdk15on:1.70"
```

Then cat the generated file and observe the following:

```json
{
  "conflict_resolution": {},
  "dependencies": [
    {
      "coord": "org.bouncycastle:bcprov-jdk15on:jar:1.70",
      "file": "/Users/alonsodomin/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/bouncycastle/bcprov-jdk15on/1.70/bcprov-jdk15on-1.70.jar",
      "directDependencies": [],
      "dependencies": []
    },
    {
      "coord": "org.bouncycastle:bcutil-jdk15on:1.70",
      "file": "/Users/alonsodomin/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/bouncycastle/bcutil-jdk15on/1.70/bcutil-jdk15on-1.70.jar",
      "directDependencies": [
        "org.bouncycastle:bcprov-jdk15on:jar:1.70"
      ],
      "dependencies": [
        "org.bouncycastle:bcprov-jdk15on:jar:1.70"
      ]
    }
  ],
  "version": "0.1.0"
}
```

As you can see, the coordinates of the dependencies have swapped the version and packaging parts.

Will create a ticket in courser for this.
@alonsodomin : Thanks for the bug report: will post a patch to work around this shortly.

It's not clear to me: does this cause the build to fail for you, or did you just notice it in the lockfile?
It does break the build too, as when running `./pants check` it attempts to fetch the artifacts based on the lockfile.

Because the surface is very limited, I can fix the lockfile by hand and move on. But a newer call to `./pants coursier-resolve` will overwrite my manual changes, making me having to fix it again.

I’ve been a bit snowed today otherwise wouldn’t mind submitting having a stab at it in courtier’s codebase.

On lun, dic 27, 2021 a las 19:12, Stu Hood ***@***.***> escribió:

> ***@***.***(https://github.com/alonsodomin) : Thanks for the bug report: will post a patch to work around this shortly.
>
> It's not clear to me: does this cause the build to fail for you, or did you just notice it in the lockfile?
>
> —
> Reply to this email directly, [view it on GitHub](https://github.com/pantsbuild/pants/issues/13986#issuecomment-1001685352), or [unsubscribe](https://github.com/notifications/unsubscribe-auth/AARKCADI7ZQ5CJXF47CJFG3UTCUAXANCNFSM5KZF356Q).
> You are receiving this because you were mentioned.Message ID: ***@***.***>
> It does break the build too, as when running `./pants check` it attempts to fetch the artifacts based on the lockfile.

Ah... because when it converts from the coordinate back to an input to coursier, it will produce the wrong thing. Got it.

Ok, then #13996 should resolve this. I'll get it cherry-picked for `2.9.0rc2` as well. Thanks!

Resolving patch:
diff --git a/src/python/pants/jvm/resolve/coursier_fetch.py b/src/python/pants/jvm/resolve/coursier_fetch.py
--- a/src/python/pants/jvm/resolve/coursier_fetch.py
+++ b/src/python/pants/jvm/resolve/coursier_fetch.py
@@ -115,13 +115,33 @@ def to_json_dict(self) -> dict:
         return ret
 
     @classmethod
-    def from_coord_str(cls, s: str) -> Coordinate:
+    def from_coord_str(cls, s: str, *, with_2315_workaround=False) -> Coordinate:
+        """Parses from a coordinate string with an optional `packaging` string.
+
+        ${organisation}:${artifact}:${version}[:${packaging}]
+
+        NB: The `with_2315_workaround` flag is for parsing in locations that are affected by
+          https://github.com/coursier/coursier/issues/2315
+        ... which swaps the `version` and `packaging`.
+        """
         parts = s.split(":")
+
+        if len(parts) == 4:
+            if with_2315_workaround:
+                version = parts[3]
+                packaging = parts[2]
+            else:
+                version = parts[2]
+                packaging = parts[3]
+        else:
+            version = parts[2]
+            packaging = "jar"
+
         return cls(
             group=parts[0],
             artifact=parts[1],
-            version=parts[2],
-            packaging=parts[3] if len(parts) == 4 else "jar",
+            version=version,
+            packaging=packaging,
         )
 
     def as_requirement(self) -> ArtifactRequirement:
@@ -477,14 +497,20 @@ async def coursier_resolve_lockfile(
         )
     )
 
+    # NB: Parsing of the resolve JSON applies the workaround for
+    #   https://github.com/coursier/coursier/issues/2315: see `Coordinate.from_coord_str`.
     first_pass_lockfile = CoursierResolvedLockfile(
         entries=tuple(
             CoursierLockfileEntry(
-                coord=Coordinate.from_coord_str(dep["coord"]),
+                coord=Coordinate.from_coord_str(dep["coord"], with_2315_workaround=True),
                 direct_dependencies=Coordinates(
-                    Coordinate.from_coord_str(dd) for dd in dep["directDependencies"]
+                    Coordinate.from_coord_str(dd, with_2315_workaround=True)
+                    for dd in dep["directDependencies"]
+                ),
+                dependencies=Coordinates(
+                    Coordinate.from_coord_str(d, with_2315_workaround=True)
+                    for d in dep["dependencies"]
                 ),
-                dependencies=Coordinates(Coordinate.from_coord_str(d) for d in dep["dependencies"]),
                 file_name=file_name,
                 file_digest=artifact_file_digest,
             )
@@ -611,7 +637,9 @@ async def coursier_fetch_one_coord(
 
     dep = report_deps[0]
 
-    resolved_coord = Coordinate.from_coord_str(dep["coord"])
+    # NB: Parsing of the resolve JSON applies the workaround for
+    #   https://github.com/coursier/coursier/issues/2315: see `Coordinate.from_coord_str`.
+    resolved_coord = Coordinate.from_coord_str(dep["coord"], with_2315_workaround=True)
     if resolved_coord != request.coord:
         raise CoursierError(
             f'Coursier resolved coord "{resolved_coord.to_coord_str()}" does not match requested coord "{request.coord.to_coord_str()}".'


Test patch:
diff --git a/src/python/pants/jvm/resolve/coursier_fetch_integration_test.py b/src/python/pants/jvm/resolve/coursier_fetch_integration_test.py
--- a/src/python/pants/jvm/resolve/coursier_fetch_integration_test.py
+++ b/src/python/pants/jvm/resolve/coursier_fetch_integration_test.py
@@ -174,6 +174,80 @@ def test_resolve_conflicting(rule_runner: RuleRunner) -> None:
         )
 
 
+@maybe_skip_jdk_test
+def test_resolve_with_packaging(rule_runner: RuleRunner) -> None:
+    # Tests that an artifact pom which actually reports packaging ends up with proper version and
+    # packaging information.
+    #   see https://github.com/pantsbuild/pants/issues/13986
+    resolved_lockfile = rule_runner.request(
+        CoursierResolvedLockfile,
+        [
+            ArtifactRequirements.from_coordinates(
+                [Coordinate(group="org.bouncycastle", artifact="bcutil-jdk15on", version="1.70")]
+            ),
+        ],
+    )
+
+    assert resolved_lockfile == CoursierResolvedLockfile(
+        entries=(
+            CoursierLockfileEntry(
+                coord=Coordinate(
+                    group="org.bouncycastle",
+                    artifact="bcprov-jdk15on",
+                    version="1.70",
+                    packaging="jar",
+                    strict=True,
+                ),
+                file_name="org.bouncycastle_bcprov-jdk15on_jar_1.70.jar",
+                direct_dependencies=Coordinates([]),
+                dependencies=Coordinates([]),
+                file_digest=FileDigest(
+                    "8f3c20e3e2d565d26f33e8d4857a37d0d7f8ac39b62a7026496fcab1bdac30d4", 5867298
+                ),
+                remote_url=None,
+                pants_address=None,
+            ),
+            CoursierLockfileEntry(
+                coord=Coordinate(
+                    group="org.bouncycastle",
+                    artifact="bcutil-jdk15on",
+                    version="1.70",
+                    packaging="jar",
+                    strict=True,
+                ),
+                file_name="org.bouncycastle_bcutil-jdk15on_1.70.jar",
+                direct_dependencies=Coordinates(
+                    [
+                        Coordinate(
+                            group="org.bouncycastle",
+                            artifact="bcprov-jdk15on",
+                            version="1.70",
+                            packaging="jar",
+                            strict=True,
+                        )
+                    ]
+                ),
+                dependencies=Coordinates(
+                    [
+                        Coordinate(
+                            group="org.bouncycastle",
+                            artifact="bcprov-jdk15on",
+                            version="1.70",
+                            packaging="jar",
+                            strict=True,
+                        )
+                    ]
+                ),
+                file_digest=FileDigest(
+                    "52dc5551b0257666526c5095424567fed7dc7b00d2b1ba7bd52298411112b1d0", 482530
+                ),
+                remote_url=None,
+                pants_address=None,
+            ),
+        )
+    )
+
+
 @maybe_skip_jdk_test
 def test_resolve_with_broken_url(rule_runner: RuleRunner) -> None:
 
diff --git a/src/python/pants/jvm/resolve/coursier_fetch_test.py b/src/python/pants/jvm/resolve/coursier_fetch_test.py
--- a/src/python/pants/jvm/resolve/coursier_fetch_test.py
+++ b/src/python/pants/jvm/resolve/coursier_fetch_test.py
@@ -12,7 +12,7 @@
 from pants.backend.java.target_types import rules as target_types_rules
 from pants.core.util_rules import config_files, source_files
 from pants.engine.addresses import Address, Addresses
-from pants.jvm.resolve.coursier_fetch import NoCompatibleResolve
+from pants.jvm.resolve.coursier_fetch import Coordinate, NoCompatibleResolve
 from pants.jvm.resolve.coursier_fetch import rules as coursier_fetch_rules
 from pants.jvm.resolve.key import CoursierResolveKey
 from pants.jvm.target_types import JvmArtifactTarget
@@ -111,3 +111,28 @@ def test_no_matching_for_root(rule_runner: RuleRunner) -> None:
 def test_no_matching_for_leaf(rule_runner: RuleRunner) -> None:
     with engine_error(NoCompatibleResolve):
         assert_resolve("n/a", rule_runner, "one", "one", ["two"])
+
+
+@pytest.mark.parametrize(
+    "coord_str,with_2315_workaround,expected",
+    (
+        *(
+            ("group:artifact:version", b, Coordinate("group", "artifact", "version"))
+            for b in [True, False]
+        ),
+        (
+            "group:artifact:packaging:version",
+            True,
+            Coordinate("group", "artifact", "version", "packaging"),
+        ),
+        (
+            "group:artifact:version:packaging",
+            False,
+            Coordinate("group", "artifact", "version", "packaging"),
+        ),
+    ),
+)
+def test_from_coord_str(coord_str: str, with_2315_workaround: bool, expected: Coordinate) -> None:
+    assert (
+        Coordinate.from_coord_str(coord_str, with_2315_workaround=with_2315_workaround) == expected
+    )