Repository: pex-tool/pex

Issue:
Failed to resolve compatible distributions when building Pex from .whl with local dependencies
I have a `pyproject.toml` file as below at `~/code/mylibrary/`:

```
[tool.poetry]
name = "mylibrary"
version = "0.1.0"

[tool.poetry.dependencies]
python = ">=3.8,<4"
anotherlibrary = {path = "../anotherlibrary", develop=true}
[tool.poetry.group.dev.dependencies]

[build-system]
requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"
```

I build a .whl for `mylibrary` and here's what its `mylibrary-0.1.0.dist-info/METADATA` file contains:

```
Metadata-Version: 2.1
Name: mylibrary
Version: 0.1.0
Requires-Python: >=3.8,<4
Classifier: Programming Language :: Python :: 3
Classifier: Programming Language :: Python :: 3.8
Classifier: Programming Language :: Python :: 3.9
Classifier: Programming Language :: Python :: 3.10
Classifier: Programming Language :: Python :: 3.11
Requires-Dist: anotherlibrary @ file:///Users/rajaiealkorani/code/anotherlibrary
Description-Content-Type: text/markdown
```

I am trying to build a Pex file by passing that .whl as a dependency, but I am getting the error below:

```
pex --resolver-version pip-2020-resolver dist/mylibrary-0.1.0-py3-none-any.whl -o /tmp/testing.pex 
Failed to resolve compatible distributions:
1: mylibrary==0.1.0 requires anotherlibrary@ file:///Users/rajaiealkorani/code/anotherlibrary but no version was resolved
```

Is this a bug in Pex, or is this kind of dependency specification not supported?

I'm able to run `pip install mylibrary-0.1.0-py3-none-any.whl` in a clean virtual environment as long as `file:///Users/rajaiealkorani/code/anotherlibrary` exists

I am using Pex `2.1.120`


Discussion:
@rajaie-sg can you provide a link to your repository or is it closed source? That would be the easiest path to a repro here.
Ok, I built a repro. This is a bug.

Resolving patch:
diff --git a/pex/requirements.py b/pex/requirements.py
--- a/pex/requirements.py
+++ b/pex/requirements.py
@@ -5,7 +5,6 @@
 
 import os
 import re
-import sys
 from contextlib import contextmanager
 
 from pex import attrs, dist_metadata, pex_warnings
diff --git a/pex/resolver.py b/pex/resolver.py
--- a/pex/resolver.py
+++ b/pex/resolver.py
@@ -16,7 +16,8 @@
 from pex.atomic_directory import AtomicDirectory, atomic_directory
 from pex.auth import PasswordEntry
 from pex.common import safe_mkdir, safe_mkdtemp
-from pex.dist_metadata import Distribution, Requirement
+from pex.compatibility import unquote, urlparse
+from pex.dist_metadata import DistMetadata, Distribution, Requirement
 from pex.fingerprinted_distribution import FingerprintedDistribution
 from pex.jobs import Raise, SpawnedJob, execute_parallel
 from pex.network_configuration import NetworkConfiguration
@@ -45,7 +46,17 @@
 from pex.variables import ENV
 
 if TYPE_CHECKING:
-    from typing import DefaultDict, Iterable, Iterator, List, Mapping, Optional, Sequence, Tuple
+    from typing import (
+        DefaultDict,
+        Iterable,
+        Iterator,
+        List,
+        Mapping,
+        Optional,
+        Sequence,
+        Set,
+        Tuple,
+    )
 
     import attr  # vendor:skip
 
@@ -657,6 +668,54 @@ def _spawn_install(
         )
         return SpawnedJob.wait(job=install_job, result=install_result)
 
+    def _resolve_direct_file_deps(
+        self,
+        install_requests,  # type: Iterable[InstallRequest]
+        max_parallel_jobs=None,  # type: Optional[int]
+        analyzed=None,  # type: Optional[Set[ProjectName]]
+    ):
+        # type: (...) -> Iterable[InstallRequest]
+
+        already_analyzed = analyzed or set()  # type: Set[ProjectName]
+
+        to_install = OrderedSet()  # type: OrderedSet[InstallRequest]
+        to_build = OrderedSet()  # type: OrderedSet[BuildRequest]
+        for install_request in install_requests:
+            metadata = DistMetadata.load(install_request.wheel_path)
+            for requirement in metadata.requires_dists:
+                if requirement.project_name in already_analyzed:
+                    continue
+                if not requirement.url:
+                    continue
+                urlinfo = urlparse.urlparse(requirement.url)
+                if urlinfo.scheme != "file":
+                    continue
+                dist_path = unquote(urlinfo.path).rstrip()
+                if not os.path.exists(dist_path):
+                    raise Unsatisfiable(
+                        "The {wheel} wheel has a dependency on {url} which does not exist on this "
+                        "machine.".format(wheel=install_request.wheel_file, url=requirement.url)
+                    )
+                if dist_path.endswith(".whl"):
+                    to_install.add(InstallRequest.create(install_request.target, dist_path))
+                else:
+                    to_build.add(BuildRequest.create(install_request.target, dist_path))
+            already_analyzed.add(metadata.project_name)
+
+        all_install_requests = OrderedSet(install_requests)
+        if to_build:
+            build_results = self._wheel_builder.build_wheels(
+                build_requests=to_build, max_parallel_jobs=max_parallel_jobs
+            )
+            to_install.update(itertools.chain.from_iterable(build_results.values()))
+        if to_install:
+            all_install_requests.update(
+                self._resolve_direct_file_deps(
+                    to_install, max_parallel_jobs=max_parallel_jobs, analyzed=already_analyzed
+                )
+            )
+        return all_install_requests
+
     def install_distributions(
         self,
         ignore_errors=False,  # type: bool
@@ -680,7 +739,15 @@ def install_distributions(
         )
         to_install.extend(itertools.chain.from_iterable(build_results.values()))
 
-        # 2. All requirements are now in wheel form: calculate any missing direct requirement
+        # 2. (Recursively) post-process all wheels with file:// URL direct references. During the
+        #    download phase, Pip considers these dependencies satisfied and does not download them
+        #    or transfer them to the download directory (although it does download their
+        #    non file:// URL dependencies); it just leaves them where they lay on the file system.
+        all_install_requests = self._resolve_direct_file_deps(
+            to_install, max_parallel_jobs=max_parallel_jobs
+        )
+
+        # 3. All requirements are now in wheel form: calculate any missing direct requirement
         #    project names from the wheel names.
         with TRACER.timed(
             "Calculating project names for direct requirements:"
@@ -724,14 +791,14 @@ def iter_direct_requirements():
                     direct_requirement
                 )
 
-        # 3. Install wheels in individual chroots.
+        # 4. Install wheels in individual chroots.
 
         # Dedup by wheel name; e.g.: only install universal wheels once even though they'll get
         # downloaded / built for each interpreter or platform.
         install_requests_by_wheel_file = (
             OrderedDict()
         )  # type: OrderedDict[str, List[InstallRequest]]
-        for install_request in to_install:
+        for install_request in all_install_requests:
             install_requests_by_wheel_file.setdefault(install_request.wheel_file, []).append(
                 install_request
             )


Test patch:
diff --git a/tests/integration/test_issue_2038.py b/tests/integration/test_issue_2038.py
new file mode 100644
--- /dev/null
+++ b/tests/integration/test_issue_2038.py
@@ -0,0 +1,104 @@
+# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import print_function
+
+import glob
+import os.path
+import subprocess
+import sys
+from textwrap import dedent
+
+import pytest
+
+from pex.common import touch
+from pex.testing import run_pex_command
+from pex.typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from typing import Any
+
+
+@pytest.mark.skipif(
+    sys.version_info[:2] < (3, 7),
+    reason="This test needs to run Poetry which requires at least Python 3.7",
+)
+def test_wheel_file_url_dep(tmpdir):
+    # type: (Any) -> None
+
+    poetry = os.path.join(str(tmpdir), "poetry.pex")
+    run_pex_command(args=["poetry==1.3.2", "-c", "poetry", "-o", poetry]).assert_success()
+
+    corelibrary = os.path.join(str(tmpdir), "corelibrary")
+    touch(os.path.join(corelibrary, "README.md"))
+    with open(os.path.join(corelibrary, "corelibrary.py"), "w") as fp:
+        print("TWO = 2", file=fp)
+    subprocess.check_call(args=[poetry, "init", "--no-interaction"], cwd=corelibrary)
+
+    anotherlibrary = os.path.join(str(tmpdir), "anotherlibrary")
+    touch(os.path.join(anotherlibrary, "README.md"))
+    with open(os.path.join(anotherlibrary, "anotherlibrary.py"), "w") as fp:
+        fp.write(
+            dedent(
+                """\
+                from corelibrary import TWO
+
+
+                MEANING_OF_LIFE = TWO * 21
+                """
+            )
+        )
+    subprocess.check_call(
+        args=[poetry, "init", "--no-interaction", "--dependency", "../corelibrary"],
+        cwd=anotherlibrary,
+    )
+
+    mylibrary = os.path.join(str(tmpdir), "mylibrary")
+    touch(os.path.join(mylibrary, "README.md"))
+    with open(os.path.join(mylibrary, "mylibrary.py"), "w") as fp:
+        fp.write(
+            dedent(
+                """\
+                from anotherlibrary import MEANING_OF_LIFE
+
+
+                def deep_thought():
+                    print(f"MEANING_OF_LIFE = {MEANING_OF_LIFE}")
+                """
+            )
+        )
+    subprocess.check_call(
+        args=[poetry, "init", "--no-interaction", "--dependency", "../anotherlibrary"],
+        cwd=mylibrary,
+    )
+
+    subprocess.check_call(args=[poetry, "build", "-f", "wheel"], cwd=mylibrary)
+    wheels = glob.glob(os.path.join(mylibrary, "dist", "*.whl"))
+    assert len(wheels) == 1
+    wheel = wheels[0]
+
+    pex_root = os.path.join(str(tmpdir), "pex_root")
+    testing_pex = os.path.join(str(tmpdir), "testing.pex")
+    run_pex_command(
+        args=[
+            "--pex-root",
+            pex_root,
+            "--runtime-pex-root",
+            pex_root,
+            "--resolver-version",
+            "pip-2020-resolver",
+            # N.B.: Modern Pip is needed to handle Poetry relative path deps. Older Pip does
+            # building off in a tmp dir and that breaks relative path references.
+            "--pip-version",
+            "22.3",
+            wheel,
+            "-e",
+            "mylibrary:deep_thought",
+            "-o",
+            testing_pex,
+        ]
+    ).assert_success()
+    assert (
+        "MEANING_OF_LIFE = 42"
+        == subprocess.check_output(args=[testing_pex]).decode("utf-8").strip()
+    )