Repository: pex-tool/pex

Issue:
Pex fails to subset a "foo @ file:///bar" URL lock.
This can be seen with this diff to the test added in #2060:
```diff
diff --git a/tests/integration/cli/commands/test_issue_2057.py b/tests/integration/cli/commands/test_issue_2057.py
index a2b0e52..bb875f9 100644
--- a/tests/integration/cli/commands/test_issue_2057.py
+++ b/tests/integration/cli/commands/test_issue_2057.py
@@ -119,6 +119,7 @@ def test_lock_create_local_project_direct_reference(tmpdir):
                 pex_root,
                 "--lock",
                 lock,
+                "ansicolors @ file://{}".format(clone_dir),
                 "--",
                 "-c",
                 "import colors; print(colors.yellow('Vogon Constructor Fleet!'))",
```

Which takes this test run from green to red:
```console
$ tox -epy311-integration -- -sk test_lock_create_local_project_direct_reference -n0 -q
py311-integration inst-nodeps: /home/jsirois/dev/pantsbuild/jsirois-pex/.tox/.tmp/package/1/pex-2.1.126.tar.gz
py311-integration installed: ansicolors==1.1.8,attrs==22.2.0,execnet==1.9.0,iniconfig==2.0.0,packaging==23.0,pex @ file:///home/jsirois/dev/pantsbuild/jsirois-pex/.tox/.tmp/package/1/pex-2.1.126.tar.gz,pkginfo==1.7.0,pluggy==1.0.0,py==1.11.0,pytest==6.2.5,pytest-forked==1.6.0,pytest-xdist==1.34.0,six==1.16.0,toml==0.10.2
py311-integration run-test-pre: PYTHONHASHSEED='2718309373'
py311-integration run-test: commands[0] | python scripts/print_env.py
Test Control Environment Variables:
py311-integration run-test: commands[1] | pytest -n auto tests/integration -sk test_lock_create_local_project_direct_reference -n0 -q
<frozen importlib._bootstrap>:1049: ImportWarning: VendorImporter.find_spec() not found; falling back to find_module()
hint: Using 'master' as the name for the initial branch. This default branch name
hint: is subject to change. To configure the initial branch name to use in all
hint: of your new repositories, which will suppress this warning, call:
hint:
hint:   git config --global init.defaultBranch <name>
hint:
hint: Names commonly chosen instead of 'master' are 'main', 'trunk' and
hint: 'development'. The just-created branch can be renamed via this command:
hint:
hint:   git branch -m <name>
Initialized empty Git repository in /tmp/pytest-of-jsirois/pytest-11/test_lock_create_local_project0/ansicolors/.git/
remote: Enumerating objects: 22, done.
remote: Counting objects: 100% (22/22), done.
remote: Compressing objects: 100% (17/17), done.
remote: Total 22 (delta 2), reused 12 (delta 1), pack-reused 0
Unpacking objects: 100% (22/22), 13.16 KiB | 2.19 MiB/s, done.
From https://github.com/jonathaneunice/colors
 * branch            c965f5b9103c5bd32a1572adb8024ebe83278fb0 -> FETCH_HEAD
HEAD is now at c965f5b corrected patch nums in changelog
F
==================================================================================================== FAILURES ====================================================================================================
________________________________________________________________________________ test_lock_create_local_project_direct_reference _________________________________________________________________________________

tmpdir = local('/tmp/pytest-of-jsirois/pytest-11/test_lock_create_local_project0')

    def test_lock_create_local_project_direct_reference(tmpdir):
        # type: (Any) -> None

        clone_dir = os.path.join(str(tmpdir), "ansicolors")
        subprocess.check_call(args=["git", "init", clone_dir])

        ansicolors_1_1_8_sha = "c965f5b9103c5bd32a1572adb8024ebe83278fb0"
        subprocess.check_call(
            args=[
                "git",
                "fetch",
                "--depth",
                "1",
                "https://github.com/jonathaneunice/colors",
                ansicolors_1_1_8_sha,
            ],
            cwd=clone_dir,
        )
        subprocess.check_call(args=["git", "reset", "--hard", ansicolors_1_1_8_sha], cwd=clone_dir)

        pex_root = os.path.join(str(tmpdir), "pex_root")
        lock = os.path.join(str(tmpdir), "lock.json")
        run_pex3(
            "lock",
            "create",
            "--pex-root",
            pex_root,
            "ansicolors @ file://{}".format(clone_dir),
            "--indent",
            "2",
            "-o",
            lock,
        ).assert_success()

        def assert_create_and_run_pex_from_lock():
            # type: () -> None
            result = run_pex_command(
                args=[
                    "--pex-root",
                    pex_root,
                    "--runtime-pex-root",
                    pex_root,
                    "--lock",
                    lock,
                    "ansicolors @ file://{}".format(clone_dir),
                    "--",
                    "-c",
                    "import colors; print(colors.yellow('Vogon Constructor Fleet!'))",
                ]
            )
            result.assert_success()
            assert colors.yellow("Vogon Constructor Fleet!") == result.output.strip()

>       assert_create_and_run_pex_from_lock()

tests/integration/cli/commands/test_issue_2057.py:131:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
tests/integration/cli/commands/test_issue_2057.py:128: in assert_create_and_run_pex_from_lock
    result.assert_success()
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = IntegResults(output='', error='pex: Parsing lock /tmp/pytest-of-jsirois/pytest-11/test_lock_create_local_project0/lock...oject0/ansicolors\'\n\'/tmp/pytest-of-jsirois/pytest-11/test_lock_create_local_project0/ansicolors\'\n', return_code=1)

    def assert_success(self):
        # type: () -> None
>       assert (
            self.return_code == 0
        ), "integration test failed: return_code={}, output={}, error={}".format(
            self.return_code, self.output, self.error
        )
E       AssertionError: integration test failed: return_code=1, output=, error=pex: Parsing lock /tmp/pytest-of-jsirois/pytest-11/test_lock_create_local_project0/lock.json
E       pex: Parsing lock /tmp/pytest-of-jsirois/pytest-11/test_lock_create_local_project0/lock.json: 0.8ms
E       pex: Building pex
E       pex: Building pex :: Resolving distributions (ansicolors @ file:///tmp/pytest-of-jsirois/pytest-11/test_lock_create_local_project0/ansicolors)
E       pex: Building pex :: Resolving distributions (ansicolors @ file:///tmp/pytest-of-jsirois/pytest-11/test_lock_create_local_project0/ansicolors) :: Resolving requirements from lock file /tmp/pytest-of-jsirois/pytest-11/test_lock_create_local_project0/lock.json
E       pex: Building pex :: Resolving distributions (ansicolors @ file:///tmp/pytest-of-jsirois/pytest-11/test_lock_create_local_project0/ansicolors) :: Resolving requirements from lock file /tmp/pytest-of-jsirois/pytest-11/test_lock_create_local_project0/lock.json :: Parsing requirements
E       Traceback (most recent call last):
E         File "/home/jsirois/dev/pantsbuild/jsirois-pex/pex/result.py", line 105, in catch
E           return func(*args, **kwargs)
E                  ^^^^^^^^^^^^^^^^^^^^^
E         File "/home/jsirois/dev/pantsbuild/jsirois-pex/pex/bin/pex.py", line 856, in do_main
E           pex_builder = build_pex(
E                         ^^^^^^^^^^
E         File "/home/jsirois/dev/pantsbuild/jsirois-pex/pex/bin/pex.py", line 689, in build_pex
E           resolve_from_lock(
E         File "/home/jsirois/dev/pantsbuild/jsirois-pex/pex/resolve/lock_resolver.py", line 244, in resolve_from_lock
E           subset(
E         File "/home/jsirois/dev/pantsbuild/jsirois-pex/pex/resolve/lockfile/subset.py", line 66, in subset
E           requirements_to_resolve = OrderedSet(
E                                     ^^^^^^^^^^^
E         File "/home/jsirois/dev/pantsbuild/jsirois-pex/pex/orderedset.py", line 29, in __init__
E           self.update(iterable)
E         File "/home/jsirois/dev/pantsbuild/jsirois-pex/pex/orderedset.py", line 45, in update
E           for key in iterable:
E         File "/home/jsirois/dev/pantsbuild/jsirois-pex/pex/resolve/lockfile/subset.py", line 67, in <genexpr>
E           lock.local_project_requirement_mapping[os.path.abspath(parsed_requirement.path)]
E           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
E       KeyError: '/tmp/pytest-of-jsirois/pytest-11/test_lock_create_local_project0/ansicolors'
E       '/tmp/pytest-of-jsirois/pytest-11/test_lock_create_local_project0/ansicolors'

.tox/py311-integration/lib/python3.11/site-packages/pex/testing.py:357: AssertionError
================================================================================================ warnings summary ================================================================================================
.tox/py311-integration/lib/python3.11/site-packages/pex/vendor/_vendored/packaging_21_3/pyparsing.py:108
  /home/jsirois/dev/pantsbuild/jsirois-pex/.tox/py311-integration/lib/python3.11/site-packages/pex/vendor/_vendored/packaging_21_3/pyparsing.py:108: DeprecationWarning: module 'sre_constants' is deprecated
    import sre_constants

-- Docs: https://docs.pytest.org/en/stable/warnings.html
============================================================================================ short test summary info =============================================================================================
FAILED tests/integration/cli/commands/test_issue_2057.py::test_lock_create_local_project_direct_reference - AssertionError: integration test failed: return_code=1, output=, error=pex: Parsing lock /tmp/pytes...
1 failed, 460 deselected, 1 warning in 4.86s
ERROR: InvocationError for command /home/jsirois/dev/pantsbuild/jsirois-pex/.tox/py311-integration/bin/pytest -n auto tests/integration -sk test_lock_create_local_project_direct_reference -n0 -q (exited with code 1)
____________________________________________________________________________________________________ summary _____________________________________________________________________________________________________
ERROR:   py311-integration: commands failed
```


Discussion:
Perhaps easier to test with to get this working:
```
$ pex3 lock create "pex @ file://$PWD" -o lock.json --indent 2

# All good.
$ pex --lock lock.json -o pex.pex

# But subset fails:
$ pex -v --lock lock.json "pex @ file://$PWD" -o pex.pex
pex: Parsing lock lock.json: 1.3ms
Traceback (most recent call last):tributions (pex @ file:///home/jsirois/dev/pantsbuild/jsirois-pex) :: Resolving requirements from lock file lock.json :: Parsing requirements
  File "/home/jsirois/bin/pex.venv/lib/python3.10/site-packages/pex/result.py", line 105, in catch
    return func(*args, **kwargs)
  File "/home/jsirois/bin/pex.venv/lib/python3.10/site-packages/pex/bin/pex.py", line 856, in do_main
    pex_builder = build_pex(
  File "/home/jsirois/bin/pex.venv/lib/python3.10/site-packages/pex/bin/pex.py", line 689, in build_pex
    resolve_from_lock(
  File "/home/jsirois/bin/pex.venv/lib/python3.10/site-packages/pex/resolve/lock_resolver.py", line 244, in resolve_from_lock
    subset(
  File "/home/jsirois/bin/pex.venv/lib/python3.10/site-packages/pex/resolve/lockfile/subset.py", line 66, in subset
    requirements_to_resolve = OrderedSet(
  File "/home/jsirois/bin/pex.venv/lib/python3.10/site-packages/pex/orderedset.py", line 29, in __init__
    self.update(iterable)
  File "/home/jsirois/bin/pex.venv/lib/python3.10/site-packages/pex/orderedset.py", line 45, in update
    for key in iterable:
  File "/home/jsirois/bin/pex.venv/lib/python3.10/site-packages/pex/resolve/lockfile/subset.py", line 67, in <genexpr>
    lock.local_project_requirement_mapping[os.path.abspath(parsed_requirement.path)]
KeyError: '/home/jsirois/dev/pantsbuild/jsirois-pex'
'/home/jsirois/dev/pantsbuild/jsirois-pex'
```


Resolving patch:
diff --git a/pex/resolve/lock_resolver.py b/pex/resolve/lock_resolver.py
--- a/pex/resolve/lock_resolver.py
+++ b/pex/resolve/lock_resolver.py
@@ -426,5 +426,10 @@ def resolve_from_lock(
             # `LockedResolve.resolve` above and need not waste time (~O(100ms)) doing this again.
             ignore_errors=True,
             max_parallel_jobs=max_parallel_jobs,
+            local_project_directory_to_sdist={
+                downloadable_artifact.artifact.directory: downloaded_artifact.path
+                for downloadable_artifact, downloaded_artifact in downloaded_artifacts.items()
+                if isinstance(downloadable_artifact.artifact, LocalProjectArtifact)
+            },
         )
     return Installed(installed_distributions=tuple(installed_distributions))
diff --git a/pex/resolve/lockfile/model.py b/pex/resolve/lockfile/model.py
--- a/pex/resolve/lockfile/model.py
+++ b/pex/resolve/lockfile/model.py
@@ -10,6 +10,7 @@
 from pex.pip.version import PipVersionValue
 from pex.requirements import LocalProjectRequirement
 from pex.resolve.locked_resolve import LocalProjectArtifact, LockedResolve, LockStyle, TargetSystem
+from pex.resolve.resolved_requirement import Pin
 from pex.resolve.resolver_configuration import ResolverVersion
 from pex.sorted_tuple import SortedTuple
 from pex.typing import TYPE_CHECKING
@@ -49,13 +50,17 @@ def create(
     ):
         # type: (...) -> Lockfile
 
-        pin_by_local_project_directory = {
-            locked_requirement.artifact.directory: locked_requirement.pin
-            for locked_resolve in locked_resolves
-            for locked_requirement in locked_resolve.locked_requirements
-            if isinstance(locked_requirement.artifact, LocalProjectArtifact)
-        }
+        pin_by_local_project_directory = {}  # type: Dict[str, Pin]
         requirement_by_local_project_directory = {}  # type: Dict[str, Requirement]
+        for locked_resolve in locked_resolves:
+            for locked_requirement in locked_resolve.locked_requirements:
+                if isinstance(locked_requirement.artifact, LocalProjectArtifact):
+                    local_directory = locked_requirement.artifact.directory
+                    local_pin = locked_requirement.pin
+                    pin_by_local_project_directory[local_directory] = local_pin
+                    requirement_by_local_project_directory[
+                        local_directory
+                    ] = local_pin.as_requirement()
 
         def extract_requirement(req):
             # type: (Union[Requirement, ParsedRequirement]) -> Requirement
@@ -74,6 +79,9 @@ def extract_requirement(req):
                         marker="; {marker}".format(marker=req.marker) if req.marker else "",
                     )
                 )
+                # N.B.: We've already mapped all available local projects above, but the user may
+                # have supplied the local project requirement with more specific constraints (
+                # extras and / or marker restrictions) and we need to honor those; so we over-write.
                 requirement_by_local_project_directory[local_project_directory] = requirement
                 return requirement
             return req.requirement
diff --git a/pex/resolver.py b/pex/resolver.py
--- a/pex/resolver.py
+++ b/pex/resolver.py
@@ -741,6 +741,7 @@ def install_distributions(
         self,
         ignore_errors=False,  # type: bool
         max_parallel_jobs=None,  # type: Optional[int]
+        local_project_directory_to_sdist=None,  # type: Optional[Mapping[str, str]]
     ):
         # type: (...) -> Iterable[InstalledDistribution]
         if not any((self._build_requests, self._install_requests)):
@@ -783,6 +784,12 @@ def iter_direct_requirements():
                         continue
 
                     install_reqs = build_results.get(requirement.path)
+                    if not install_reqs and local_project_directory_to_sdist:
+                        local_project_directory = local_project_directory_to_sdist.get(
+                            requirement.path
+                        )
+                        if local_project_directory:
+                            install_reqs = build_results.get(local_project_directory)
                     if not install_reqs:
                         raise AssertionError(
                             "Failed to compute a project name for {requirement}. No corresponding "


Test patch:
diff --git a/tests/integration/cli/commands/test_issue_2057.py b/tests/integration/cli/commands/test_issue_2057.py
--- a/tests/integration/cli/commands/test_issue_2057.py
+++ b/tests/integration/cli/commands/test_issue_2057.py
@@ -4,10 +4,12 @@
 import os.path
 import shutil
 import subprocess
+import sys
 import tempfile
 from textwrap import dedent
 
 import colors
+import pytest
 
 from pex.cli.testing import run_pex3
 from pex.resolve.lockfile import json_codec
@@ -15,15 +17,31 @@
 from pex.typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
-    from typing import Any
+    from typing import Any, Callable, List
 
 
-def test_pex_archive_direct_reference(tmpdir):
-    # type: (Any) -> None
+@pytest.mark.parametrize(
+    ["archive_pex_requirement"],
+    [
+        pytest.param(
+            "https://github.com/VaasuDevanS/cowsay-python/archive/v5.0.zip#egg=cowsay",
+            id="Create Pex [Pip Proprietary]",
+        ),
+        pytest.param(
+            "cowsay @ https://github.com/VaasuDevanS/cowsay-python/archive/v5.0.zip",
+            id="Create Pex [PEP-508]",
+        ),
+    ],
+)
+def test_pex_archive_direct_reference(
+    tmpdir,  # type: Any
+    archive_pex_requirement,  # type: str
+):
+    # type: (...) -> None
 
     result = run_pex_command(
         args=[
-            "cowsay @ https://github.com/VaasuDevanS/cowsay-python/archive/v5.0.zip",
+            archive_pex_requirement,
             "-c",
             "cowsay",
             "--",
@@ -34,8 +52,39 @@ def test_pex_archive_direct_reference(tmpdir):
     assert "Moo!" in result.output
 
 
-def test_lock_create_archive_direct_reference(tmpdir):
-    # type: (Any) -> None
+@pytest.mark.parametrize(
+    ["archive_lock_requirement"],
+    [
+        pytest.param(
+            "https://github.com/VaasuDevanS/cowsay-python/archive/v5.0.zip#egg=cowsay",
+            id="Create Lock [Pip Proprietary]",
+        ),
+        pytest.param(
+            "cowsay @ https://github.com/VaasuDevanS/cowsay-python/archive/v5.0.zip",
+            id="Create Lock [PEP-508]",
+        ),
+    ],
+)
+@pytest.mark.parametrize(
+    ["archive_pex_requirements"],
+    [
+        pytest.param(
+            ["https://github.com/VaasuDevanS/cowsay-python/archive/v5.0.zip#egg=cowsay"],
+            id="Subset [Pip Proprietary]",
+        ),
+        pytest.param(
+            ["cowsay @ https://github.com/VaasuDevanS/cowsay-python/archive/v5.0.zip"],
+            id="Subset [PEP-508]",
+        ),
+        pytest.param([], id="Full"),
+    ],
+)
+def test_lock_create_archive_direct_reference(
+    tmpdir,  # type: Any
+    archive_lock_requirement,  # type: str
+    archive_pex_requirements,  # type: List[str]
+):
+    # type: (...) -> None
 
     pex_root = os.path.join(str(tmpdir), "pex_root")
     lock = os.path.join(str(tmpdir), "lock.json")
@@ -44,7 +93,7 @@ def test_lock_create_archive_direct_reference(tmpdir):
         "create",
         "--pex-root",
         pex_root,
-        "cowsay @ https://github.com/VaasuDevanS/cowsay-python/archive/v5.0.zip",
+        archive_lock_requirement,
         "--indent",
         "2",
         "-o",
@@ -54,7 +103,8 @@ def test_lock_create_archive_direct_reference(tmpdir):
     def assert_create_and_run_pex_from_lock():
         # type: () -> None
         result = run_pex_command(
-            args=[
+            args=archive_pex_requirements
+            + [
                 "--pex-root",
                 pex_root,
                 "--runtime-pex-root",
@@ -75,8 +125,31 @@ def assert_create_and_run_pex_from_lock():
     assert_create_and_run_pex_from_lock()
 
 
-def test_lock_create_local_project_direct_reference(tmpdir):
-    # type: (Any) -> None
+@pytest.mark.parametrize(
+    ["create_local_project_lock_requirement"],
+    [
+        pytest.param(lambda clone_dir: clone_dir, id="Create Lock [Pip Proprietary]"),
+        pytest.param(
+            lambda clone_dir: "ansicolors @ file://{}".format(clone_dir), id="Create Lock [PEP-508]"
+        ),
+    ],
+)
+@pytest.mark.parametrize(
+    ["create_local_project_pex_requirements"],
+    [
+        pytest.param(lambda clone_dir: [clone_dir], id="Subset [Pip Proprietary]"),
+        pytest.param(
+            lambda clone_dir: ["ansicolors @ file://{}".format(clone_dir)], id="Subset [PEP-508]"
+        ),
+        pytest.param(lambda clone_dir: [], id="Full"),
+    ],
+)
+def test_lock_create_local_project_direct_reference(
+    tmpdir,  # type: Any
+    create_local_project_lock_requirement,  # type: Callable[[str], str]
+    create_local_project_pex_requirements,  # type: Callable[[str], List[str]]
+):
+    # type: (...) -> None
 
     clone_dir = os.path.join(str(tmpdir), "ansicolors")
     subprocess.check_call(args=["git", "init", clone_dir])
@@ -95,6 +168,9 @@ def test_lock_create_local_project_direct_reference(tmpdir):
     )
     subprocess.check_call(args=["git", "reset", "--hard", ansicolors_1_1_8_sha], cwd=clone_dir)
 
+    lock_requirement = create_local_project_lock_requirement(clone_dir)
+    pex_requirements = create_local_project_pex_requirements(clone_dir)
+
     pex_root = os.path.join(str(tmpdir), "pex_root")
     lock = os.path.join(str(tmpdir), "lock.json")
     run_pex3(
@@ -102,7 +178,7 @@ def test_lock_create_local_project_direct_reference(tmpdir):
         "create",
         "--pex-root",
         pex_root,
-        "ansicolors @ file://{}".format(clone_dir),
+        lock_requirement,
         "--indent",
         "2",
         "-o",
@@ -112,7 +188,8 @@ def test_lock_create_local_project_direct_reference(tmpdir):
     def assert_create_and_run_pex_from_lock():
         # type: () -> None
         result = run_pex_command(
-            args=[
+            args=pex_requirements
+            + [
                 "--pex-root",
                 pex_root,
                 "--runtime-pex-root",