Repository: pex-tool/pex

Issue:
Support VCS requirements 1st class
Right now Pex requirement parsing lumps these under `URLRequirement` precluding any special handling required. One way or another #1556 will require recognizing VCS requirements.


Discussion:
These VCS requirements can come in two forms:
1. The more limited, but now tandard PEP-440 direct reference form: https://www.python.org/dev/peps/pep-0440/#direct-references
2. The original more full-featured Pip proprietary form: https://pip.pypa.io/en/stable/topics/vcs-support/

Resolving patch:
diff --git a/pex/cli/commands/lockfile/__init__.py b/pex/cli/commands/lockfile/__init__.py
--- a/pex/cli/commands/lockfile/__init__.py
+++ b/pex/cli/commands/lockfile/__init__.py
@@ -6,8 +6,8 @@
 from pex import resolver
 from pex.cli.commands.lockfile.lockfile import Lockfile as Lockfile  # For re-export.
 from pex.commands.command import Error
-from pex.common import safe_open
-from pex.requirements import LocalProjectRequirement
+from pex.common import pluralize, safe_open
+from pex.requirements import LocalProjectRequirement, VCSRequirement
 from pex.resolve.locked_resolve import LockConfiguration
 from pex.resolve.requirement_configuration import RequirementConfiguration
 from pex.resolve.resolver_configuration import PipConfiguration
@@ -86,20 +86,29 @@ def create(
 
     network_configuration = pip_configuration.network_configuration
     requirements = []  # type: List[Requirement]
-    local_projects = []  # type: List[LocalProjectRequirement]
+    projects = []  # type: List[str]
     for parsed_requirement in requirement_configuration.parse_requirements(network_configuration):
         if isinstance(parsed_requirement, LocalProjectRequirement):
-            local_projects.append(parsed_requirement)
+            projects.append("local project at {path}".format(path=parsed_requirement.path))
+        elif isinstance(parsed_requirement, VCSRequirement):
+            projects.append(
+                "{vcs} project {project_name} at {url}".format(
+                    vcs=parsed_requirement.vcs,
+                    project_name=parsed_requirement.requirement.project_name,
+                    url=parsed_requirement.url,
+                )
+            )
         else:
             requirements.append(parsed_requirement.requirement)
-    if local_projects:
+    if projects:
         return Error(
-            "Cannot create a lock for local project requirements. Given {count}:\n"
-            "{projects}".format(
-                count=len(local_projects),
-                projects="\n".join(
-                    "{index}.) {project}".format(index=index, project=project.path)
-                    for index, project in enumerate(local_projects, start=1)
+            "Cannot create a lock for project requirements built from local or version "
+            "controlled sources. Given {count} such {projects}:\n{project_descriptions}".format(
+                count=len(projects),
+                projects=pluralize(projects, "project"),
+                project_descriptions="\n".join(
+                    "{index}.) {project}".format(index=index, project=project)
+                    for index, project in enumerate(projects, start=1)
                 ),
             )
         )
diff --git a/pex/requirements.py b/pex/requirements.py
--- a/pex/requirements.py
+++ b/pex/requirements.py
@@ -10,12 +10,13 @@
 from pex import attrs, dist_metadata
 from pex.compatibility import urlparse
 from pex.dist_metadata import MetadataError, ProjectNameAndVersion
+from pex.enum import Enum
 from pex.fetcher import URLFetcher
 from pex.third_party.packaging.markers import Marker
 from pex.third_party.packaging.specifiers import SpecifierSet
 from pex.third_party.packaging.version import InvalidVersion, Version
 from pex.third_party.pkg_resources import Requirement, RequirementParseError
-from pex.typing import TYPE_CHECKING
+from pex.typing import TYPE_CHECKING, cast
 
 if TYPE_CHECKING:
     import attr  # vendor:skip
@@ -143,7 +144,7 @@ class PyPIRequirement(object):
 
 @attr.s(frozen=True)
 class URLRequirement(object):
-    """A requirement realized through an distribution archive at a fixed URL."""
+    """A requirement realized through a distribution archive at a fixed URL."""
 
     line = attr.ib()  # type: LogicalLine
     url = attr.ib()  # type: Text
@@ -151,6 +152,27 @@ class URLRequirement(object):
     editable = attr.ib(default=False)  # type: bool
 
 
+class VCS(Enum):
+    class Value(Enum.Value):
+        pass
+
+    Bazaar = Value("bzr")
+    Git = Value("git")
+    Mercurial = Value("hg")
+    Subversion = Value("svn")
+
+
+@attr.s(frozen=True)
+class VCSRequirement(object):
+    """A requirement realized by building a distribution from sources retrieved from a VCS."""
+
+    line = attr.ib()  # type: LogicalLine
+    vcs = attr.ib()  # type: VCS.Value
+    url = attr.ib()  # type: Text
+    requirement = attr.ib()  # type: Requirement
+    editable = attr.ib(default=False)  # type: bool
+
+
 def parse_requirement_from_project_name_and_specifier(
     project_name,  # type: Text
     extras=None,  # type: Optional[Iterable[str]]
@@ -208,7 +230,9 @@ def as_requirement(self, dist):
 
 
 if TYPE_CHECKING:
-    ParsedRequirement = Union[PyPIRequirement, URLRequirement, LocalProjectRequirement]
+    ParsedRequirement = Union[
+        PyPIRequirement, URLRequirement, VCSRequirement, LocalProjectRequirement
+    ]
 
 
 @attr.s(frozen=True)
@@ -243,29 +267,54 @@ def _strip_requirement_options(line):
     return editable, re.sub(r"\s--(global-option|install-option|hash).*$", "", processed_text)
 
 
-def _is_recognized_non_local_pip_url_scheme(scheme):
-    # type: (str) -> bool
-    return bool(
-        re.match(
-            r"""
-            (
+class ArchiveScheme(Enum):
+    class Value(Enum.Value):
+        pass
+
+    FTP = Value("ftp")
+    HTTP = Value("http")
+    HTTPS = Value("https")
+
+
+@attr.s(frozen=True)
+class VCSScheme(object):
+    vcs = attr.ib()  # type: VCS.Value
+    scheme = attr.ib()  # type: str
+
+
+def _parse_non_local_scheme(scheme):
+    # type: (str) -> Optional[Union[ArchiveScheme.Value, VCSScheme]]
+    match = re.match(
+        r"""
+        ^
+        (?:
+            (?P<archive_scheme>
                 # Archives
                   ftp
                 | https?
-
-                # VCSs: https://pip.pypa.io/en/stable/reference/pip_install/#vcs-support
-                | (
-                      bzr
-                    | git
-                    | hg
-                    | svn
-                  )\+
             )
-            """,
-            scheme,
-            re.VERBOSE,
+            |
+            (?P<vcs_type>
+                # VCSs: https://pip.pypa.io/en/stable/reference/pip_install/#vcs-support       
+                  bzr
+                | git
+                | hg
+                | svn
+            )\+(?P<vcs_scheme>.+)
         )
+        $
+        """,
+        scheme,
+        re.VERBOSE,
     )
+    if not match:
+        return None
+
+    archive_scheme = match.group("archive_scheme")
+    if archive_scheme:
+        return cast(ArchiveScheme.Value, ArchiveScheme.for_value(archive_scheme))
+
+    return VCSScheme(vcs=VCS.for_value(match.group("vcs_type")), scheme=match.group("vcs_scheme"))
 
 
 @attr.s(frozen=True)
@@ -416,8 +465,9 @@ def _parse_requirement_line(
     project_name, direct_reference_url = _split_direct_references(processed_text)
     parsed_url = urlparse.urlparse(direct_reference_url or processed_text)
 
-    # Handle non local URLs (Pip proprietary).
-    if _is_recognized_non_local_pip_url_scheme(parsed_url.scheme):
+    # Handle non local URLs.
+    non_local_scheme = _parse_non_local_scheme(parsed_url.scheme)
+    if non_local_scheme:
         project_name_extras_and_marker = _try_parse_fragment_project_name_and_marker(
             parsed_url.fragment
         )
@@ -454,6 +504,9 @@ def _parse_requirement_line(
             specifier=specifier,
             marker=marker,
         )
+        if isinstance(non_local_scheme, VCSScheme):
+            url = urlparse.urlparse(url)._replace(scheme=non_local_scheme.scheme).geturl()
+            return VCSRequirement(line, non_local_scheme.vcs, url, requirement, editable=editable)
         return URLRequirement(line, url, requirement, editable=editable)
 
     # Handle local archives and project directories via path or file URL (Pip proprietary).


Test patch:
diff --git a/tests/integration/cli/commands/test_lock.py b/tests/integration/cli/commands/test_lock.py
--- a/tests/integration/cli/commands/test_lock.py
+++ b/tests/integration/cli/commands/test_lock.py
@@ -105,6 +105,36 @@ def create_lock(style):
     assert 1 == len(create_lock("sources").additional_artifacts)
 
 
+def test_create_local_unsupported(pex_project_dir):
+    # type: (str) -> None
+
+    result = run_pex3("lock", "create", pex_project_dir)
+    result.assert_failure()
+    assert (
+        "Cannot create a lock for project requirements built from local or version controlled "
+        "sources. Given 1 such project:\n"
+        "1.) local project at {path}\n".format(path=pex_project_dir)
+    ) == result.error
+
+
+def test_create_vcs_unsupported():
+    # type: () -> None
+
+    result = run_pex3(
+        "lock",
+        "create",
+        "pex @ git+https://github.com/pantsbuild/pex@473c6ac7",
+        "git+https://github.com/pypa/pip@f0f67af3#egg=pip",
+    )
+    result.assert_failure()
+    assert (
+        "Cannot create a lock for project requirements built from local or version controlled "
+        "sources. Given 2 such projects:\n"
+        "1.) git project pex at https://github.com/pantsbuild/pex@473c6ac7\n"
+        "2.) git project pip at https://github.com/pypa/pip@f0f67af3\n"
+    ) == result.error
+
+
 UPDATE_LOCKFILE_CONTENTS = """\
 {
   "allow_builds": true,
diff --git a/tests/test_requirements.py b/tests/test_requirements.py
--- a/tests/test_requirements.py
+++ b/tests/test_requirements.py
@@ -11,6 +11,7 @@
 from pex.common import safe_open, temporary_dir, touch
 from pex.fetcher import URLFetcher
 from pex.requirements import (
+    VCS,
     Constraint,
     LocalProjectRequirement,
     LogicalLine,
@@ -18,6 +19,8 @@
     PyPIRequirement,
     Source,
     URLRequirement,
+    VCSRequirement,
+    VCSScheme,
     parse_requirement_file,
     parse_requirement_from_project_name_and_specifier,
     parse_requirements,
@@ -162,6 +165,27 @@ def url_req(
     )
 
 
+def vcs_req(
+    vcs,  # type: VCS.Value
+    url,  # type: str
+    project_name,  # type: str
+    extras=None,  # type: Optional[Iterable[str]]
+    specifier=None,  # type: Optional[str]
+    marker=None,  # type: Optional[str]
+    editable=False,  # type: bool
+):
+    # type: (...) -> VCSRequirement
+    return VCSRequirement(
+        line=DUMMY_LINE,
+        vcs=vcs,
+        url=url,
+        requirement=parse_requirement_from_project_name_and_specifier(
+            project_name, extras=extras, specifier=specifier, marker=marker
+        ),
+        editable=editable,
+    )
+
+
 def local_req(
     path,  # type: str
     extras=None,  # type: Optional[Iterable[str]]
@@ -343,12 +367,13 @@ def test_parse_requirements_stress(chroot):
         url_req(project_name="SomeProject", url="https://example.com/somewhere/over/here"),
         local_req(path=os.path.realpath("somewhere/over/here")),
         req(project_name="FooProject", specifier=">=1.2"),
-        url_req(
+        vcs_req(
+            vcs=VCS.Git,
             project_name="MyProject",
-            url="git+https://git.example.com/MyProject.git@da39a3ee5e6b4b0d3255bfef95601890afd80709",
+            url="https://git.example.com/MyProject.git@da39a3ee5e6b4b0d3255bfef95601890afd80709",
         ),
-        url_req(project_name="MyProject", url="git+ssh://git.example.com/MyProject"),
-        url_req(project_name="MyProject", url="git+file:/home/user/projects/MyProject"),
+        vcs_req(vcs=VCS.Git, project_name="MyProject", url="ssh://git.example.com/MyProject"),
+        vcs_req(vcs=VCS.Git, project_name="MyProject", url="file:///home/user/projects/MyProject"),
         Constraint(DUMMY_LINE, Requirement.parse("AnotherProject")),
         local_req(
             path=os.path.realpath("extra/a/local/project"),
@@ -368,9 +393,10 @@ def test_parse_requirements_stress(chroot):
             extras=["foo"],
             marker="python_version == '3.9'",
         ),
-        url_req(
+        vcs_req(
+            vcs=VCS.Mercurial,
             project_name="AnotherProject",
-            url="hg+http://hg.example.com/MyProject@da39a3ee5e6b",
+            url="http://hg.example.com/MyProject@da39a3ee5e6b",
             extras=["more", "extra"],
             marker="python_version == '3.9.*'",
         ),
@@ -387,11 +413,16 @@ def test_parse_requirements_stress(chroot):
             specifier="==1.9.2",
             marker="python_version == '3.4.*' and sys_platform == 'win32'",
         ),
-        url_req(project_name="Django", url="git+https://github.com/django/django.git"),
-        url_req(project_name="Django", url="git+https://github.com/django/django.git@stable/2.1.x"),
-        url_req(
+        vcs_req(vcs=VCS.Git, project_name="Django", url="https://github.com/django/django.git"),
+        vcs_req(
+            vcs=VCS.Git,
+            project_name="Django",
+            url="https://github.com/django/django.git@stable/2.1.x",
+        ),
+        vcs_req(
+            vcs=VCS.Git,
             project_name="Django",
-            url="git+https://github.com/django/django.git@fd209f62f1d83233cc634443cfac5ee4328d98b8",
+            url="https://github.com/django/django.git@fd209f62f1d83233cc634443cfac5ee4328d98b8",
         ),
         url_req(
             project_name="Django",