Repository: sbidoul/pip-deepfreeze (language: python)

Issue:
Use pip inspect when available
Instead of our `pip_list_json.py` script. This will also resolve #54.


Discussion / hints:

Fix patch:
diff --git a/news/97.feature b/news/97.feature
new file mode 100644
index 0000000..71fa331
--- /dev/null
+++ b/news/97.feature
@@ -0,0 +1,2 @@
+Use ``pip inspect`` when the target pip supports it. This allows working in virtual
+environments where ``pkg_resources and thus ``setuptools`` are not installed.
diff --git a/pyproject.toml b/pyproject.toml
index a6b2458..5ad6d6c 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -25,7 +25,7 @@ classifiers = [
 requires-python = ">=3.7"
 dependencies=[
     "httpx",
-    "packaging>=20.4",
+    "packaging>=23",
     "toml",
     "typer[all]>=0.3.2",
     "typing-extensions ; python_version<'3.8'",  # for Protocol, TypedDict
diff --git a/src/pip_deepfreeze/installed_dist.py b/src/pip_deepfreeze/installed_dist.py
index 6bf7f00..9f391a3 100644
--- a/src/pip_deepfreeze/installed_dist.py
+++ b/src/pip_deepfreeze/installed_dist.py
@@ -1,3 +1,4 @@
+from abc import ABC, abstractproperty
 from typing import Any, Dict, List, Optional
 
 from packaging.requirements import Requirement
@@ -21,7 +22,9 @@ def __str__(self) -> str:
             return str(url)
 
 
-class InstalledDistribution:
+class InstalledDistribution(ABC):
+    """Abstract class for an installed distribution."""
+
     def __init__(self, data: Dict[str, Any]):
         self.data = data
 
@@ -44,8 +47,23 @@ def direct_url(self) -> Optional[DirectUrl]:
 
     @property
     def requires_dist(self) -> List[Requirement]:
+        """Requires-Dist metadata."""
         return [Requirement(r) for r in self.data["metadata"].get("requires_dist", [])]
 
+    @abstractproperty
+    def requires(self) -> List[Requirement]:
+        """Base dependencies, filtered for the environment."""
+        ...
+
+    @abstractproperty
+    def extra_requires(self) -> Dict[NormalizedName, List[Requirement]]:
+        """Extra dependencies, filtered for the environment."""
+        ...
+
+
+class EnvInfoInstalledDistribution(InstalledDistribution):
+    """An InstalledDistribution built from env_info_json.py output."""
+
     @property
     def requires(self) -> List[Requirement]:
         return [Requirement(req) for req in self.data.get("requires", [])]
@@ -58,4 +76,32 @@ def extra_requires(self) -> Dict[NormalizedName, List[Requirement]]:
         }
 
 
+class PipInspectInstalledDistribution(InstalledDistribution):
+    """An InstalledDistribution built from pip inspect output."""
+
+    def __init__(self, data: Dict[str, Any], environment: Dict[str, str]):
+        super().__init__(data)
+        self.environment = environment
+
+    @property
+    def requires(self) -> List[Requirement]:
+        return [
+            req
+            for req in self.requires_dist
+            if req.marker is None or req.marker.evaluate(self.environment)
+        ]
+
+    @property
+    def extra_requires(self) -> Dict[NormalizedName, List[Requirement]]:
+        return {
+            canonicalize_name(extra): [
+                req
+                for req in self.requires_dist
+                if req.marker is not None
+                and req.marker.evaluate(dict(self.environment, extra=extra))
+            ]
+            for extra in self.data["metadata"].get("provides_extra", [])
+        }
+
+
 InstalledDistributions = Dict[NormalizedName, InstalledDistribution]
diff --git a/src/pip_deepfreeze/pip.py b/src/pip_deepfreeze/pip.py
index 9322d3c..7a51c68 100644
--- a/src/pip_deepfreeze/pip.py
+++ b/src/pip_deepfreeze/pip.py
@@ -1,10 +1,16 @@
 import json
 from importlib.resources import path as resource_path
 from pathlib import Path
-from typing import Dict, Iterable, List, Optional, Sequence, Tuple
+from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple, cast
 
-from .compat import NormalizedName, shlex_join
-from .installed_dist import InstalledDistribution, InstalledDistributions
+from packaging.version import Version
+
+from .compat import NormalizedName, TypedDict, shlex_join
+from .installed_dist import (
+    EnvInfoInstalledDistribution,
+    InstalledDistributions,
+    PipInspectInstalledDistribution,
+)
 from .list_installed_depends import (
     list_installed_depends,
     list_installed_depends_by_extra,
@@ -16,6 +22,7 @@
     parse as parse_req_file,
 )
 from .req_parser import get_req_name
+from .sanity import get_pip_version
 from .utils import (
     check_call,
     check_output,
@@ -26,6 +33,12 @@
 )
 
 
+class PipInspectReport(TypedDict, total=False):
+    version: str
+    installed: List[Dict[str, Any]]
+    environment: Dict[str, str]
+
+
 def pip_upgrade_project(
     python: str,
     constraints_filename: Path,
@@ -106,18 +119,42 @@ def pip_upgrade_project(
     check_call(cmd)
 
 
-def pip_list(python: str) -> InstalledDistributions:
-    """List installed distributions.
-
-    Currently works via pip_list_json.py, but this could become a native
-    pip feature in the future.
-    """
+def _pip_list__env_info_json(python: str) -> InstalledDistributions:
     with resource_path("pip_deepfreeze", "pip_list_json.py") as pip_list_json:
         json_dists = json.loads(check_output([python, str(pip_list_json)]))
-        dists = [InstalledDistribution(json_dist) for json_dist in json_dists]
+        dists = [EnvInfoInstalledDistribution(json_dist) for json_dist in json_dists]
         return {dist.name: dist for dist in dists}
 
 
+def _pip_inspect(python: str) -> PipInspectReport:
+    return cast(
+        PipInspectReport,
+        json.loads(check_output([python, "-m", "pip", "--quiet", "inspect"])),
+    )
+
+
+def _pip_list__pip_inspect(python: str) -> InstalledDistributions:
+    inspect = _pip_inspect(python)
+    if inspect["version"] not in ("0", "1"):
+        raise SystemExit(
+            f"Unspported 'pip inspect' output format version '{inspect['version']}'"
+        )
+    environment = inspect["environment"]
+    dists = [
+        PipInspectInstalledDistribution(json_dist, environment)
+        for json_dist in inspect["installed"]
+    ]
+    return {dist.name: dist for dist in dists}
+
+
+def pip_list(python: str) -> InstalledDistributions:
+    """List installed distributions."""
+    if get_pip_version(python) >= Version("22.2"):
+        return _pip_list__pip_inspect(python)
+    else:
+        return _pip_list__env_info_json(python)
+
+
 def pip_freeze(python: str) -> Iterable[str]:
     """Run pip freeze."""
     cmd = [python, "-m", "pip", "freeze"]
diff --git a/src/pip_deepfreeze/pip_list_json.py b/src/pip_deepfreeze/pip_list_json.py
index 948cbe3..5723ac4 100755
--- a/src/pip_deepfreeze/pip_list_json.py
+++ b/src/pip_deepfreeze/pip_list_json.py
@@ -2,11 +2,9 @@
 """List installed distributions with some of their metadata in json format.
 
 This currently assumes pkg_resources is installed.
-TODO: use importlib.metadata and fallback on pkg_resources if not available.
+In environments without setuptools, we'll use 'pip inspect'.
 
 This script must be python 2 compatible.
-
-This may one day become a native pip feature (https://github.com/pypa/pip/pull/8008).
 """
 
 import json
diff --git a/src/pip_deepfreeze/sanity.py b/src/pip_deepfreeze/sanity.py
index f30cf89..9405cce 100644
--- a/src/pip_deepfreeze/sanity.py
+++ b/src/pip_deepfreeze/sanity.py
@@ -1,5 +1,6 @@
 import json
 import subprocess
+from functools import lru_cache
 from importlib.resources import path as resource_path
 from typing import Optional, cast
 
@@ -23,6 +24,7 @@
 )
 
 
+@lru_cache()
 def _get_env_info(python: str) -> EnvInfo:
     with resource_path("pip_deepfreeze", "env_info_json.py") as env_info_json_script:
         try:
@@ -35,7 +37,15 @@ def _get_env_info(python: str) -> EnvInfo:
             return cast(EnvInfo, json.loads(env_info_json))
 
 
+def get_pip_version(python: str) -> Version:
+    pip_version = _get_env_info(python).get("pip_version")
+    # assert because we have checked the pip availability before
+    assert pip_version, "pip is not available"
+    return Version(pip_version)
+
+
 def check_env(python: str) -> bool:
+    _get_env_info.cache_clear()
     env_info = _get_env_info(python)
     if not env_info.get("in_virtualenv"):
         log_error(
@@ -50,17 +60,27 @@ def check_env(python: str) -> bool:
             f"refusing to start."
         )
         return False
-    if not env_info.get("has_pkg_resources"):
+    pip_version = env_info.get("pip_version")
+    if not env_info.get("has_pkg_resources") and (
+        # pip_version is None if pkg_resources is not installed for python<3.8
+        not pip_version
+        or Version(pip_version) < Version("22.2")
+    ):
         setuptools_install_cmd = shlex_join(
             [python, "-m", "pip", "install", "setuptools"]
         )
+        pip_upgrade_cmd = shlex_join(
+            [python, "-m", "pip", "install", "--upgrade", "pip"]
+        )
         log_error(
             f"pkg_resources is not available to {python}. It is currently "
-            f"required by pip-deepfreeze. "
-            f"You can install it with {setuptools_install_cmd}."
+            f"required by pip-deepfreeze unless you have pip>=22.2. "
+            f"You can either upgrade pip with '{pip_upgrade_cmd}' or "
+            f"installs pkg_resources with '{setuptools_install_cmd}'."
         )
         return False
-    pip_version = env_info.get("pip_version")
+    # Testing for pip must be done after testing for pkg_resources, because
+    # pkg_resources is needed to obtain the pip version for python < 3.8.
     if not pip_version:
         log_error(f"pip is not available to {python}. Please install it.")
         return False
@@ -69,7 +89,7 @@ def check_env(python: str) -> bool:
         log_warning(
             f"pip-deepfreeze works best with pip>=20.1, "
             f"in particular if you use direct URL references. "
-            f"You can upgrade pip it with {pip_install_cmd}."
+            f"You can upgrade pip it with '{pip_install_cmd}'."
         )
     if not env_info.get("wheel_version") and Version(pip_version) < Version("23.1"):
         wheel_install_cmd = shlex_join([python, "-m", "pip", "install", "wheel"])
diff --git a/src/pip_deepfreeze/tree.py b/src/pip_deepfreeze/tree.py
index f846912..d8318c4 100644
--- a/src/pip_deepfreeze/tree.py
+++ b/src/pip_deepfreeze/tree.py
@@ -14,6 +14,12 @@
 NodeKey = Tuple[NormalizedName, Tuple[NormalizedName, ...]]
 
 
+def _req_name_with_extras(req: Requirement) -> str:
+    if req.extras:
+        return f"{req.name}[{','.join(sorted(req.extras))}]"
+    return req.name
+
+
 class Node:
     def __init__(self, req: Requirement, dist: Optional[InstalledDistribution]):
         self.req = req
@@ -36,7 +42,7 @@ def _print(indent: List[str], node: Node) -> None:
             BRANCH = "│   "
             TEE = "├── "
             LAST = "└── "
-            typer.echo(f"{''.join(indent)}{node.req}", nl=False)
+            typer.echo(f"{''.join(indent)}{_req_name_with_extras(node.req)}", nl=False)
             if node in seen:
                 typer.secho(" ⬆", dim=True)
                 return


Test patch:
diff --git a/tests/test_pip.py b/tests/test_pip.py
index 7de97a5..5c128f0 100644
--- a/tests/test_pip.py
+++ b/tests/test_pip.py
@@ -6,6 +6,8 @@
 from packaging.requirements import Requirement
 
 from pip_deepfreeze.pip import (
+    _pip_list__env_info_json,
+    _pip_list__pip_inspect,
     pip_freeze,
     pip_freeze_dependencies,
     pip_freeze_dependencies_by_extra,
@@ -310,7 +312,10 @@ def test_pip_upgrade_vcs_url(virtualenv_python, tmp_path, capfd):
     ]
 
 
-def test_pip_list(virtualenv_python, testpkgs):
+@pytest.mark.parametrize(
+    "pip_list_function", (_pip_list__env_info_json, _pip_list__pip_inspect, pip_list)
+)
+def test_pip_list(virtualenv_python, testpkgs, pip_list_function):
     subprocess.check_call(
         [
             virtualenv_python,
@@ -323,7 +328,7 @@ def test_pip_list(virtualenv_python, testpkgs):
             "pkgb",
         ]
     )
-    installed_dists = pip_list(virtualenv_python)
+    installed_dists = pip_list_function(virtualenv_python)
     assert set(installed_dists.keys()) == {"pkga", "pkgb", "pip", "wheel", "setuptools"}
     pkga_dist = installed_dists["pkga"]
     assert pkga_dist.name == "pkga"
@@ -332,12 +337,15 @@ def test_pip_list(virtualenv_python, testpkgs):
     pkgb_dist = installed_dists["pkgb"]
     assert pkgb_dist.name == "pkgb"
     assert pkgb_dist.version == "0.0.0"
-    assert repr(pkgb_dist.requires) == repr([Requirement("pkga")])
+    assert [r.name for r in pkgb_dist.requires] == ["pkga"]
     assert repr(pkgb_dist.requires_dist) == repr([Requirement("pkga<0.0.1")])
     # assert pkga_dist.direct_url is None
 
 
-def test_pip_list_extras(virtualenv_python, testpkgs):
+@pytest.mark.parametrize(
+    "pip_list_function", (_pip_list__env_info_json, _pip_list__pip_inspect, pip_list)
+)
+def test_pip_list_extras(virtualenv_python, testpkgs, pip_list_function):
     subprocess.check_call(
         [
             virtualenv_python,
@@ -350,7 +358,7 @@ def test_pip_list_extras(virtualenv_python, testpkgs):
             "pkge",
         ]
     )
-    installed_dists = pip_list(virtualenv_python)
+    installed_dists = pip_list_function(virtualenv_python)
     assert set(installed_dists.keys()) == {
         "pkga",
         "pkgb",
@@ -363,8 +371,8 @@ def test_pip_list_extras(virtualenv_python, testpkgs):
     }
     pkgd_dist = installed_dists["pkgd"]
     assert repr(pkgd_dist.requires) == repr([Requirement("pkga")])
-    assert repr(pkgd_dist.extra_requires) == repr(
-        {"b": [Requirement("pkgb")], "c": [Requirement("pkgc")]}
-    )
+    assert len(pkgd_dist.extra_requires) == 2
+    assert [r.name for r in pkgd_dist.extra_requires["b"]] == ["pkgb"]
+    assert [r.name for r in pkgd_dist.extra_requires["c"]] == ["pkgc"]
     pkge_dist = installed_dists["pkge"]
     assert repr(pkge_dist.requires) == repr([Requirement("pkgd[b,c]")])
diff --git a/tests/test_sanity.py b/tests/test_sanity.py
index 1069156..e013685 100644
--- a/tests/test_sanity.py
+++ b/tests/test_sanity.py
@@ -25,10 +25,11 @@ def test_sanity_pip_version(virtualenv_python, capsys):
 
 def test_sanity_pkg_resources(virtualenv_python, capsys):
     assert check_env(virtualenv_python)
+    subprocess.check_call([virtualenv_python, "-m", "pip", "install", "-q", "pip<22.2"])
     subprocess.check_call(
         [virtualenv_python, "-m", "pip", "uninstall", "-qy", "setuptools"]
     )
-    # pkg_resources is currently required
+    # pkg_resources is required when `pip inspect` is not available
     assert not check_env(virtualenv_python)
     captured = capsys.readouterr()
     assert "pkg_resources is not available" in captured.err