Repository: tox-dev/pipdeptree (python)

Issue:
duplicate packages when using `importlib.metadata`
Hmm, the thought of duplicate packages when using `importlib.metadata` came to my mind and it looks like the discovery algorithm will need more work. I installed `pytest` both in my system environment and in a temporary directory. I assigned this temporary directory to `PYTHONPATH` and was able to see it twice, including its dependencies. Here's an example:
```console
$ mkdir temp-site-pkgs
$ pip install --target temp-site-pkgs/ pytest
Collecting pytest
  Using cached pytest-8.1.1-py3-none-any.whl.metadata (7.6 kB)
. . .
Installing collected packages: tomli, pluggy, packaging, iniconfig, exceptiongroup, pytest
Successfully installed exceptiongroup-1.2.0 iniconfig-2.0.0 packaging-24.0 pluggy-1.4.0 pytest-8.1.1 tomli-2.0.1
$ PYTHONPATH=/workspaces/pipdeptree/temp-site-pkgs pipdeptree -a -d 0
attrs==23.2.0
Automat==22.10.0
cachetools==5.3.3
certifi==2024.2.2
cffi==1.16.0
cfgv==3.4.0
chardet==5.2.0
charset-normalizer==3.3.2
colorama==0.4.6
constantly==23.10.4
covdefaults==2.3.0
coverage==7.4.4
cryptography==42.0.5
cssselect==1.2.0
diff_cover==8.0.3
distlib==0.3.8
exceptiongroup==1.2.0                             # <--- Here
exceptiongroup==1.2.0
filelock==3.13.1
gitdb==4.0.11
GitPython==3.1.41
hyperlink==21.0.0
identify==2.5.35
idna==3.6
incremental==22.10.0
iniconfig==2.0.0                             # <--- Here
iniconfig==2.0.0
itemadapter==0.8.0
itemloaders==1.1.0
Jinja2==3.1.3
jmespath==1.0.1
lxml==5.2.1
MarkupSafe==2.1.5
nodeenv==1.8.0
packaging==24.0                         # <--- Here
packaging==24.0
parsel==1.9.0
pip==24.0
pip==23.0.1
pipdeptree==2.16.3.dev3+g91d21e3
platformdirs==4.2.0
pluggy==1.4.0                                # <--- Here
pluggy==1.4.0
pre-commit==3.7.0
Protego==0.3.0
pyasn1==0.6.0
pyasn1_modules==0.4.0
pycparser==2.22
PyDispatcher==2.0.7
Pygments==2.17.2
pyOpenSSL==24.1.0
pyproject-api==1.6.1
pytest==8.1.1                             # <--- Here
pytest==8.1.1
pytest-cov==4.1.0
pytest-mock==3.14.0
PyYAML==6.0.1
queuelib==1.6.2
requests==2.31.0
requests-file==2.0.0
Scrapy==2.11.1
service-identity==24.1.0
setuptools==69.0.3
six==1.16.0
smmap==5.0.1
tldextract==5.1.2
tomli==2.0.1                             # <--- Here
tomli==2.0.1
tox==4.14.2
Twisted==24.3.0
typing_extensions==4.10.0
urllib3==2.2.1
virtualenv==20.25.1
w3lib==2.1.2
wheel==0.42.0
zope.interface==6.2
```

_Originally posted by @kemzeb in https://github.com/tox-dev/pipdeptree/pull/333#issuecomment-2035034418_

Pull request description:
Handle duplicated package metadata when using importlib.metadata
Fixes #341.

Using an environment similar to the one shown in the issue:
```console
$ PYTHONPATH=/workspaces/pipdeptree/temp-site-pkgs/ pipdeptree -a -d 0
Warning!!! Duplicate package metadata found:
"/home/vscode/.local/lib/python3.9/site-packages"
  pluggy                           1.4.0            (using 1.4.0, "/workspaces/pipdeptree/temp-site-pkgs")
  packaging                        24.0             (using 24.0, "/workspaces/pipdeptree/temp-site-pkgs")
  tomli                            2.0.1            (using 2.0.1, "/workspaces/pipdeptree/temp-site-pkgs")
  pytest                           8.1.1            (using 8.1.1, "/workspaces/pipdeptree/temp-site-pkgs")
  exceptiongroup                   1.2.0            (using 1.2.0, "/workspaces/pipdeptree/temp-site-pkgs")
  iniconfig                        2.0.0            (using 2.0.0, "/workspaces/pipdeptree/temp-site-pkgs")
"/usr/local/lib/python3.9/site-packages"
  pip                              23.0.1           (using 24.0, "/home/vscode/.local/lib/python3.9/site-packages")
------------------------------------------------------------------------
chardet==5.2.0
covdefaults==2.3.0
coverage==7.4.4
diff_cover==8.0.3
distlib==0.3.8
exceptiongroup==1.2.0
filelock==3.13.3
gitdb==4.0.11
GitPython==3.1.41
iniconfig==2.0.0
Jinja2==3.1.3
MarkupSafe==2.1.5
packaging==24.0
pip==24.0
pipdeptree==2.16.3.dev3+g91d21e3.d20240403
platformdirs==4.2.0
pluggy==1.4.0
Pygments==2.17.2
pytest==8.1.1
pytest-cov==5.0.0
pytest-mock==3.14.0
setuptools==69.0.3
smmap==5.0.1
tomli==2.0.1
virtualenv==20.25.1
wheel==0.42.0
```

Patch:
diff --git a/src/pipdeptree/_discovery.py b/src/pipdeptree/_discovery.py
index 1773963..44e0a13 100644
--- a/src/pipdeptree/_discovery.py
+++ b/src/pipdeptree/_discovery.py
@@ -2,28 +2,75 @@ from __future__ import annotations
 
 import site
 import sys
-from importlib.metadata import distributions
-from typing import TYPE_CHECKING
+from importlib.metadata import Distribution, distributions
+from typing import Iterable, Tuple
 
-if TYPE_CHECKING:
-    from importlib.metadata import Distribution
+from packaging.utils import canonicalize_name
 
 
 def get_installed_distributions(
     local_only: bool = False,  # noqa: FBT001, FBT002
     user_only: bool = False,  # noqa: FBT001, FBT002
 ) -> list[Distribution]:
-    if user_only:
-        return list(distributions(path=[site.getusersitepackages()]))
-
-    # NOTE: See https://docs.python.org/3/library/venv.html#how-venvs-work for more details.
+    # See https://docs.python.org/3/library/venv.html#how-venvs-work for more details.
     in_venv = sys.prefix != sys.base_prefix
+    original_dists: Iterable[Distribution] = []
 
     if local_only and in_venv:
         venv_site_packages = site.getsitepackages([sys.prefix])
-        return list(distributions(path=venv_site_packages))
+        original_dists = distributions(path=venv_site_packages)
+    elif user_only:
+        original_dists = distributions(path=[site.getusersitepackages()])
+    else:
+        original_dists = distributions()
+
+    # Since importlib.metadata.distributions() can return duplicate packages, we need to handle this. pip's approach is
+    # to keep track of each package metadata it finds, and if it encounters one again it will simply just ignore it. We
+    # take it one step further and warn the user that there are duplicate packages in their environment.
+    # See https://github.com/pypa/pip/blob/7c49d06ea4be4635561f16a524e3842817d1169a/src/pip/_internal/metadata/importlib/_envs.py#L34
+    seen_dists: dict[str, Distribution] = {}
+    first_seen_to_already_seen_dists_dict: dict[Distribution, list[Distribution]] = {}
+    dists = []
+    for dist in original_dists:
+        normalized_name = canonicalize_name(dist.metadata["Name"])
+        if normalized_name not in seen_dists:
+            seen_dists[normalized_name] = dist
+            dists.append(dist)
+            continue
+        already_seen_dists = first_seen_to_already_seen_dists_dict.setdefault(seen_dists[normalized_name], [])
+        already_seen_dists.append(dist)
+
+    if first_seen_to_already_seen_dists_dict:
+        render_duplicated_dist_metadata_text(first_seen_to_already_seen_dists_dict)
+
+    return dists
+
+
+FirstSeenWithDistsPair = Tuple[Distribution, Distribution]
+
+
+def render_duplicated_dist_metadata_text(
+    first_seen_to_already_seen_dists_dict: dict[Distribution, list[Distribution]],
+) -> None:
+    entries_to_pairs_dict: dict[str, list[FirstSeenWithDistsPair]] = {}
+    for first_seen, dists in first_seen_to_already_seen_dists_dict.items():
+        for dist in dists:
+            entry = str(dist.locate_file(""))
+            dist_list = entries_to_pairs_dict.setdefault(entry, [])
+            dist_list.append((first_seen, dist))
 
-    return list(distributions())
+    print("Warning!!! Duplicate package metadata found:", file=sys.stderr)  # noqa: T201
+    for entry, pairs in entries_to_pairs_dict.items():
+        print(f'"{entry}"', file=sys.stderr)  # noqa: T201
+        for first_seen, dist in pairs:
+            print(  # noqa: T201
+                (
+                    f"  {dist.metadata['Name']:<32} {dist.version:<16} (using {first_seen.version},"
+                    f" \"{first_seen.locate_file('')}\")"
+                ),
+                file=sys.stderr,
+            )
+    print("-" * 72, file=sys.stderr)  # noqa: T201
 
 
 __all__ = [
diff --git a/src/pipdeptree/_models/dag.py b/src/pipdeptree/_models/dag.py
index 4b0acb6..80f77fa 100644
--- a/src/pipdeptree/_models/dag.py
+++ b/src/pipdeptree/_models/dag.py
@@ -5,11 +5,13 @@ from fnmatch import fnmatch
 from itertools import chain
 from typing import TYPE_CHECKING, Iterator, List, Mapping
 
+from packaging.utils import canonicalize_name
+
 if TYPE_CHECKING:
     from importlib.metadata import Distribution
 
 
-from .package import DistPackage, ReqPackage, pep503_normalize
+from .package import DistPackage, ReqPackage
 
 
 class PackageDAG(Mapping[DistPackage, List[ReqPackage]]):
@@ -43,7 +45,7 @@ class PackageDAG(Mapping[DistPackage, List[ReqPackage]]):
         for p in dist_pkgs:
             reqs = []
             for r in p.requires():
-                d = idx.get(pep503_normalize(r.name))
+                d = idx.get(canonicalize_name(r.name))
                 # Distribution.requires only return the name of requirements in metadata file, which may not be
                 # the same with the capitalized one in pip. We should retain the casing of required package name.
                 # see https://github.com/tox-dev/pipdeptree/issues/242
@@ -112,8 +114,8 @@ class PackageDAG(Mapping[DistPackage, List[ReqPackage]]):
         include_with_casing_preserved: list[str] = []
         if include:
             include_with_casing_preserved = include
-            include = [pep503_normalize(i) for i in include]
-        exclude = {pep503_normalize(s) for s in exclude} if exclude else set()
+            include = [canonicalize_name(i) for i in include]
+        exclude = {canonicalize_name(s) for s in exclude} if exclude else set()
 
         # Check for mutual exclusion of show_only and exclude sets
         # after normalizing the values to lowercase
@@ -159,7 +161,7 @@ class PackageDAG(Mapping[DistPackage, List[ReqPackage]]):
                             continue
 
         non_existent_includes = [
-            i for i in include_with_casing_preserved if pep503_normalize(i) not in matched_includes
+            i for i in include_with_casing_preserved if canonicalize_name(i) not in matched_includes
         ]
         if non_existent_includes:
             raise ValueError("No packages matched using the following patterns: " + ", ".join(non_existent_includes))
diff --git a/src/pipdeptree/_models/package.py b/src/pipdeptree/_models/package.py
index 65df5a9..370a6cd 100644
--- a/src/pipdeptree/_models/package.py
+++ b/src/pipdeptree/_models/package.py
@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-import re
 from abc import ABC, abstractmethod
 from importlib import import_module
 from importlib.metadata import Distribution, PackageNotFoundError, metadata, version
@@ -8,6 +7,7 @@ from inspect import ismodule
 from typing import TYPE_CHECKING
 
 from packaging.requirements import Requirement
+from packaging.utils import canonicalize_name
 
 if TYPE_CHECKING:
     from importlib.metadata import Distribution
@@ -15,10 +15,6 @@ if TYPE_CHECKING:
 from pipdeptree._adapter import PipBaseDistributionAdapter
 
 
-def pep503_normalize(name: str) -> str:
-    return re.sub("[-_.]+", "-", name).lower()
-
-
 class Package(ABC):
     """Abstract class for wrappers around objects that pip returns."""
 
@@ -26,7 +22,7 @@ class Package(ABC):
 
     def __init__(self, project_name: str) -> None:
         self.project_name = project_name
-        self.key = pep503_normalize(project_name)
+        self.key = canonicalize_name(project_name)
 
     def licenses(self) -> str:
         try:


Test patch:
diff --git a/tests/test_discovery.py b/tests/test_discovery.py
index 54bf553..76a9435 100644
--- a/tests/test_discovery.py
+++ b/tests/test_discovery.py
@@ -9,9 +9,11 @@ from unittest.mock import Mock
 import virtualenv
 
 from pipdeptree.__main__ import main
+from pipdeptree._discovery import get_installed_distributions
 
 if TYPE_CHECKING:
     import pytest
+    from pytest_mock import MockerFixture
 
 
 def test_local_only(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capfd: pytest.CaptureFixture[str]) -> None:
@@ -53,3 +55,27 @@ def test_user_only(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capfd: pytes
     expected = {"foo"}
 
     assert found == expected
+
+
+def test_duplicate_metadata(mocker: MockerFixture, capfd: pytest.CaptureFixture[str]) -> None:
+    mocker.patch(
+        "pipdeptree._discovery.distributions",
+        Mock(
+            return_value=[
+                Mock(metadata={"Name": "foo"}, version="1.2.5", locate_file=Mock(return_value="/path/1")),
+                Mock(metadata={"Name": "foo"}, version="5.9.0", locate_file=Mock(return_value="/path/2")),
+            ]
+        ),
+    )
+
+    dists = get_installed_distributions()
+    assert len(dists) == 1
+    # we expect it to use the first distribution found
+    assert dists[0].version == "1.2.5"
+
+    _, err = capfd.readouterr()
+    expected = (
+        'Warning!!! Duplicate package metadata found:\n"/path/2"\n  foo                              5.9.0       '
+        '     (using 1.2.5, "/path/1")\n------------------------------------------------------------------------\n'
+    )
+    assert err == expected