Repository: pex-tool/pex

Issue:
Unexpected distribution hash
I expect this to be a pex issue, if not I'll re-open in pants.

When running `./pants package` for a `pex_binary` target with a dependency on a `python_distribution` (i.e. a local dist scenario) I get this error:
```
09:35:57.99 [INFO] Completed: Extracting 2 requirements to build bin.pex from libs_lockfile.pex: y-cdi-server<7.0.0,>=6.18.10, y-logging<3.0.0,>=2.1.1
09:35:57.99 [ERROR] 1 Exception encountered:

  ProcessExecutionFailure: Process 'Extracting 2 requirements to build bin.pex from libs_lockfile.pex: y-cdi-server<7.0.0,>=6.18.10, y-logging<3.0.0,>=2.1.1' failed with exit code 1.
stdout:

stderr:
Traceback (most recent call last):
  File "/Users/x/.cache/pants/named_caches/pex_root/unzipped_pexes/38cdfc09438399cde0c3c3c4ee091074f928b118/.bootstrap/pex/pex.py", line 504, in execute
    exit_value = self._wrap_coverage(self._wrap_profiling, self._execute)
  File "/Users/x/.cache/pants/named_caches/pex_root/unzipped_pexes/38cdfc09438399cde0c3c3c4ee091074f928b118/.bootstrap/pex/pex.py", line 409, in _wrap_coverage
    return runner(*args)
  File "/Users/x/.cache/pants/named_caches/pex_root/unzipped_pexes/38cdfc09438399cde0c3c3c4ee091074f928b118/.bootstrap/pex/pex.py", line 440, in _wrap_profiling
    return runner(*args)
  File "/Users/x/.cache/pants/named_caches/pex_root/unzipped_pexes/38cdfc09438399cde0c3c3c4ee091074f928b118/.bootstrap/pex/pex.py", line 560, in _execute
    return self.execute_entry(self._pex_info.entry_point)
  File "/Users/x/.cache/pants/named_caches/pex_root/unzipped_pexes/38cdfc09438399cde0c3c3c4ee091074f928b118/.bootstrap/pex/pex.py", line 696, in execute_entry
    return self.execute_pkg_resources(entry_point)
  File "/Users/x/.cache/pants/named_caches/pex_root/unzipped_pexes/38cdfc09438399cde0c3c3c4ee091074f928b118/.bootstrap/pex/pex.py", line 728, in execute_pkg_resources
    return runner()
  File "/Users/x/.cache/pants/named_caches/pex_root/installed_wheels/011143db8b77ec9508f2599e80f422621fdb0de7d868cb08cd2f859026236c83/pex-2.1.72-py2.py3-none-any.whl/pex/bin/pex.py", line 730, in main
    env=env,
  File "/Users/x/.cache/pants/named_caches/pex_root/installed_wheels/011143db8b77ec9508f2599e80f422621fdb0de7d868cb08cd2f859026236c83/pex-2.1.72-py2.py3-none-any.whl/pex/bin/pex.py", line 750, in do_main
    cache=ENV.PEX_ROOT,
  File "/Users/x/.cache/pants/named_caches/pex_root/installed_wheels/011143db8b77ec9508f2599e80f422621fdb0de7d868cb08cd2f859026236c83/pex-2.1.72-py2.py3-none-any.whl/pex/bin/pex.py", line 571, in build_pex
    pex_builder.add_from_requirements_pex(requirements_pex)
  File "/Users/x/.cache/pants/named_caches/pex_root/installed_wheels/011143db8b77ec9508f2599e80f422621fdb0de7d868cb08cd2f859026236c83/pex-2.1.72-py2.py3-none-any.whl/pex/pex_builder.py", line 318, in add_from_requirements_pex
    add(cached_location, dist_name, dist_hash)
  File "/Users/x/.cache/pants/named_caches/pex_root/installed_wheels/011143db8b77ec9508f2599e80f422621fdb0de7d868cb08cd2f859026236c83/pex-2.1.72-py2.py3-none-any.whl/pex/pex_builder.py", line 307, in add
    dname, location, dhash, expected_dhash
pex.pex_builder.InvalidDistribution: Distribution y_cdi_integration_testrunner-0.0.local0-py3-none-any.whl at /Users/x/.pex/installed_wheels/d6149d42626fa7ae2943f479b1cf777bb6e174bb1a35163bed4ca9ee4cab922d/y_cdi_integration_testrunner-0.0.local0-py3-none-any.whl had hash c7c3dd152b10a06e28f4820d5f596f2854a4d979, expected d6149d42626fa7ae2943f479b1cf777bb6e174bb1a35163bed4ca9ee4cab922d

```

Running with verbose pex output, I see it's using a cached install for the wheel:
```
pex: Using cached /Users/x/.pex/installed_wheels/d6149d42626fa7ae2943f479b1cf777bb6e174bb1a35163bed4ca9ee4cab922d/y_cdi_integration_testrunner-0.0.local0-py3-none-any.whl
```


The `local_dists.pex` has one distribution in it:
```json
$ unzip -p local_dists.pex PEX-INFO | jq .
{
  "bootstrap_hash": "fe55fb2e0e98b7d8da0952a72ec38912a578a692",
  "build_properties": {
    "pex_version": "2.1.72"
  },
  "code_hash": "da39a3ee5e6b4b0d3255bfef95601890afd80709",
  "distributions": {
    "y_cdi_integration_testrunner-0.0.local0-py3-none-any.whl": "d6149d42626fa7ae2943f479b1cf777bb6e174bb1a35163bed4ca9ee4cab922d"
  },
  "emit_warnings": false,
  "ignore_errors": false,
  "includes_tools": false,
  "inherit_path": "false",
  "interpreter_constraints": [
    "CPython>=3.8,>=3.8.*"
  ],
  "pex_hash": "500c8488326aa7976a5e60bfc3e62f76f7cf923e",
  "pex_path": null,
  "requirements": [
    "y-cdi-integration-testrunner===0.0.local0"
  ],
  "strip_pex_env": true,
  "venv": false,
  "venv_bin_path": "false",
  "venv_copies": false,
  "venv_site_packages_copies": false
}
```

Looking at the cached wheel vs the unzipped `.whl` from the sandbox, there's one file difference:
```diff
$ diff /Users/x/.pex/installed_wheels/d6149d42626fa7ae2943f479b1cf777bb6e174bb1a35163bed4ca9ee4cab922d/y_cdi_integration_testrunner-0.0.local0-py3-none-any.whl whl
Only in /Users/x/.pex/installed_wheels/d6149d42626fa7ae2943f479b1cf777bb6e174bb1a35163bed4ca9ee4cab922d/y_cdi_integration_testrunner-0.0.local0-py3-none-any.whl: .layout.json
Common subdirectories: /Users/x/.pex/installed_wheels/d6149d42626fa7ae2943f479b1cf777bb6e174bb1a35163bed4ca9ee4cab922d/y_cdi_integration_testrunner-0.0.local0-py3-none-any.whl/cdi_integration_testrunner and whl/cdi_integration_testrunner
Common subdirectories: /Users/x/.pex/installed_wheels/d6149d42626fa7ae2943f479b1cf777bb6e174bb1a35163bed4ca9ee4cab922d/y_cdi_integration_testrunner-0.0.local0-py3-none-any.whl/y_cdi_integration_testrunner-0.0.local0.dist-info and whl/y_cdi_integration_testrunner-0.0.local0.dist-info
```

This `.layout.json` file:
```json
{
  "record_relpath": "y_cdi_integration_testrunner-0.0.local0.dist-info/RECORD",
  "stash_dir": ".prefix"
}
```


Where `x` and `y` are redacted data. 

This is for a fresh repo, so there's practically no code in it, yet, so if a reproduction repo is desirable it shouldn't be too difficult to setup.

I suspect it is the `.layout.json` that trips up the dist hash, but I'm not sure of what the fix needs to be.
Any details I've left out that would be helpful, let me know :)



Discussion:
Confirmed this works on pants 2.11.0.dev1 / PEX 2.1.67, prior to #1630 

Thanks for the detailed info @kaos. This is a feature-for-pants-only slated for destruction in #1284 so this makes a bit of sense. There is ~no test coverage IIRC. I'll fold this into the 2.1.73 release.
Ok, so this is the case of new Pex trying to add requirements from an old PEX. That old PEX is coming from the Pants local LMDB cache; so the problem can be worked around by either clearing that - a very heavy hammer, or tweaking something about the `python_distribution`; i.e.: add a comment or make a whitespace change to the files that make it up.

My fix on the Pex end will make this new / old combination work, but if you need to move on with your life before this gets rolled out through to a Pants release; those workarounds are available.
Hrm, diagnosis and workarounds above are untrue. I think this is a pure new Pex / new PEX issue.

Resolving patch:
diff --git a/pex/environment.py b/pex/environment.py
--- a/pex/environment.py
+++ b/pex/environment.py
@@ -254,7 +254,7 @@ def path(self):
         # type: () -> str
         return self._pex
 
-    def _load_internal_cache(self):
+    def iter_distributions(self):
         # type: () -> Iterator[FingerprintedDistribution]
         internal_cache = os.path.join(self._pex, self._pex_info.internal_cache)
         with TRACER.timed("Searching dependency cache: %s" % internal_cache, V=2):
@@ -499,7 +499,7 @@ def resolve(self):
     def resolve_dists(self, reqs):
         # type: (Iterable[Requirement]) -> Iterable[FingerprintedDistribution]
 
-        self._update_candidate_distributions(self._load_internal_cache())
+        self._update_candidate_distributions(self.iter_distributions())
 
         unresolved_reqs = OrderedDict()  # type: OrderedDict[Requirement, OrderedSet]
 
diff --git a/pex/pex_builder.py b/pex/pex_builder.py
--- a/pex/pex_builder.py
+++ b/pex/pex_builder.py
@@ -14,7 +14,6 @@
     atomic_directory,
     chmod_plus_x,
     is_pyc_temporary_file,
-    open_zip,
     safe_copy,
     safe_mkdir,
     safe_mkdtemp,
@@ -25,6 +24,7 @@
 from pex.compatibility import to_bytes
 from pex.compiler import Compiler
 from pex.enum import Enum
+from pex.environment import PEXEnvironment
 from pex.finders import get_entry_point_from_console_script, get_script_from_distributions
 from pex.interpreter import PythonInterpreter
 from pex.layout import Layout
@@ -298,29 +298,13 @@ def add_from_requirements_pex(self, pex):
         """
         self._ensure_unfrozen("Adding from pex")
         pex_info = PexInfo.from_pex(pex)
-
-        def add(location, dname, expected_dhash):
-            dhash = self._add_dist_dir(location, dname)
-            if dhash != expected_dhash:
-                raise self.InvalidDistribution(
-                    "Distribution {} at {} had hash {}, expected {}".format(
-                        dname, location, dhash, expected_dhash
-                    )
-                )
-            self._pex_info.add_distribution(dname, dhash)
-
-        if os.path.isfile(pex):
-            with open_zip(pex) as zf:
-                for dist_name, dist_hash in pex_info.distributions.items():
-                    internal_dist_path = "/".join([pex_info.internal_cache, dist_name])
-                    cached_location = os.path.join(pex_info.install_cache, dist_hash, dist_name)
-                    CacheHelper.cache_distribution(zf, internal_dist_path, cached_location)
-                    add(cached_location, dist_name, dist_hash)
-        else:
-            for dist_name, dist_hash in pex_info.distributions.items():
-                add(os.path.join(pex, pex_info.internal_cache, dist_name), dist_name, dist_hash)
-        for req in pex_info.requirements:
-            self._pex_info.add_requirement(req)
+        pex_environment = PEXEnvironment.mount(pex, pex_info=pex_info)
+        for fingerprinted_dist in pex_environment.iter_distributions():
+            self.add_distribution(
+                dist=fingerprinted_dist.distribution, fingerprint=fingerprinted_dist.fingerprint
+            )
+        for requirement in pex_info.requirements:
+            self.add_requirement(requirement)
 
     def set_executable(self, filename, env_filename=None):
         """Set the executable for this environment.


Test patch:
diff --git a/tests/integration/test_issue_1683.py b/tests/integration/test_issue_1683.py
new file mode 100644
--- /dev/null
+++ b/tests/integration/test_issue_1683.py
@@ -0,0 +1,66 @@
+# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+import os.path
+import subprocess
+from textwrap import dedent
+
+from colors import crossed, red
+
+from pex.common import safe_open
+from pex.testing import run_pex_command
+from pex.typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from typing import Any
+
+
+def test_requirements_pex(tmpdir):
+    # type: (Any) -> None
+
+    pex_root = os.path.join(str(tmpdir), "pex_root")
+    requirements_pex = os.path.join(str(tmpdir), "requirements.pex")
+    run_pex_command(
+        args=[
+            "--pex-root",
+            pex_root,
+            "--runtime-pex-root",
+            pex_root,
+            "ansicolors==1.1.8",
+            "-o",
+            requirements_pex,
+        ]
+    ).assert_success()
+
+    src_dir = os.path.join(str(tmpdir), "src")
+    with safe_open(os.path.join(src_dir, "exe.py"), "w") as fp:
+        fp.write(
+            dedent(
+                """\
+                from colors import crossed, red
+
+
+                print(red(crossed("Broken")))
+                """
+            )
+        )
+
+    app_pex = os.path.join(str(tmpdir), "app.pex")
+    run_pex_command(
+        args=[
+            "--pex-root",
+            pex_root,
+            "--runtime-pex-root",
+            pex_root,
+            "--requirements-pex",
+            requirements_pex,
+            "-D",
+            src_dir,
+            "-m",
+            "exe",
+            "-o",
+            app_pex,
+        ]
+    ).assert_success()
+
+    assert red(crossed("Broken")) == subprocess.check_output(args=[app_pex]).decode("utf-8").strip()