Repository: pantsbuild/pants

Issue:
Resolve failure on MacOS after upgrading from applying mac os patches.
With pants 2.0.0, after an OSX upgrade from `10.14` to `10.15`, previously built pexes are unusable. Clearing the LMDB cache works around the issue.

```
Failed to execute PEX file. Needed macosx_10_15_x86_64-cp-38-cp38 compatible dependencies for:
 1: pyyaml
    But this pex only contains:
      PyYAML-5.3.1-cp38-cp38-macosx_10_14_6_x86_64.whl
 2: pyyaml; python_version != "3.4"
    Required by:
      cfn-lint==0.39.0; python_version != "3.0.*" and python_version != "3.1.*" and python_version != "3.2.*" and python_version != "3.3.*" and python_version >= "2.7"
    But this pex only contains:
      PyYAML-5.3.1-cp38-cp38-macosx_10_14_6_x86_64.whl
 3: pyyaml>=3.12
    Required by:
      kubernetes==12.0.0
    But this pex only contains:
      PyYAML-5.3.1-cp38-cp38-macosx_10_14_6_x86_64.whl
 4: PyYAML>=5.1
    Required by:
      moto==1.3.16
    But this pex only contains:
      PyYAML-5.3.1-cp38-cp38-macosx_10_14_6_x86_64.whl
 5: PyYAML<5.4,>=5.3.1
    Required by:
      pantsbuild.pants==2.0.0
    But this pex only contains:
      PyYAML-5.3.1-cp38-cp38-macosx_10_14_6_x86_64.whl
```


Discussion:
Note that `PyYAML` is an sdist, so I suspect it was built when Asher was using 14.6, then wasn't improperly invalidated when upgrading.
This probably should move over to a Pex issue, but before that I could use some help.

This code in Pex's vendord `packaging` generates the `sys_tags()` for macOS:
https://github.com/pantsbuild/pex/blob/2a8b950284d4a8be5eb2bcf9dcb63e2bc5f60fe4/pex/vendor/_vendored/packaging/packaging/tags.py#L243-L261

That should include, for a CPython 3.8 interpreter on OSX 10.15: 10.15, 10.14, 10.13, .... etc in which case the error in the original post should not occur. @asherf can you provide the contents of `~/.pex/interpreters/` and `~/.cache/pants/named_caches/pex_root/interpreters`? Those directories should contain `INTERP-INFO` json files which should contain the list of supported tags for each interpreter like so:
```
$ jq '{binary:.binary, tags:.supported_tags}' /home/jsirois/.pex/interpreters/ece31615e928edaa688468dab817c8110a0f5044/1b67864221a28d5d97b038c0d801b1240ae6f252/usr.bin.python3.6/INTERP-INFO
{
  "binary": "/usr/bin/python3.6",
  "tags": [
    [
      "cp36",
      "cp36m",
      "manylinux2014_x86_64"
    ],
    [
      "cp36",
      "cp36m",
      "manylinux2010_x86_64"
    ],
    [
      "cp36",
      "cp36m",
      "manylinux1_x86_64"
    ],
    [
      "cp36",
      "cp36m",
      "linux_x86_64"
    ],
    [
      "cp36",
      "abi3",
      "manylinux2014_x86_64"
    ],
    [
      "cp36",
      "abi3",
      "manylinux2010_x86_64"
    ],
    [
      "cp36",
      "abi3",
      "manylinux1_x86_64"
    ],
    [
      "cp36",
      "abi3",
      "linux_x86_64"
    ],
    [
      "cp36",
      "none",
      "manylinux2014_x86_64"
    ],
    [
      "cp36",
      "none",
      "manylinux2010_x86_64"
    ],
    [
      "cp36",
      "none",
      "manylinux1_x86_64"
    ],
    [
      "cp36",
      "none",
      "linux_x86_64"
    ],
    [
      "cp35",
      "abi3",
      "manylinux2014_x86_64"
    ],
    [
      "cp35",
      "abi3",
      "manylinux2010_x86_64"
    ],
    [
      "cp35",
      "abi3",
      "manylinux1_x86_64"
    ],
    [
      "cp35",
      "abi3",
      "linux_x86_64"
    ],
    [
      "cp34",
      "abi3",
      "manylinux2014_x86_64"
    ],
    [
      "cp34",
      "abi3",
      "manylinux2010_x86_64"
    ],
    [
      "cp34",
      "abi3",
      "manylinux1_x86_64"
    ],
    [
      "cp34",
      "abi3",
      "linux_x86_64"
    ],
    [
      "cp33",
      "abi3",
      "manylinux2014_x86_64"
    ],
    [
      "cp33",
      "abi3",
      "manylinux2010_x86_64"
    ],
    [
      "cp33",
      "abi3",
      "manylinux1_x86_64"
    ],
    [
      "cp33",
      "abi3",
      "linux_x86_64"
    ],
    [
      "cp32",
      "abi3",
      "manylinux2014_x86_64"
    ],
    [
      "cp32",
      "abi3",
      "manylinux2010_x86_64"
    ],
    [
      "cp32",
      "abi3",
      "manylinux1_x86_64"
    ],
    [
      "cp32",
      "abi3",
      "linux_x86_64"
    ],
    [
      "py36",
      "none",
      "manylinux2014_x86_64"
    ],
    [
      "py36",
      "none",
      "manylinux2010_x86_64"
    ],
    [
      "py36",
      "none",
      "manylinux1_x86_64"
    ],
    [
      "py36",
      "none",
      "linux_x86_64"
    ],
    [
      "py3",
      "none",
      "manylinux2014_x86_64"
    ],
    [
      "py3",
      "none",
      "manylinux2010_x86_64"
    ],
    [
      "py3",
      "none",
      "manylinux1_x86_64"
    ],
    [
      "py3",
      "none",
      "linux_x86_64"
    ],
    [
      "py35",
      "none",
      "manylinux2014_x86_64"
    ],
    [
      "py35",
      "none",
      "manylinux2010_x86_64"
    ],
    [
      "py35",
      "none",
      "manylinux1_x86_64"
    ],
    [
      "py35",
      "none",
      "linux_x86_64"
    ],
    [
      "py34",
      "none",
      "manylinux2014_x86_64"
    ],
    [
      "py34",
      "none",
      "manylinux2010_x86_64"
    ],
    [
      "py34",
      "none",
      "manylinux1_x86_64"
    ],
    [
      "py34",
      "none",
      "linux_x86_64"
    ],
    [
      "py33",
      "none",
      "manylinux2014_x86_64"
    ],
    [
      "py33",
      "none",
      "manylinux2010_x86_64"
    ],
    [
      "py33",
      "none",
      "manylinux1_x86_64"
    ],
    [
      "py33",
      "none",
      "linux_x86_64"
    ],
    [
      "py32",
      "none",
      "manylinux2014_x86_64"
    ],
    [
      "py32",
      "none",
      "manylinux2010_x86_64"
    ],
    [
      "py32",
      "none",
      "manylinux1_x86_64"
    ],
    [
      "py32",
      "none",
      "linux_x86_64"
    ],
    [
      "py31",
      "none",
      "manylinux2014_x86_64"
    ],
    [
      "py31",
      "none",
      "manylinux2010_x86_64"
    ],
    [
      "py31",
      "none",
      "manylinux1_x86_64"
    ],
    [
      "py31",
      "none",
      "linux_x86_64"
    ],
    [
      "py30",
      "none",
      "manylinux2014_x86_64"
    ],
    [
      "py30",
      "none",
      "manylinux2010_x86_64"
    ],
    [
      "py30",
      "none",
      "manylinux1_x86_64"
    ],
    [
      "py30",
      "none",
      "linux_x86_64"
    ],
    [
      "cp36",
      "none",
      "any"
    ],
    [
      "py36",
      "none",
      "any"
    ],
    [
      "py3",
      "none",
      "any"
    ],
    [
      "py35",
      "none",
      "any"
    ],
    [
      "py34",
      "none",
      "any"
    ],
    [
      "py33",
      "none",
      "any"
    ],
    [
      "py32",
      "none",
      "any"
    ],
    [
      "py31",
      "none",
      "any"
    ],
    [
      "py30",
      "none",
      "any"
    ]
  ]
}
```
@jsirois : My superficial guess on this issue was that Pants had built a PEX previously, and was then hitting the Pants process cache post-OSX upgrade to try and use the previously built PEX. The OS tag won't end up in Pants' process cache key, because the `@rules` aren't aware of it, and aren't passing it to PEX.
My point above is a that a PyYAML-5.3.1-cp38-cp38-macosx_10_14_6_x86_64.whl wheel _is_ compatible with 10.15. So the pre-existing PEX should have been compatible post upgrade save for what appears to be a bug in the list of tags compatible with the interpreter. That's the data I've asked @asher for. IOW whether or not we wish to have the OSX upgrade invalidate Pants caches, the PEX built under 10.14 should work just fine under 10.15.
@asherf can you provide any of the INTERP-INFO files in `~/.pex/interpreters/` and `~/.cache/pants/named_caches/pex_root/interpreters/`? Without those this will be near impossible for me to debug.
Do we think that this is still an issue? We haven't had a repro across the Big Sur boundary, so perhaps? Closing for now, but please reopen if we have reason to believe it's still out there.
Seeing something maybe similar over in https://github.com/pantsbuild/pants/runs/2196133667?check_suite_focus=true#step:10:1743

The key similarity is - upon closer inspection - a crazy wheel tag: `cp38-cp38-macosx_10_14_6_x86_64.whl`
Somehow the `10_14_6` escaped me in the original analysis. That is ... not what I or `packaging.tags` is expecting at least. You'd expect `10_14`.
OK - there is a way around these badly configured macOS Python interpreters without simply ignoring them - which would be bad when a user only has that Python installed:
Exporting `_PYTHON_HOST_PLATFORM` with the correct platform; e.g.: `macosx-10.14-x86-64` instead of `macosx-10.14.6-x86-64`, gets `sysconfig.get_platform()` reporting the corrected platform which is deep enough in the bowels for distutils to pick it up and thus wheel, etc. It looks like this undocumented backdoor exists in all CPythons Pex supports:
+ https://github.com/python/cpython/blob/v2.7.18/Lib/sysconfig.py#L567-L569
... through ...
+ https://github.com/python/cpython/blob/v3.9.2/Lib/sysconfig.py#L652-L654

Resolving patch:
diff --git a/src/python/pants/backend/python/util_rules/pex.py b/src/python/pants/backend/python/util_rules/pex.py
--- a/src/python/pants/backend/python/util_rules/pex.py
+++ b/src/python/pants/backend/python/util_rules/pex.py
@@ -8,8 +8,6 @@
 import itertools
 import json
 import logging
-import os
-import re
 import shlex
 from collections import defaultdict
 from dataclasses import dataclass
@@ -842,33 +840,17 @@ async def create_venv_pex(
 
     pex_request = request.pex_request
     seeded_venv_request = dataclasses.replace(
-        pex_request, additional_args=pex_request.additional_args + ("--venv", "--seed")
+        pex_request, additional_args=pex_request.additional_args + ("--venv", "--seed", "verbose")
     )
     venv_pex_result = await Get(BuildPexResult, PexRequest, seeded_venv_request)
-    # Pex --seed mode outputs the path of the PEX executable. In the --venv case this is the `pex`
-    # script in the venv root directory.
-    abs_venv_dir = PurePath(venv_pex_result.result.stdout.decode().strip()).parent
-
-    # TODO(John Sirois): Instead of performing this calculation, which uses knowledge of the
-    #  PEX_ROOT layout (It assumes a `venvs` dir), rely on verbose seeding or a pex tool to get
-    #  the relevant PEX_ROOT / venv pex directory combination in a single Process execution so
-    #  we can relpath them.
-
-    pex_root = pex_environment.pex_root
-    if pex_root.is_absolute():
-        if not os.path.commonprefix([pex_root, abs_venv_dir]) == str(pex_root):
-            raise AssertionError(
-                f"The PEX_ROOT is absolute {pex_root} but does not match the venv pex "
-                f"{abs_venv_dir}."
-            )
-        venv_dir = abs_venv_dir.relative_to(pex_root)
-    else:
-        match = re.search(r"/(?P<venv_dir>venvs/.*)$", str(abs_venv_dir))
-        if match is None:
-            raise AssertionError(
-                f"Failed to find PEX_ROOT {pex_root} in venv_pex path {abs_venv_dir}."
-            )
-        venv_dir = pex_root / match["venv_dir"]
+    # Pex verbose --seed mode outputs the absolute path of the PEX executable as well as the
+    # absolute path of the PEX_ROOT.  In the --venv case this is the `pex` script in the venv root
+    # directory.
+    seed_info = json.loads(venv_pex_result.result.stdout.decode())
+    abs_pex_root = PurePath(seed_info["pex_root"])
+    abs_pex_path = PurePath(seed_info["pex"])
+    venv_rel_dir = abs_pex_path.relative_to(abs_pex_root).parent
+    venv_dir = pex_environment.pex_root / venv_rel_dir
 
     venv_script_writer = VenvScriptWriter(pex=venv_pex_result.create_pex(), venv_dir=venv_dir)
     pex = venv_script_writer.exe(bash, pex_environment)
diff --git a/src/python/pants/backend/python/util_rules/pex_cli.py b/src/python/pants/backend/python/util_rules/pex_cli.py
--- a/src/python/pants/backend/python/util_rules/pex_cli.py
+++ b/src/python/pants/backend/python/util_rules/pex_cli.py
@@ -37,7 +37,7 @@ class PexBinary(TemplatedExternalTool):
     name = "pex"
     help = "The PEX (Python EXecutable) tool (https://github.com/pantsbuild/pex)."
 
-    default_version = "v2.1.35"
+    default_version = "v2.1.36"
     default_url_template = "https://github.com/pantsbuild/pex/releases/download/{version}/pex"
 
     @classproperty
@@ -47,8 +47,8 @@ def default_known_versions(cls):
                 (
                     cls.default_version,
                     plat,
-                    "3b1c97ebd79a8b650c81f0afbdbefef17c3d1a128219a8e95b5f35c2c9dfd532",
-                    "3600775",
+                    "b93c015325c3d4cfa0c4e6b8ec8967edeb5fdac909adc8f6cb09b09c14055d93",
+                    "3603974",
                 )
             )
             for plat in ["darwin", "linux"]


Test patch: