Repository: pypa/virtualenv

Issue:
3.11 support
**Issue**

On python 3.11+, virtualenv returns an invalid `system_executable` for python binaries out of virtual environments made with `--copies`

See:

https://github.com/python-poetry/poetry/issues/6940#issuecomment-1304272130

Part of this is due to a change in 3.11 that changes the value of `sys._base_executable` to substitute the value of the "home" key from pyvenv.cfg.

https://bugs.python.org/issue46028
https://github.com/python/cpython/pull/29041
https://github.com/python/cpython/pull/30144

On it's own, Virtualenv is setting this to a path that does not actually include the python binary so the system_executable is "wrong"

```shell
root@vfazio2:/tmp/tmp.U2ZVtBvloJ# python3 -m virtualenv --copies .venv2
created virtual environment CPython3.11.0.final.0-64 in 1164ms
  creator CPython3Posix(dest=/tmp/tmp.U2ZVtBvloJ/.venv2, clear=False, no_vcs_ignore=False, global=False)
  seeder FromAppData(download=False, pip=bundle, setuptools=bundle, wheel=bundle, via=copy, app_data_dir=/root/.local/share/virtualenv)
    added seed packages: pip==22.3, setuptools==65.5.0, wheel==0.37.1
  activators BashActivator,CShellActivator,FishActivator,NushellActivator,PowerShellActivator,PythonActivator

root@vfazio2:/tmp/tmp.U2ZVtBvloJ# ./.venv2/bin/python3 /usr/local/lib/python3.11/site-packages/virtualenv/discovery/py_info.py | jq .system_executable
"/usr/local/python3"

root@vfazio2:/tmp/tmp.U2ZVtBvloJ# test -e /usr/local/python3; echo $?
1

root@vfazio2:/tmp/tmp.U2ZVtBvloJ# ./.venv2/bin/python /usr/local/lib/python3.11/site-packages/virtualenv/discovery/py_info.py | jq .system_executable
"/usr/local/python"
root@vfazio2:/tmp/tmp.U2ZVtBvloJ# test -e /usr/local/python ; echo $?
1

```


For virtual environments created from a venv parent, subsequent virtualenvs continue to return a binary that does not exist. Poetry is a good example:

```
root@vfazio2:/tmp/tmp.U2ZVtBvloJ# cat /opt/poetry/venv/pyvenv.cfg 
home = /usr/local/bin
include-system-site-packages = false
version = 3.11.0
executable = /usr/local/bin/python3.11
command = /usr/local/bin/python3 -m venv --copies --clear /opt/poetry/venv

root@vfazio2:/tmp/tmp.U2ZVtBvloJ# /opt/poetry/venv/bin/python /usr/local/lib/python3.11/site-packages/virtualenv/discovery/py_info.py | jq .system_executable
"/usr/local/bin/python"

root@vfazio2:/tmp/tmp.U2ZVtBvloJ# test -e /usr/local/bin/python; echo $?
1

```

This causes virtualenv to fail to create virtual environments due to not being able to find the system executable

```
  RuntimeError

  failed to query /usr/local/bin/python with code 2 err: 'No such file or directory'

  at /opt/poetry/venv/lib/python3.11/site-packages/virtualenv/discovery/cached_py_info.py:31 in from_exe
       27│     env = os.environ if env is None else env
       28│     result = _get_from_cache(cls, app_data, exe, env, ignore_cache=ignore_cache)
       29│     if isinstance(result, Exception):
       30│         if raise_on_error:
    →  31│             raise result
       32│         else:
       33│             logging.info("%s", result)
       34│         result = None
       35│     return result

```

Stack:
```
root@vfazio2:/tmp/tmp.U2ZVtBvloJ# /opt/poetry/venv/bin/python
Python 3.11.0 (main, Nov  2 2022, 16:46:02) [GCC 10.2.1 20210110] on linux
Type "help", "copyright", "credits" or "license" for more information.
>>> import virtualenv
>>> virtualenv.cli_run("tempvenv")
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/opt/poetry/venv/lib/python3.11/site-packages/virtualenv/run/__init__.py", line 28, in cli_run
    of_session = session_via_cli(args, options, setup_logging, env)
                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/poetry/venv/lib/python3.11/site-packages/virtualenv/run/__init__.py", line 46, in session_via_cli
    parser, elements = build_parser(args, options, setup_logging, env)
                       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/poetry/venv/lib/python3.11/site-packages/virtualenv/run/__init__.py", line 68, in build_parser
    parser._interpreter = interpreter = discover.interpreter
                                        ^^^^^^^^^^^^^^^^^^^^
  File "/opt/poetry/venv/lib/python3.11/site-packages/virtualenv/discovery/discover.py", line 38, in interpreter
    self._interpreter = self.run()
                        ^^^^^^^^^^
  File "/opt/poetry/venv/lib/python3.11/site-packages/virtualenv/discovery/builtin.py", line 44, in run
    result = get_interpreter(python_spec, self.try_first_with, self.app_data, self._env)
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/poetry/venv/lib/python3.11/site-packages/virtualenv/discovery/builtin.py", line 59, in get_interpreter
    for interpreter, impl_must_match in propose_interpreters(spec, try_first_with, app_data, env):
  File "/opt/poetry/venv/lib/python3.11/site-packages/virtualenv/discovery/builtin.py", line 90, in propose_interpreters
    yield PythonInfo.from_exe(os.path.abspath(spec.path), app_data, env=env), True
          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/poetry/venv/lib/python3.11/site-packages/virtualenv/discovery/py_info.py", line 372, in from_exe
    raise exception
  File "/opt/poetry/venv/lib/python3.11/site-packages/virtualenv/discovery/py_info.py", line 369, in from_exe
    proposed = proposed._resolve_to_system(app_data, proposed)
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/poetry/venv/lib/python3.11/site-packages/virtualenv/discovery/py_info.py", line 409, in _resolve_to_system
    target = cls.from_exe(target.system_executable, app_data)
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/poetry/venv/lib/python3.11/site-packages/virtualenv/discovery/py_info.py", line 365, in from_exe
    proposed = from_exe(cls, app_data, exe, env=env, raise_on_error=raise_on_error, ignore_cache=ignore_cache)
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/poetry/venv/lib/python3.11/site-packages/virtualenv/discovery/cached_py_info.py", line 31, in from_exe
    raise result
RuntimeError: failed to query /usr/local/bin/python with code 2 err: 'No such file or directory'
```

I've proposed a fix in upstream cpython to try to paper over this discrepancy between binaries available in a venv and what's available in the system install path. I don't expect that to move quickly. Virtualenv could, in the interim, catch this known problematic scenario and return a system python that does exist.


Discussion:
PR welcome.
I'm not totally sure virtualenv is doing it wrong seeing as how this has been consistent for previous 3.x versions up until 3.11 and the reworking of `Modules/getpath.py`

I'm not well versed enough in virtualenv to make real suggestions,.. I assume we're intentionally searching for the executable one level higher than current?

Resolving patch:
diff --git a/src/virtualenv/create/creator.py b/src/virtualenv/create/creator.py
--- a/src/virtualenv/create/creator.py
+++ b/src/virtualenv/create/creator.py
@@ -157,7 +157,7 @@ def run(self):
 
     def set_pyenv_cfg(self):
         self.pyenv_cfg.content = OrderedDict()
-        self.pyenv_cfg["home"] = self.interpreter.system_exec_prefix
+        self.pyenv_cfg["home"] = os.path.dirname(os.path.abspath(self.interpreter.system_executable))
         self.pyenv_cfg["implementation"] = self.interpreter.implementation
         self.pyenv_cfg["version_info"] = ".".join(str(i) for i in self.interpreter.version_info)
         self.pyenv_cfg["virtualenv"] = __version__


Test patch:
diff --git a/tests/unit/create/test_creator.py b/tests/unit/create/test_creator.py
--- a/tests/unit/create/test_creator.py
+++ b/tests/unit/create/test_creator.py
@@ -318,6 +318,28 @@ def test_prompt_set(tmp_path, creator, prompt):
             assert cfg["prompt"] == actual_prompt
 
 
+@pytest.mark.parametrize("creator", CURRENT_CREATORS)
+def test_home_path_is_exe_parent(tmp_path, creator):
+    cmd = [str(tmp_path), "--seeder", "app-data", "--without-pip", "--creator", creator]
+
+    result = cli_run(cmd)
+    cfg = PyEnvCfg.from_file(result.creator.pyenv_cfg.path)
+
+    # Cannot assume "home" path is a specific value as path resolution may change
+    # between versions (symlinks, framework paths, etc) but we can check that a
+    # python executable is present from the configured path per PEP 405
+    if sys.platform == "win32":
+        exes = ("python.exe",)
+    else:
+        exes = (
+            "python",
+            f"python{sys.version_info.major}",
+            f"python{sys.version_info.major}.{sys.version_info.minor}",
+        )
+
+    assert any(os.path.exists(os.path.join(cfg["home"], exe)) for exe in exes)
+
+
 @pytest.mark.slow()
 @pytest.mark.usefixtures("current_fastest")
 def test_cross_major(cross_python, coverage_env, tmp_path, session_app_data):