Repository: pypa/virtualenv

Issue:
20.0.2: unable to find system python2.7 on macos
```console
$ pip freeze --all
appdirs==1.4.3
distlib==0.3.0
filelock==3.0.12
importlib-metadata==1.5.0
pip==20.0.2
setuptools==45.2.0
six==1.14.0
virtualenv==20.0.2
wheel==0.34.2
zipp==2.2.0
```

```console
$ virtualenv vvv -p python2.7 -vvv
72 setup logging to NOTSET [DEBUG report:43]
77 find interpreter for spec PythonSpec(implementation=CPython, major=2, minor=7) [INFO builtin:44]
77 discover system for PythonInfo(spec=CPython3.7.2.final.0-64, exe=/private/tmp/venv/bin/python3, platform=darwin, version='3.7.2 (v3.7.2:9a3ffc0492, Dec 24 2018, 02:44:43) \n[Clang 6.0 (clang-600.0.57)]', encoding_fs_io=utf-8-UTF-8) in /Library/Frameworks/Python.framework/Versions/3.7 [DEBUG py_info:332]
79 filesystem is not case-sensitive [DEBUG info:28]
80 Attempting to acquire lock 4513505352 on /Users/asottile/Library/Application Support/virtualenv/py-info/20.0.2/fb7ec9939e66e8ed189e2fe772cae61fcc992510c23be22bd7dadeab5367beb4.lock [DEBUG filelock:270]
80 Lock 4513505352 acquired on /Users/asottile/Library/Application Support/virtualenv/py-info/20.0.2/fb7ec9939e66e8ed189e2fe772cae61fcc992510c23be22bd7dadeab5367beb4.lock [INFO filelock:274]
80 get interpreter info via cmd: /Library/Frameworks/Python.framework/Versions/3.7/python -s /private/tmp/venv/lib/python3.7/site-packages/virtualenv/discovery/py_info.py [DEBUG cached_py_info:107]
82 Attempting to release lock 4513505352 on /Users/asottile/Library/Application Support/virtualenv/py-info/20.0.2/fb7ec9939e66e8ed189e2fe772cae61fcc992510c23be22bd7dadeab5367beb4.lock [DEBUG filelock:315]
82 Lock 4513505352 released on /Users/asottile/Library/Application Support/virtualenv/py-info/20.0.2/fb7ec9939e66e8ed189e2fe772cae61fcc992510c23be22bd7dadeab5367beb4.lock [INFO filelock:318]
82 failed to query /Library/Frameworks/Python.framework/Versions/3.7/python with code 8 err: 'Exec format error' [INFO cached_py_info:39]
83 done in 80ms [INFO __main__:24]
83 RuntimeError: failed to detect cpython3.7.2-64|cpython3.7.2|cpython3.7-64|cpython3.7|cpython3-64|cpython3|cpython-64|cpython|python3.7.2-64|python3.7.2|python3.7-64|python3.7|python3-64|python3|python-64|python in /Library/Frameworks/Python.framework/Versions/3.7 [ERROR __main__:36]
```

How I got here:

```bash
virtualenv venv  # old virtualenv
venv/bin/pip install virtualenv --upgrade
```

I'm not exactly sure why it's trying to look up python3.7 here 🤔 

I can dig more into it as well, but figured I'd post this first 👍 


Discussion:
We always try with the system python of the host python first when searching for python environments, we assume we can always find this, but in this case seems not. Ideally I'd need the py info of the host python here to understand what's needed. Might need to add a py info command flag to allow to do this easily, but for now you'd need that file. Can you post here the py info locked up via that lock?
Can you post the --with-traceback output?
here's the python infos:

```console
$ venv/bin/python3 /private/tmp/venv/lib/python3.7/site-packages/virtualenv/discovery/py_info.py
{
  "platform": "darwin",
  "implementation": "CPython",
  "version_info": {
    "major": 3,
    "minor": 7,
    "micro": 2,
    "releaselevel": "final",
    "serial": 0
  },
  "architecture": 64,
  "version": "3.7.2 (v3.7.2:9a3ffc0492, Dec 24 2018, 02:44:43) \n[Clang 6.0 (clang-600.0.57)]",
  "os": "posix",
  "prefix": "/private/tmp/venv/bin/..",
  "base_prefix": "/private/tmp/venv/bin/..",
  "real_prefix": "/Library/Frameworks/Python.framework/Versions/3.7",
  "base_exec_prefix": "/private/tmp/venv/bin/..",
  "exec_prefix": "/private/tmp/venv/bin/..",
  "executable": "/private/tmp/venv/bin/python3",
  "original_executable": "/private/tmp/venv/bin/python3",
  "system_executable": null,
  "has_venv": true,
  "path": [
    "/private/tmp/venv/lib/python3.7/site-packages/virtualenv/discovery",
    "/private/tmp/venv/lib/python37.zip",
    "/private/tmp/venv/lib/python3.7",
    "/private/tmp/venv/lib/python3.7/lib-dynload",
    "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7",
    "/private/tmp/venv/lib/python3.7/site-packages"
  ],
  "file_system_encoding": "utf-8",
  "stdout_encoding": "UTF-8",
  "sysconfig_paths": {
    "stdlib": "{installed_base}/lib/python{py_version_short}",
    "platstdlib": "{platbase}/lib/python{py_version_short}",
    "purelib": "{base}/lib/python{py_version_short}/site-packages",
    "platlib": "{platbase}/lib/python{py_version_short}/site-packages",
    "include": "{installed_base}/include/python{py_version_short}{abiflags}",
    "scripts": "{base}/bin",
    "data": "{base}"
  },
  "sysconfig_vars": {
    "abiflags": "m",
    "base": "/private/tmp/venv",
    "py_version_short": "3.7",
    "platbase": "/private/tmp/venv",
    "installed_base": "/private/tmp/venv"
  },
  "distutils_install": {
    "purelib": "lib/python3.7/site-packages",
    "platlib": "lib/python3.7/site-packages",
    "headers": "include/python3.7m/UNKNOWN",
    "scripts": "bin",
    "data": ""
  },
  "system_stdlib": "/private/tmp/venv/lib/python3.7",
  "_creators": null
}
$ /usr/local/bin/python3 /private/tmp/venv/lib/python3.7/site-packages/virtualenv/discovery/py_info.py
{
  "platform": "darwin",
  "implementation": "CPython",
  "version_info": {
    "major": 3,
    "minor": 7,
    "micro": 6,
    "releaselevel": "final",
    "serial": 0
  },
  "architecture": 64,
  "version": "3.7.6 (default, Dec 30 2019, 19:38:28) \n[Clang 11.0.0 (clang-1100.0.33.16)]",
  "os": "posix",
  "prefix": "/usr/local/Cellar/python/3.7.6_1/Frameworks/Python.framework/Versions/3.7",
  "base_prefix": "/usr/local/Cellar/python/3.7.6_1/Frameworks/Python.framework/Versions/3.7",
  "real_prefix": null,
  "base_exec_prefix": "/usr/local/Cellar/python/3.7.6_1/Frameworks/Python.framework/Versions/3.7",
  "exec_prefix": "/usr/local/Cellar/python/3.7.6_1/Frameworks/Python.framework/Versions/3.7",
  "executable": "/usr/local/opt/python/bin/python3.7",
  "original_executable": "/usr/local/opt/python/bin/python3.7",
  "system_executable": "/usr/local/opt/python/bin/python3.7",
  "has_venv": true,
  "path": [
    "/private/tmp/venv/lib/python3.7/site-packages/virtualenv/discovery",
    "/usr/local/Cellar/python/3.7.6_1/Frameworks/Python.framework/Versions/3.7/lib/python37.zip",
    "/usr/local/Cellar/python/3.7.6_1/Frameworks/Python.framework/Versions/3.7/lib/python3.7",
    "/usr/local/Cellar/python/3.7.6_1/Frameworks/Python.framework/Versions/3.7/lib/python3.7/lib-dynload",
    "/usr/local/lib/python3.7/site-packages"
  ],
  "file_system_encoding": "utf-8",
  "stdout_encoding": "UTF-8",
  "sysconfig_paths": {
    "stdlib": "{installed_base}/lib/python{py_version_short}",
    "platstdlib": "{platbase}/lib/python{py_version_short}",
    "purelib": "{base}/lib/python{py_version_short}/site-packages",
    "platlib": "{platbase}/lib/python{py_version_short}/site-packages",
    "include": "{installed_base}/include/python{py_version_short}{abiflags}",
    "scripts": "{base}/bin",
    "data": "{base}"
  },
  "sysconfig_vars": {
    "py_version_short": "3.7",
    "platbase": "/usr/local/Cellar/python/3.7.6_1/Frameworks/Python.framework/Versions/3.7",
    "abiflags": "m",
    "base": "/usr/local/Cellar/python/3.7.6_1/Frameworks/Python.framework/Versions/3.7",
    "installed_base": "/usr/local/Cellar/python/3.7.6_1/Frameworks/Python.framework/Versions/3.7"
  },
  "distutils_install": {
    "purelib": "lib/python3.7/site-packages",
    "platlib": "lib/python3.7/site-packages",
    "headers": "include/python3.7m/UNKNOWN",
    "scripts": "bin",
    "data": ""
  },
  "system_stdlib": "/usr/local/Cellar/python/3.7.6_1/Frameworks/Python.framework/Versions/3.7/lib/python3.7",
  "_creators": null
}
$ python2.7 /private/tmp/venv/lib/python3.7/site-packages/virtualenv/discovery/py_info.py
{
  "base_exec_prefix": null, 
  "system_stdlib": "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7", 
  "has_venv": false, 
  "prefix": "/System/Library/Frameworks/Python.framework/Versions/2.7", 
  "stdout_encoding": "UTF-8", 
  "executable": "/System/Library/Frameworks/Python.framework/Versions/2.7/Resources/Python.app/Contents/MacOS/Python", 
  "implementation": "CPython", 
  "exec_prefix": "/System/Library/Frameworks/Python.framework/Versions/2.7", 
  "platform": "darwin", 
  "version": "2.7.16 (default, Oct 16 2019, 00:34:56) \n[GCC 4.2.1 Compatible Apple LLVM 10.0.1 (clang-1001.0.37.14)]", 
  "sysconfig_paths": {
    "platstdlib": "{platbase}/lib/python{py_version_short}", 
    "platlib": "{platbase}/lib/python{py_version_short}/site-packages", 
    "purelib": "{base}/lib/python{py_version_short}/site-packages", 
    "stdlib": "{base}/lib/python{py_version_short}", 
    "scripts": "{base}/bin", 
    "include": "{base}/include/python{py_version_short}", 
    "data": "{base}"
  }, 
  "base_prefix": null, 
  "_creators": null, 
  "file_system_encoding": "utf-8", 
  "version_info": {
    "major": 2, 
    "minor": 7, 
    "micro": 16, 
    "releaselevel": "final", 
    "serial": 0
  }, 
  "sysconfig_vars": {
    "base": "/System/Library/Frameworks/Python.framework/Versions/2.7", 
    "platbase": "/System/Library/Frameworks/Python.framework/Versions/2.7", 
    "py_version_short": "2.7"
  }, 
  "path": [
    "/private/tmp/venv/lib/python3.7/site-packages/virtualenv/discovery", 
    "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python27.zip", 
    "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7", 
    "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/plat-darwin", 
    "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/plat-mac", 
    "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/plat-mac/lib-scriptpackages", 
    "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/lib-tk", 
    "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/lib-old", 
    "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/lib-dynload", 
    "/Library/Python/2.7/site-packages", 
    "/System/Library/Frameworks/Python.framework/Versions/2.7/Extras/lib/python", 
    "/System/Library/Frameworks/Python.framework/Versions/2.7/Extras/lib/python/PyObjC"
  ], 
  "real_prefix": null, 
  "distutils_install": {
    "purelib": "lib/python2.7/site-packages", 
    "headers": "include/python2.7/UNKNOWN", 
    "platlib": "lib/python2.7/site-packages", 
    "data": "", 
    "scripts": "bin"
  }, 
  "architecture": 64, 
  "original_executable": "/System/Library/Frameworks/Python.framework/Versions/2.7/Resources/Python.app/Contents/MacOS/Python", 
  "os": "posix", 
  "system_executable": "/System/Library/Frameworks/Python.framework/Versions/2.7/Resources/Python.app/Contents/MacOS/Python"
}
```


and the traceback:

```console
$ virtualenv vvv -vvv --with-traceback
65 setup logging to NOTSET [DEBUG report:43]
71 find interpreter for spec PythonSpec(path=/private/tmp/venv/bin/python3) [INFO builtin:44]
71 discover system for PythonInfo(spec=CPython3.7.2.final.0-64, exe=/private/tmp/venv/bin/python3, platform=darwin, version='3.7.2 (v3.7.2:9a3ffc0492, Dec 24 2018, 02:44:43) \n[Clang 6.0 (clang-600.0.57)]', encoding_fs_io=utf-8-UTF-8) in /Library/Frameworks/Python.framework/Versions/3.7 [DEBUG py_info:332]
73 filesystem is not case-sensitive [DEBUG info:28]
74 Attempting to acquire lock 4525809448 on /Users/asottile/Library/Application Support/virtualenv/py-info/20.0.2/fb7ec9939e66e8ed189e2fe772cae61fcc992510c23be22bd7dadeab5367beb4.lock [DEBUG filelock:270]
74 Lock 4525809448 acquired on /Users/asottile/Library/Application Support/virtualenv/py-info/20.0.2/fb7ec9939e66e8ed189e2fe772cae61fcc992510c23be22bd7dadeab5367beb4.lock [INFO filelock:274]
74 get interpreter info via cmd: /Library/Frameworks/Python.framework/Versions/3.7/python -s /private/tmp/venv/lib/python3.7/site-packages/virtualenv/discovery/py_info.py [DEBUG cached_py_info:107]
76 Attempting to release lock 4525809448 on /Users/asottile/Library/Application Support/virtualenv/py-info/20.0.2/fb7ec9939e66e8ed189e2fe772cae61fcc992510c23be22bd7dadeab5367beb4.lock [DEBUG filelock:315]
76 Lock 4525809448 released on /Users/asottile/Library/Application Support/virtualenv/py-info/20.0.2/fb7ec9939e66e8ed189e2fe772cae61fcc992510c23be22bd7dadeab5367beb4.lock [INFO filelock:318]
76 failed to query /Library/Frameworks/Python.framework/Versions/3.7/python with code 8 err: 'Exec format error' [INFO cached_py_info:39]
77 done in 75ms [INFO __main__:24]
Traceback (most recent call last):
  File "/private/tmp/venv/bin/virtualenv", line 8, in <module>
    sys.exit(run_with_catch())
  File "/private/tmp/venv/lib/python3.7/site-packages/virtualenv/__main__.py", line 30, in run_with_catch
    run(args, options)
  File "/private/tmp/venv/lib/python3.7/site-packages/virtualenv/__main__.py", line 17, in run
    run_via_cli(args, options)
  File "/private/tmp/venv/lib/python3.7/site-packages/virtualenv/run/__init__.py", line 23, in run_via_cli
    session = session_via_cli(args, options)
  File "/private/tmp/venv/lib/python3.7/site-packages/virtualenv/run/__init__.py", line 36, in session_via_cli
    parser = build_parser(args, options)
  File "/private/tmp/venv/lib/python3.7/site-packages/virtualenv/run/__init__.py", line 56, in build_parser
    parser._interpreter = interpreter = discover.interpreter
  File "/private/tmp/venv/lib/python3.7/site-packages/virtualenv/discovery/discover.py", line 44, in interpreter
    self._interpreter = self.run()
  File "/private/tmp/venv/lib/python3.7/site-packages/virtualenv/discovery/builtin.py", line 33, in run
    return get_interpreter(self.python_spec)
  File "/private/tmp/venv/lib/python3.7/site-packages/virtualenv/discovery/builtin.py", line 46, in get_interpreter
    for interpreter, impl_must_match in propose_interpreters(spec):
  File "/private/tmp/venv/lib/python3.7/site-packages/virtualenv/discovery/builtin.py", line 60, in propose_interpreters
    yield PythonInfo.from_exe(spec.path), True
  File "/private/tmp/venv/lib/python3.7/site-packages/virtualenv/discovery/py_info.py", line 290, in from_exe
    proposed = proposed._resolve_to_system(proposed)
  File "/private/tmp/venv/lib/python3.7/site-packages/virtualenv/discovery/py_info.py", line 318, in _resolve_to_system
    target = target.discover_exe(prefix=prefix, exact=False)
  File "/private/tmp/venv/lib/python3.7/site-packages/virtualenv/discovery/py_info.py", line 374, in discover_exe
    "failed to detect {} in {}".format(what, os.pathsep.join(possible_folders))
RuntimeError: failed to detect cpython3.7.2-64|cpython3.7.2|cpython3.7-64|cpython3.7|cpython3-64|cpython3|cpython-64|cpython|python3.7.2-64|python3.7.2|python3.7-64|python3.7|python3-64|python3|python-64|python in /Library/Frameworks/Python.framework/Versions/3.7
```

Resolving patch:
diff --git a/src/virtualenv/__main__.py b/src/virtualenv/__main__.py
--- a/src/virtualenv/__main__.py
+++ b/src/virtualenv/__main__.py
@@ -5,6 +5,8 @@
 import sys
 from datetime import datetime
 
+import six
+
 
 def run(args=None, options=None):
     start = datetime.now()
@@ -14,14 +16,18 @@ def run(args=None, options=None):
     if args is None:
         args = sys.argv[1:]
     try:
-        run_via_cli(args, options)
+        session = run_via_cli(args, options)
+        logging.warning(
+            "created virtual environment in %.0fms %s with seeder %s",
+            (datetime.now() - start).total_seconds() * 1000,
+            six.ensure_text(str(session.creator)),
+            six.ensure_text(str(session.seeder)),
+        )
     except ProcessCallFailed as exception:
         print("subprocess call failed for {}".format(exception.cmd))
         print(exception.out, file=sys.stdout, end="")
         print(exception.err, file=sys.stderr, end="")
         raise SystemExit(exception.code)
-    finally:
-        logging.info("done in %.0fms", (datetime.now() - start).total_seconds() * 1000)
 
 
 def run_with_catch(args=None):
diff --git a/src/virtualenv/discovery/py_info.py b/src/virtualenv/discovery/py_info.py
--- a/src/virtualenv/discovery/py_info.py
+++ b/src/virtualenv/discovery/py_info.py
@@ -35,6 +35,9 @@ def __init__(self):
         def u(v):
             return v.decode("utf-8") if isinstance(v, bytes) else v
 
+        def abs_path(v):
+            return None if v is None else os.path.abspath(v)  # unroll relative elements from path (e.g. ..)
+
         # qualifies the python
         self.platform = u(sys.platform)
         self.implementation = u(platform.python_implementation())
@@ -49,16 +52,16 @@ def u(v):
         self.os = u(os.name)
 
         # information about the prefix - determines python home
-        self.prefix = u(getattr(sys, "prefix", None))  # prefix we think
-        self.base_prefix = u(getattr(sys, "base_prefix", None))  # venv
-        self.real_prefix = u(getattr(sys, "real_prefix", None))  # old virtualenv
+        self.prefix = u(abs_path(getattr(sys, "prefix", None)))  # prefix we think
+        self.base_prefix = u(abs_path(getattr(sys, "base_prefix", None)))  # venv
+        self.real_prefix = u(abs_path(getattr(sys, "real_prefix", None)))  # old virtualenv
 
         # information about the exec prefix - dynamic stdlib modules
-        self.base_exec_prefix = u(getattr(sys, "base_exec_prefix", None))
-        self.exec_prefix = u(getattr(sys, "exec_prefix", None))
+        self.base_exec_prefix = u(abs_path(getattr(sys, "base_exec_prefix", None)))
+        self.exec_prefix = u(abs_path(getattr(sys, "exec_prefix", None)))
 
-        self.executable = u(sys.executable)  # the executable we were invoked via
-        self.original_executable = u(self.executable)  # the executable as known by the interpreter
+        self.executable = u(abs_path(sys.executable))  # the executable we were invoked via
+        self.original_executable = u(abs_path(self.executable))  # the executable as known by the interpreter
         self.system_executable = self._fast_get_system_executable()  # the executable we are based of (if available)
 
         try:
@@ -413,10 +416,6 @@ def _find_possible_folders(self, inside_folder):
 
         # or at root level
         candidate_folder[inside_folder] = None
-        if self.executable.startswith(self.prefix):
-            binary_within = os.path.relpath(os.path.dirname(self.executable), self.prefix)
-            candidate_folder[os.path.join(inside_folder, binary_within)] = None
-
         return list(candidate_folder.keys())
 
     def _find_possible_exe_names(self):
diff --git a/src/virtualenv/run/__init__.py b/src/virtualenv/run/__init__.py
--- a/src/virtualenv/run/__init__.py
+++ b/src/virtualenv/run/__init__.py
@@ -2,8 +2,6 @@
 
 import logging
 
-import six
-
 from ..config.cli.parser import VirtualEnvConfigParser
 from ..report import LEVELS, setup_report
 from ..session import Session
@@ -22,12 +20,6 @@ def run_via_cli(args, options=None):
     """
     session = session_via_cli(args, options)
     session.run()
-
-    logging.warning(
-        "created virtual environment %s with seeder %s",
-        six.ensure_text(str(session.creator)),
-        six.ensure_text(str(session.seeder)),
-    )
     return session
 
 


Test patch: