Repository: pypa/virtualenv

Issue:
conda commands as Python3 subprocesses inside a virtualenv fail under macOS
**Issue**

Describe what's the expected behaviour and what you're observing.

Running any conda command as a Python3 subprocess inside of a virtualenv fails under macOS with `ModuleNotFoundError: No module named 'conda'`. This works fine under Linux or when using Python2 or virtualenv <20.
The same works fine also before activating and after deactivating the virtualenv.

Tested on the latest virtualenv 20.0.9.

See https://travis-ci.org/nsoranzo/bioblend/builds/660456892 for how to reproduce on TravisCI and example of success and failure of the same commands under Linux and macOS respectively.

**Environment**

Provide at least:
- OS: macOS
- ``pip list`` of the host python where ``virtualenv`` is installed:

  ```console
  Package    Version
  ---------- -------
  GDAL       2.4.2  
  mercurial  5.2    
  numpy      1.17.4 
  pip        19.3.1 
  protobuf   3.11.1 
  setuptools 41.6.0 
  six        1.13.0 
  wheel      0.33.6 
  ```

**Output of the virtual environment creation**

Make sure to run the creation with `-vvv --with-traceback`:

```console
152 setup logging to NOTSET [DEBUG report:43]
152 created app data folder /Users/travis/Library/Application Support/virtualenv [DEBUG app_data:52]
165 find interpreter for spec PythonSpec(implementation=CPython, major=3) [INFO builtin:44]
166 proposed PythonInfo(spec=CPython3.7.5.final.0-64, exe=/usr/local/opt/python/bin/python3.7, platform=darwin, version='3.7.5 (default, Nov  1 2019, 02:16:38) \n[Clang 10.0.0 (clang-1000.11.45.5)]', encoding_fs_io=utf-8-UTF-8) [INFO builtin:50]
166 accepted PythonInfo(spec=CPython3.7.5.final.0-64, exe=/usr/local/opt/python/bin/python3.7, platform=darwin, version='3.7.5 (default, Nov  1 2019, 02:16:38) \n[Clang 10.0.0 (clang-1000.11.45.5)]', encoding_fs_io=utf-8-UTF-8) [DEBUG builtin:52]
173 filesystem is not case-sensitive [DEBUG info:28]
278 create virtual environment via CPython3Posix(dest=/Users/travis/build/nsoranzo/bioblend/.venv, clear=False, global=False) [INFO session:52]
278 create folder /Users/travis/build/nsoranzo/bioblend/.venv/bin [DEBUG _sync:21]
278 create folder /Users/travis/build/nsoranzo/bioblend/.venv/lib/python3.7/site-packages [DEBUG _sync:21]
279 write /Users/travis/build/nsoranzo/bioblend/.venv/pyvenv.cfg [DEBUG pyenv_cfg:34]
279 	home = /usr/local/Cellar/python/3.7.5/Frameworks/Python.framework/Versions/3.7 [DEBUG pyenv_cfg:38]
279 	implementation = CPython [DEBUG pyenv_cfg:38]
279 	version_info = 3.7.5.final.0 [DEBUG pyenv_cfg:38]
279 	virtualenv = 20.0.9 [DEBUG pyenv_cfg:38]
280 	include-system-site-packages = false [DEBUG pyenv_cfg:38]
280 	base-prefix = /usr/local/Cellar/python/3.7.5/Frameworks/Python.framework/Versions/3.7 [DEBUG pyenv_cfg:38]
280 	base-exec-prefix = /usr/local/Cellar/python/3.7.5/Frameworks/Python.framework/Versions/3.7 [DEBUG pyenv_cfg:38]
280 	base-executable = /usr/local/opt/python/bin/python3.7 [DEBUG pyenv_cfg:38]
280 symlink /usr/local/opt/python/bin/python3.7 to /Users/travis/build/nsoranzo/bioblend/.venv/bin/python [DEBUG _sync:40]
282 create /Users/travis/build/nsoranzo/bioblend/.venv/lib/python3.7/site-packages/_virtualenv.py [DEBUG api:60]
283 create virtualenv import hook file /Users/travis/build/nsoranzo/bioblend/.venv/lib/python3.7/site-packages/_virtualenv.pth [DEBUG api:63]
283 ============================== target debug ============================== [DEBUG session:54]
284 debug via /Users/travis/build/nsoranzo/bioblend/.venv/bin/python /usr/local/lib/python3.7/site-packages/virtualenv/create/debug.py [DEBUG creator:182]
283 {
  "sys": {
    "executable": "/Users/travis/build/nsoranzo/bioblend/.venv/bin/python",
    "_base_executable": "/Users/travis/build/nsoranzo/bioblend/.venv/bin/python",
    "prefix": "/Users/travis/build/nsoranzo/bioblend/.venv",
    "base_prefix": "/usr/local/opt/python/bin/../Frameworks/Python.framework/Versions/3.7",
    "real_prefix": null,
    "exec_prefix": "/Users/travis/build/nsoranzo/bioblend/.venv",
    "base_exec_prefix": "/usr/local/opt/python/bin/../Frameworks/Python.framework/Versions/3.7",
    "path": [
      "/usr/local/opt/python/Frameworks/Python.framework/Versions/3.7/lib/python37.zip",
      "/usr/local/opt/python/Frameworks/Python.framework/Versions/3.7/lib/python3.7",
      "/usr/local/opt/python/Frameworks/Python.framework/Versions/3.7/lib/python3.7/lib-dynload",
      "/Users/travis/build/nsoranzo/bioblend/.venv/lib/python3.7/site-packages"
    ],
    "meta_path": [
      "<class '_virtualenv._Finder'>",
      "<class '_frozen_importlib.BuiltinImporter'>",
      "<class '_frozen_importlib.FrozenImporter'>",
      "<class '_frozen_importlib_external.PathFinder'>"
    ],
    "fs_encoding": "utf-8",
    "io_encoding": "UTF-8"
  },
  "version": "3.7.5 (default, Nov  1 2019, 02:16:38) \n[Clang 10.0.0 (clang-1000.11.45.5)]",
  "os": "<module 'os' from '/usr/local/opt/python/bin/../Frameworks/Python.framework/Versions/3.7/lib/python3.7/os.py'>",
  "site": "<module 'site' from '/usr/local/opt/python/bin/../Frameworks/Python.framework/Versions/3.7/lib/python3.7/site.py'>",
  "datetime": "<module 'datetime' from '/usr/local/opt/python/Frameworks/Python.framework/Versions/3.7/lib/python3.7/datetime.py'>",
  "math": "<module 'math' from '/usr/local/opt/python/Frameworks/Python.framework/Versions/3.7/lib/python3.7/lib-dynload/math.cpython-37m-darwin.so'>",
  "json": "<module 'json' from '/usr/local/opt/python/Frameworks/Python.framework/Versions/3.7/lib/python3.7/json/__init__.py'>"
} [DEBUG session:55]
354 add seed packages via FromAppData(download=False, pip=latest, setuptools=latest, wheel=latest, via=copy, app_data_dir=/Users/travis/Library/Application Support/virtualenv/seed-app-data/v1) [INFO session:59]
355 Attempting to acquire lock 4344232784 on /Users/travis/Library/Application Support/virtualenv/seed-app-data/v1/3.7/wheels.lock [DEBUG filelock:270]
356 Lock 4344232784 acquired on /Users/travis/Library/Application Support/virtualenv/seed-app-data/v1/3.7/wheels.lock [INFO filelock:274]
357 get bundled wheel /usr/local/lib/python3.7/site-packages/virtualenv/seed/embed/wheels/pip-20.0.2-py2.py3-none-any.whl [DEBUG acquire:51]
357 get bundled wheel /usr/local/lib/python3.7/site-packages/virtualenv/seed/embed/wheels/setuptools-45.3.0-py3-none-any.whl [DEBUG acquire:51]
358 get bundled wheel /usr/local/lib/python3.7/site-packages/virtualenv/seed/embed/wheels/wheel-0.34.2-py2.py3-none-any.whl [DEBUG acquire:51]
373 install wheel from wheel /Users/travis/Library/Application Support/virtualenv/seed-app-data/v1/3.7/wheels/wheel-0.34.2-py2.py3-none-any.whl via CopyPipInstall [DEBUG via_app_data:46]
373 build install image to /Users/travis/Library/Application Support/virtualenv/seed-app-data/v1/3.7/image/CopyPipInstall/wheel-0.34.2-py2.py3-none-any of wheel-0.34.2-py2.py3-none-any.whl [DEBUG base:57]
374 install setuptools from wheel /Users/travis/Library/Application Support/virtualenv/seed-app-data/v1/3.7/wheels/setuptools-45.3.0-py3-none-any.whl via CopyPipInstall [DEBUG via_app_data:46]
375 build install image to /Users/travis/Library/Application Support/virtualenv/seed-app-data/v1/3.7/image/CopyPipInstall/setuptools-45.3.0-py3-none-any of setuptools-45.3.0-py3-none-any.whl [DEBUG base:57]
375 install pip from wheel /Users/travis/Library/Application Support/virtualenv/seed-app-data/v1/3.7/wheels/pip-20.0.2-py2.py3-none-any.whl via CopyPipInstall [DEBUG via_app_data:46]
376 build install image to /Users/travis/Library/Application Support/virtualenv/seed-app-data/v1/3.7/image/CopyPipInstall/pip-20.0.2-py2.py3-none-any of pip-20.0.2-py2.py3-none-any.whl [DEBUG base:57]
418 changing mode of /var/folders/nz/vv4_9tw56nv9k3tkvyszvwg80000gn/T/tmp4niwmfd9/wheel-3.7 to 755 [INFO util:566]
419 changing mode of /var/folders/nz/vv4_9tw56nv9k3tkvyszvwg80000gn/T/tmp4niwmfd9/wheel to 755 [INFO util:566]
421 changing mode of /var/folders/nz/vv4_9tw56nv9k3tkvyszvwg80000gn/T/tmp4niwmfd9/wheel3 to 755 [INFO util:566]
432 copy directory /Users/travis/Library/Application Support/virtualenv/seed-app-data/v1/3.7/image/CopyPipInstall/wheel-0.34.2-py2.py3-none-any/wheel to /Users/travis/build/nsoranzo/bioblend/.venv/lib/python3.7/site-packages/wheel [DEBUG _sync:48]
466 copy directory /Users/travis/Library/Application Support/virtualenv/seed-app-data/v1/3.7/image/CopyPipInstall/wheel-0.34.2-py2.py3-none-any/wheel-0.34.2.dist-info to /Users/travis/build/nsoranzo/bioblend/.venv/lib/python3.7/site-packages/wheel-0.34.2.dist-info [DEBUG _sync:48]
474 copy /Users/travis/Library/Application Support/virtualenv/seed-app-data/v1/3.7/image/CopyPipInstall/wheel-0.34.2-py2.py3-none-any/wheel-0.34.2.virtualenv to /Users/travis/build/nsoranzo/bioblend/.venv/lib/python3.7/site-packages/wheel-0.34.2.virtualenv [DEBUG _sync:48]
475 changing mode of /Users/travis/build/nsoranzo/bioblend/.venv/bin/wheel-3.7 to 755 [INFO util:566]
475 changing mode of /Users/travis/build/nsoranzo/bioblend/.venv/bin/wheel to 755 [INFO util:566]
476 changing mode of /Users/travis/build/nsoranzo/bioblend/.venv/bin/wheel3 to 755 [INFO util:566]
476 generated console scripts wheel3 wheel-3.7 wheel [DEBUG base:53]
542 changing mode of /var/folders/nz/vv4_9tw56nv9k3tkvyszvwg80000gn/T/tmpdfedv6sv/easy_install to 755 [INFO util:566]
543 changing mode of /var/folders/nz/vv4_9tw56nv9k3tkvyszvwg80000gn/T/tmpdfedv6sv/easy_install3 to 755 [INFO util:566]
544 changing mode of /var/folders/nz/vv4_9tw56nv9k3tkvyszvwg80000gn/T/tmpdfedv6sv/easy_install-3.7 to 755 [INFO util:566]
563 copy /Users/travis/Library/Application Support/virtualenv/seed-app-data/v1/3.7/image/CopyPipInstall/setuptools-45.3.0-py3-none-any/easy_install.py to /Users/travis/build/nsoranzo/bioblend/.venv/lib/python3.7/site-packages/easy_install.py [DEBUG _sync:48]
564 copy directory /Users/travis/Library/Application Support/virtualenv/seed-app-data/v1/3.7/image/CopyPipInstall/setuptools-45.3.0-py3-none-any/pkg_resources to /Users/travis/build/nsoranzo/bioblend/.venv/lib/python3.7/site-packages/pkg_resources [DEBUG _sync:48]
584 copy directory /Users/travis/Library/Application Support/virtualenv/seed-app-data/v1/3.7/image/CopyPipInstall/setuptools-45.3.0-py3-none-any/setuptools to /Users/travis/build/nsoranzo/bioblend/.venv/lib/python3.7/site-packages/setuptools [DEBUG _sync:48]
655 copy directory /Users/travis/Library/Application Support/virtualenv/seed-app-data/v1/3.7/image/CopyPipInstall/setuptools-45.3.0-py3-none-any/setuptools-45.3.0.dist-info to /Users/travis/build/nsoranzo/bioblend/.venv/lib/python3.7/site-packages/setuptools-45.3.0.dist-info [DEBUG _sync:48]
667 copy /Users/travis/Library/Application Support/virtualenv/seed-app-data/v1/3.7/image/CopyPipInstall/setuptools-45.3.0-py3-none-any/setuptools-45.3.0.virtualenv to /Users/travis/build/nsoranzo/bioblend/.venv/lib/python3.7/site-packages/setuptools-45.3.0.virtualenv [DEBUG _sync:48]
668 changing mode of /Users/travis/build/nsoranzo/bioblend/.venv/bin/easy_install to 755 [INFO util:566]
669 changing mode of /Users/travis/build/nsoranzo/bioblend/.venv/bin/easy_install3 to 755 [INFO util:566]
670 changing mode of /Users/travis/build/nsoranzo/bioblend/.venv/bin/easy_install-3.7 to 755 [INFO util:566]
671 generated console scripts easy_install3 easy_install-3.7 easy_install [DEBUG base:53]
746 changing mode of /var/folders/nz/vv4_9tw56nv9k3tkvyszvwg80000gn/T/tmp9zqbno9l/pip to 755 [INFO util:566]
746 changing mode of /var/folders/nz/vv4_9tw56nv9k3tkvyszvwg80000gn/T/tmp9zqbno9l/pip3 to 755 [INFO util:566]
747 changing mode of /var/folders/nz/vv4_9tw56nv9k3tkvyszvwg80000gn/T/tmp9zqbno9l/pip-3.7 to 755 [INFO util:566]
781 copy directory /Users/travis/Library/Application Support/virtualenv/seed-app-data/v1/3.7/image/CopyPipInstall/pip-20.0.2-py2.py3-none-any/pip to /Users/travis/build/nsoranzo/bioblend/.venv/lib/python3.7/site-packages/pip [DEBUG _sync:48]
969 copy directory /Users/travis/Library/Application Support/virtualenv/seed-app-data/v1/3.7/image/CopyPipInstall/pip-20.0.2-py2.py3-none-any/pip-20.0.2.dist-info to /Users/travis/build/nsoranzo/bioblend/.venv/lib/python3.7/site-packages/pip-20.0.2.dist-info [DEBUG _sync:48]
972 copy /Users/travis/Library/Application Support/virtualenv/seed-app-data/v1/3.7/image/CopyPipInstall/pip-20.0.2-py2.py3-none-any/pip-20.0.2.virtualenv to /Users/travis/build/nsoranzo/bioblend/.venv/lib/python3.7/site-packages/pip-20.0.2.virtualenv [DEBUG _sync:48]
973 changing mode of /Users/travis/build/nsoranzo/bioblend/.venv/bin/pip to 755 [INFO util:566]
974 changing mode of /Users/travis/build/nsoranzo/bioblend/.venv/bin/pip3 to 755 [INFO util:566]
975 changing mode of /Users/travis/build/nsoranzo/bioblend/.venv/bin/pip-3.7 to 755 [INFO util:566]
975 generated console scripts pip3 pip pip-3.7 [DEBUG base:53]
975 Attempting to release lock 4344232784 on /Users/travis/Library/Application Support/virtualenv/seed-app-data/v1/3.7/wheels.lock [DEBUG filelock:315]
975 Lock 4344232784 released on /Users/travis/Library/Application Support/virtualenv/seed-app-data/v1/3.7/wheels.lock [INFO filelock:318]
976 add activators for Bash, CShell, Fish, PowerShell, Python, Xonsh [INFO session:65]
980 write /Users/travis/build/nsoranzo/bioblend/.venv/pyvenv.cfg [DEBUG pyenv_cfg:34]
980 	home = /usr/local/Cellar/python/3.7.5/Frameworks/Python.framework/Versions/3.7 [DEBUG pyenv_cfg:38]
980 	implementation = CPython [DEBUG pyenv_cfg:38]
981 	version_info = 3.7.5.final.0 [DEBUG pyenv_cfg:38]
981 	virtualenv = 20.0.9 [DEBUG pyenv_cfg:38]
981 	include-system-site-packages = false [DEBUG pyenv_cfg:38]
981 	base-prefix = /usr/local/Cellar/python/3.7.5/Frameworks/Python.framework/Versions/3.7 [DEBUG pyenv_cfg:38]
981 	base-exec-prefix = /usr/local/Cellar/python/3.7.5/Frameworks/Python.framework/Versions/3.7 [DEBUG pyenv_cfg:38]
982 	base-executable = /usr/local/opt/python/bin/python3.7 [DEBUG pyenv_cfg:38]
982 created virtual environment CPython3.7.5.final.0-64 in 833ms
  creator CPython3Posix(dest=/Users/travis/build/nsoranzo/bioblend/.venv, clear=False, global=False)
  seeder FromAppData(download=False, pip=latest, setuptools=latest, wheel=latest, via=copy, app_data_dir=/Users/travis/Library/Application Support/virtualenv/seed-app-data/v1)
  activators BashActivator,CShellActivator,FishActivator,PowerShellActivator,PythonActivator,XonshActivator [WARNING __main__:21]
```


Discussion:

Patch:
diff --git a/docs/changelog/1704.feature.rst b/docs/changelog/1704.feature.rst
new file mode 100644
index 00000000..99cffebb
--- /dev/null
+++ b/docs/changelog/1704.feature.rst
@@ -0,0 +1,2 @@
+Remove ``__PYVENV_LAUNCHER__`` on macOs for Python ``3.7.(<8)`` and ``3.8.(<3)`` on interpreter startup via ``pth``
+file, this pulls in the `upstream patch <https://github.com/python/cpython/pull/9516>`_ - by :user:`gaborbernat`.
diff --git a/src/virtualenv/create/via_global_ref/api.py b/src/virtualenv/create/via_global_ref/api.py
index aa864c83..e0e90a9d 100644
--- a/src/virtualenv/create/via_global_ref/api.py
+++ b/src/virtualenv/create/via_global_ref/api.py
@@ -67,19 +67,23 @@ class ViaGlobalRefApi(Creator):
             )
 
     def create(self):
-        self.patch_distutils_via_pth()
+        self.install_patch()
 
-    def patch_distutils_via_pth(self):
+    def install_patch(self):
+        text = self.env_patch_text()
+        if text:
+            pth = self.purelib / "_virtualenv.pth"
+            logging.debug("create virtualenv import hook file %s", pth)
+            pth.write_text("import _virtualenv")
+            dest_path = self.purelib / "_virtualenv.py"
+            logging.debug("create %s", dest_path)
+            dest_path.write_text(text)
+
+    def env_patch_text(self):
         """Patch the distutils package to not be derailed by its configuration files"""
         with ensure_file_on_disk(Path(__file__).parent / "_virtualenv.py", self.app_data) as resolved_path:
             text = resolved_path.read_text()
-            text = text.replace('"__SCRIPT_DIR__"', repr(os.path.relpath(str(self.script_dir), str(self.purelib))))
-        dest_path = self.purelib / "_virtualenv.py"
-        logging.debug("create %s", dest_path)
-        dest_path.write_text(text)
-        pth = self.purelib / "_virtualenv.pth"
-        logging.debug("create virtualenv import hook file %s", pth)
-        pth.write_text("import _virtualenv")
+            return text.replace('"__SCRIPT_DIR__"', repr(os.path.relpath(str(self.script_dir), str(self.purelib))))
 
     def _args(self):
         return super(ViaGlobalRefApi, self)._args() + [("global", self.enable_system_site_package)]
diff --git a/src/virtualenv/create/via_global_ref/builtin/cpython/cpython3.py b/src/virtualenv/create/via_global_ref/builtin/cpython/cpython3.py
index 589544c1..3e49a520 100644
--- a/src/virtualenv/create/via_global_ref/builtin/cpython/cpython3.py
+++ b/src/virtualenv/create/via_global_ref/builtin/cpython/cpython3.py
@@ -1,6 +1,7 @@
 from __future__ import absolute_import, unicode_literals
 
 import abc
+from textwrap import dedent
 
 from six import add_metaclass
 
@@ -22,6 +23,24 @@ class CPython3Posix(CPythonPosix, CPython3):
     def can_describe(cls, interpreter):
         return is_mac_os_framework(interpreter) is False and super(CPython3Posix, cls).can_describe(interpreter)
 
+    def env_patch_text(self):
+        text = super(CPython3Posix, self).env_patch_text()
+        if self.pyvenv_launch_patch_active(self.interpreter):
+            text += dedent(
+                """
+                # for https://github.com/python/cpython/pull/9516, see https://github.com/pypa/virtualenv/issues/1704
+                import os
+                if "__PYVENV_LAUNCHER__" in os.environ:
+                    del os.environ["__PYVENV_LAUNCHER__"]
+                """
+            )
+        return text
+
+    @classmethod
+    def pyvenv_launch_patch_active(cls, interpreter):
+        ver = interpreter.version_info
+        return interpreter.platform == "darwin" and ((3, 7, 8) > ver >= (3, 7) or (3, 8, 3) > ver >= (3, 8))
+
 
 class CPython3Windows(CPythonWindows, CPython3):
     """"""


Test patch:
diff --git a/tests/unit/create/test_creator.py b/tests/unit/create/test_creator.py
index 9fe810d9..60f5b866 100644
--- a/tests/unit/create/test_creator.py
+++ b/tests/unit/create/test_creator.py
@@ -20,6 +20,7 @@ import pytest
 
 from virtualenv.__main__ import run, run_with_catch
 from virtualenv.create.creator import DEBUG_SCRIPT, Creator, get_env_debug_info
+from virtualenv.create.via_global_ref.builtin.cpython.cpython3 import CPython3Posix
 from virtualenv.create.via_global_ref.builtin.python2.python2 import Python2
 from virtualenv.discovery.builtin import get_interpreter
 from virtualenv.discovery.py_info import PythonInfo
@@ -219,6 +220,13 @@ def test_create_no_seed(python, creator, isolated, system, coverage_env, special
         assert python_w.exists()
         assert python_w.read_bytes() != result.creator.exe.read_bytes()
 
+    if CPython3Posix.pyvenv_launch_patch_active(PythonInfo.from_exe(python)) and creator_key != "venv":
+        result = subprocess.check_output(
+            [str(result.creator.exe), "-c", 'import os; print(os.environ.get("__PYVENV_LAUNCHER__"))'],
+            universal_newlines=True,
+        ).strip()
+        assert result == "None"
+
 
 @pytest.mark.skipif(not CURRENT.has_venv, reason="requires interpreter with venv")
 def test_venv_fails_not_inline(tmp_path, capsys, mocker):