Repository: microsoft/ptvsd

Issue:
Monkey-patch os.posix_spawn
But succeeds on Python 3.6 and 3.7. Looking at pydevd logs for the parent process, the difference is that there's no "Patching args" in case of 3.8. However, there *is* a pydevd logfile for the child process, which only contains the "Using Cython speedups" line.




Discussion:
Initial investigation:

I believe that this is due to https://bugs.python.org/issue35537 - Python 3.8 now uses `posix_spawn` instead of `fork`+`exec` to implement a "fast path" for `subprocess.Popen()`. Note that this doesn't always apply - e.g. on Linux it only kicks in above a certain version of glibc, and on all platforms it only applies if certain arguments to `Popen()` have specific values. Here's how that conditional looks in the source:
```py
            if (_USE_POSIX_SPAWN
                    and os.path.dirname(executable)
                    and preexec_fn is None
                    and not close_fds
                    and not pass_fds
                    and cwd is None
                    and (p2cread == -1 or p2cread > 2)
                    and (c2pwrite == -1 or c2pwrite > 2)
                    and (errwrite == -1 or errwrite > 2)
                    and not start_new_session):
                self._posix_spawn(args, executable, env, restore_signals,
                                  p2cread, p2cwrite,
                                  c2pread, c2pwrite,
                                  errread, errwrite)
                return
```
That's why `test_subprocess` works fine - it redirects the output of the child process (not sure why), and thus skips `posix_spawn`.

On Linux, `posix_spawn` is itself implemented as vfork + exec inside glibc, and it seems that pydevd can handle that correctly? But on macOS, it's a syscall, and I guess that breaks something.

@fabioz, looks like this is another thing that needs to go into `pydev_monkey.py`.
Same thing applies to `test_django_breakpoint_multiproc`.
Yes, it seems that `os.posix_spawn` isn't monkey-patched. I'll provide a patch shortly.

Resolving patch:
diff --git a/src/ptvsd/_vendored/pydevd/_pydev_bundle/pydev_monkey.py b/src/ptvsd/_vendored/pydevd/_pydev_bundle/pydev_monkey.py
--- a/src/ptvsd/_vendored/pydevd/_pydev_bundle/pydev_monkey.py
+++ b/src/ptvsd/_vendored/pydevd/_pydev_bundle/pydev_monkey.py
@@ -391,12 +391,12 @@ def warn_multiproc():
 
 def create_warn_multiproc(original_name):
 
-    def new_warn_multiproc(*args):
+    def new_warn_multiproc(*args, **kwargs):
         import os
 
         warn_multiproc()
 
-        return getattr(os, original_name)(*args)
+        return getattr(os, original_name)(*args, **kwargs)
 
     return new_warn_multiproc
 
@@ -499,6 +499,21 @@ def new_spawnve(mode, path, args, env):
     return new_spawnve
 
 
+def create_posix_spawn(original_name):
+    """
+    os.posix_spawn(executable, args, env, **kwargs)
+    """
+
+    def new_posix_spawn(executable, args, env, **kwargs):
+        if _get_apply_arg_patching():
+            args = patch_args(args)
+            send_process_created_message()
+
+        return getattr(os, original_name)(executable, args, env, **kwargs)
+
+    return new_posix_spawn
+
+
 def create_fork_exec(original_name):
     """
     _posixsubprocess.fork_exec(args, executable_list, close_fds, ... (13 more))
@@ -646,6 +661,7 @@ def patch_new_process_functions():
     monkey_patch_os('spawnve', create_spawnve)
     monkey_patch_os('spawnvp', create_spawnv)
     monkey_patch_os('spawnvpe', create_spawnve)
+    monkey_patch_os('posix_spawn', create_posix_spawn)
 
     if not IS_JYTHON:
         if not IS_WINDOWS:
@@ -681,6 +697,7 @@ def patch_new_process_functions_with_warning():
     monkey_patch_os('spawnve', create_warn_multiproc)
     monkey_patch_os('spawnvp', create_warn_multiproc)
     monkey_patch_os('spawnvpe', create_warn_multiproc)
+    monkey_patch_os('posix_spawn', create_warn_multiproc)
 
     if not IS_JYTHON:
         if not IS_WINDOWS:


Test patch:
diff --git a/src/ptvsd/_vendored/pydevd/tests_python/resources/_debugger_case_pydevd_customization.py b/src/ptvsd/_vendored/pydevd/tests_python/resources/_debugger_case_pydevd_customization.py
--- a/src/ptvsd/_vendored/pydevd/tests_python/resources/_debugger_case_pydevd_customization.py
+++ b/src/ptvsd/_vendored/pydevd/tests_python/resources/_debugger_case_pydevd_customization.py
@@ -22,21 +22,28 @@ def main():
 
     import subprocess
     if '--use-c-switch' in sys.argv:
-        p = subprocess.Popen(
+        child_process = subprocess.Popen(
             [sys.executable, '-u', '-c', 'import _debugger_case_pydevd_customization;_debugger_case_pydevd_customization.call()'],
             stdout=subprocess.PIPE,
             env=env,
         )
+    elif '--posix-spawn' in sys.argv:
+        env = os.environ.copy()
+        args = ['-u', '_debugger_case_pydevd_customization.py', '--simple-call']
+        pid = os.posix_spawn(sys.executable, args, env)
+        os.waitpid(pid, 0)
+        child_process = None  # We don't really have a subprocess.Popen instance in this case.
     else:
-        p = subprocess.Popen(
+        child_process = subprocess.Popen(
             [sys.executable, '-u', '_debugger_case_pydevd_customization.py', '--simple-call'],
             cwd=os.path.dirname(__file__),
             stdout=subprocess.PIPE,
             env=env,
         )
 
-    stdout, stderr = p.communicate()
-    assert b'called' in stdout, 'Did not find b"called" in: %s' % (stdout,)
+    if child_process:
+        stdout, stderr = child_process.communicate()
+        assert b'called' in stdout, 'Did not find b"called" in: %s' % (stdout,)
     print('TEST SUCEEDED!')  # break 2 here
 
 
diff --git a/src/ptvsd/_vendored/pydevd/tests_python/test_debugger_json.py b/src/ptvsd/_vendored/pydevd/tests_python/test_debugger_json.py
--- a/src/ptvsd/_vendored/pydevd/tests_python/test_debugger_json.py
+++ b/src/ptvsd/_vendored/pydevd/tests_python/test_debugger_json.py
@@ -2972,14 +2972,19 @@ def test_remote_debugger_basic(case_setup_remote):
         writer.finished_ok = True
 
 
-@pytest.mark.parametrize('use_c_switch', [True, False])
-def test_subprocess_pydevd_customization(case_setup_remote, use_c_switch):
+PYDEVD_CUSTOMIZATION_COMMAND_LINE_ARGS = ['', '--use-c-switch']
+if hasattr(os, 'posix_spawn'):
+    PYDEVD_CUSTOMIZATION_COMMAND_LINE_ARGS.append('--posix-spawn')
+
+
+@pytest.mark.parametrize('command_line_args', PYDEVD_CUSTOMIZATION_COMMAND_LINE_ARGS)
+def test_subprocess_pydevd_customization(case_setup_remote, command_line_args):
     import threading
     from tests_python.debugger_unittest import AbstractWriterThread
 
     with case_setup_remote.test_file(
             '_debugger_case_pydevd_customization.py',
-            append_command_line_args=['--use-c-switch'] if use_c_switch else []
+            append_command_line_args=command_line_args if command_line_args else [],
         ) as writer:
         json_facade = JsonFacade(writer, send_json_startup_messages=False)
         json_facade.writer.write_multi_threads_single_notification(True)