Repository: pyinstaller/pyinstaller

Issue:
Multiprocess forking does not work on osx
This was introduced in 7f63111f4f19d4815b6a3235b6d14122e8f9510d. We probably need to disable the spawn fix for forking.



Discussion:
Related #2322.
/cc @mkassner
@xoviat   Instead of the current fix in pyinstaller, I m using this solution by @bpteague:

https://gist.github.com/bpteague/bc574e4d92c9d9904ceba895fb694fc8


It works perfectly without side-effects.
The commit is the patch with some minor integration adjustments. It apparently does not work perfectly.
Ah ok. I m running this on Mac with bundles that mulitprocess in  forking and non-forking manner. 
FYI [test failure is](https://travis-ci.org/pyinstaller/pyinstaller/jobs/263153179#L4237)

```
[8253] LOADER: Running pyi_multiprocess_forking.py
Fatal Python error: Cannot recover from stack overflow.
Current thread 0x00007fff78b41000 (most recent call first):
  File "PyInstaller/loader/rthooks/pyi_rth_multiprocessing.py", line 53 in __init__
  File "PyInstaller/loader/rthooks/pyi_rth_multiprocessing.py", line 53 in __init__
  File "PyInstaller/loader/rthooks/pyi_rth_multiprocessing.py", line 53 in __init__
  File "PyInstaller/loader/rthooks/pyi_rth_multiprocessing.py", line 53 in __init__
  File "PyInstaller/loader/rthooks/pyi_rth_multiprocessing.py", line 53 in __init__
  File "PyInstaller/loader/rthooks/pyi_rth_multiprocessing.py", line 53 in __init__
  File "PyInstaller/loader/rthooks/pyi_rth_multiprocessing.py", line 53 in __init__
  File "PyInstaller/loader/rthooks/pyi_rth_multiprocessing.py", line 53 in __init__
  File "PyInstaller/loader/rthooks/pyi_rth_multiprocessing.py", line 53 in __init__
  File "PyInstaller/loader/rthooks/pyi_rth_multiprocessing.py", line 53 in __init__
  File "PyInstaller/loader/rthooks/pyi_rth_multiprocessing.py", line 53 in __init__
  File "PyInstaller/loader/rthooks/pyi_rth_multiprocessing.py", line 53 in __init__
  File "PyInstaller/loader/rthooks/pyi_rth_multiprocessing.py", line 53 in __init__
  File "PyInstaller/loader/rthooks/pyi_rth_multiprocessing.py", line 53 in __init__
  File "PyInstaller/loader/rthooks/pyi_rth_multiprocessing.py", line 53 in __init__
  File "PyInstaller/loader/rthooks/pyi_rth_multiprocessing.py", line 53 in __init__
```
Huh, that's curious: Line 53 is:
```
    class _Popen(forking.Popen):
        def __init__(self, *args, **kw):
                …
                super(_Popen, self).__init__(*args, **kw)
```
Which would mean `pyi_rth_multiprocessing._Popen` is it's own super-class.
@htgoebel I would consider this a blocker for 3.3 as it breaks existing functionality. 
@springermac What is the way forward on this issue?

2017-08-31 18:12 GMT-05:00 Jonathan Springer <notifications@github.com>:

> @htgoebel <https://github.com/htgoebel> I would consider this a blocker
> for 3.3 as it breaks existing functionality.
>
> —
> You are receiving this because you were mentioned.
> Reply to this email directly, view it on GitHub
> <https://github.com/pyinstaller/pyinstaller/issues/2759#issuecomment-326444622>,
> or mute the thread
> <https://github.com/notifications/unsubscribe-auth/AOo8_GrRkuMQq46nfRft0kayTc-AJYE7ks5sdz34gaJpZM4O9_TU>
> .
>

I agree this is a heavy issue. But is anybody working on it? Is there an idea how to solve it? I want to release 3.3 ASAP (maybe even this weekend), so I'd rather have a known issue then to wait for weeks until this one is solved.

Resolving patch:
diff --git a/PyInstaller/loader/rthooks/pyi_rth_multiprocessing.py b/PyInstaller/loader/rthooks/pyi_rth_multiprocessing.py
--- a/PyInstaller/loader/rthooks/pyi_rth_multiprocessing.py
+++ b/PyInstaller/loader/rthooks/pyi_rth_multiprocessing.py
@@ -1,22 +1,35 @@
-from sys import platform as _platform
+import sys
 
 # 'spawn' multiprocessing needs some adjustments on osx
-if _platform == 'darwin':
-    import sys
+if sys.version_info >= (3, 4):
     import os
     import re
     import multiprocessing
     import multiprocessing.spawn as spawn
+    from subprocess import _args_from_interpreter_flags
 
     # prevent spawn from trying to read __main__ in from the main script
     multiprocessing.process.ORIGINAL_DIR = None
 
     def _freeze_support():
-        if 'multiprocessing.semaphore_tracker' in sys.argv[-1]:
-            m = re.compile('.*main\((\d+)\)$').match(sys.argv[-1])
-            fd = int(m.group(1))
-            import multiprocessing.semaphore_tracker
-            multiprocessing.semaphore_tracker.main(fd)
+        # we want to catch the two processes that are spawned by the
+        # multiprocessing code:
+        # - the semaphore tracker, which cleans up named semaphores in 
+        #   the spawn multiprocessing mode
+        # - the fork server, which keeps track of worker processes in 
+        #   forkserver mode.
+        # both of these processes are started by spawning a new copy of the
+        # running executable, passing it the flags from
+        # _args_from_interpreter_flags and then "-c" and an import statement.
+        # look for those flags and the import statement, then exec() the
+        # code ourselves.
+
+        if len(sys.argv) >= 2 and \
+                set(sys.argv[1:-2]) == set(_args_from_interpreter_flags()) and \
+                sys.argv[-2] == '-c' and \
+                (sys.argv[-1].startswith('from multiprocessing.semaphore_tracker import main') or \
+                 sys.argv[-1].startswith('from multiprocessing.forkserver import main')):
+            exec(sys.argv[-1])
             sys.exit()
 
         if spawn.is_forking(sys.argv):
@@ -32,34 +45,39 @@ def _freeze_support():
 
     multiprocessing.freeze_support = spawn.freeze_support = _freeze_support
 
-    # Module multiprocessing is organized differently in Python 3.4+
-    try:
-        # Python 3.4+
-        if sys.platform.startswith('win'):
-            import multiprocessing.popen_spawn_win32 as forking
-        else:
-            import multiprocessing.popen_fork as forking
-    except ImportError:
-        import multiprocessing.forking as forking
+# Bootloader unsets _MEIPASS2 for child processes to allow running
+# PyInstaller binaries inside pyinstaller binaries.
+# This is ok for mac or unix with fork() system call.
+# But on Windows we need to overcome missing fork() function.
+
+# Module multiprocessing is organized differently in Python 3.4+
+try:
+    # Python 3.4+
+    if sys.platform.startswith('win'):
+        import multiprocessing.popen_spawn_win32 as forking
+    else:
+        import multiprocessing.popen_fork as forking
+except ImportError:
+    import multiprocessing.forking as forking
 
-    # First define a modified version of Popen.
-    class _Popen(forking.Popen):
-        def __init__(self, *args, **kw):
+# Patch Popen to re-set _MEIPASS2 from sys._MEIPASS.
+class _Popen(forking.Popen):
+    def __init__(self, *args, **kw):
+        if hasattr(sys, 'frozen'):
+            # We have to set original _MEIPASS2 value from sys._MEIPASS
+            # to get --onefile mode working.
+            os.putenv('_MEIPASS2', sys._MEIPASS)  # @UndefinedVariable
+        try:
+            super(_Popen, self).__init__(*args, **kw)
+        finally:
             if hasattr(sys, 'frozen'):
-                # We have to set original _MEIPASS2 value from sys._MEIPASS
-                # to get --onefile mode working.
-                os.putenv('_MEIPASS2', sys._MEIPASS)  # @UndefinedVariable
-            try:
-                super(_Popen, self).__init__(*args, **kw)
-            finally:
-                if hasattr(sys, 'frozen'):
-                    # On some platforms (e.g. AIX) 'os.unsetenv()' is not
-                    # available. In those cases we cannot delete the variable
-                    # but only set it to the empty string. The bootloader
-                    # can handle this case.
-                    if hasattr(os, 'unsetenv'):
-                        os.unsetenv('_MEIPASS2')
-                    else:
-                        os.putenv('_MEIPASS2', '')
+                # On some platforms (e.g. AIX) 'os.unsetenv()' is not
+                # available. In those cases we cannot delete the variable
+                # but only set it to the empty string. The bootloader
+                # can handle this case.
+                if hasattr(os, 'unsetenv'):
+                    os.unsetenv('_MEIPASS2')
+                else:
+                    os.putenv('_MEIPASS2', '')
 
-    forking.Popen = _Popen
+forking.Popen = _Popen


Test patch:
diff --git a/tests/functional/scripts/pyi_multiprocess.py b/tests/functional/scripts/pyi_multiprocess.py
--- a/tests/functional/scripts/pyi_multiprocess.py
+++ b/tests/functional/scripts/pyi_multiprocess.py
@@ -7,16 +7,6 @@
 # The full license is in the file COPYING.txt, distributed with this software.
 #-----------------------------------------------------------------------------
 
-
-# Bootloader unsets _MEIPASS2 for child processes to allow running
-# PyInstaller binaries inside pyinstaller binaries.
-# This is ok for mac or unix with fork() system call.
-# But on Windows we need to overcome missing fork() fuction for onefile
-# mode.
-#
-# See http://www.pyinstaller.org/wiki/Recipe/Multiprocessing
-
-
 import multiprocessing
 import sys
 
@@ -34,9 +24,7 @@ def run(self):
 
 
 if __name__ == '__main__':
-    # On Windows calling this function is necessary.
-    if sys.platform.startswith('win'):
-        multiprocessing.freeze_support()
+    multiprocessing.freeze_support()
     print('main begins')
     resultQueue = multiprocessing.Queue()
     sp = SendeventProcess(resultQueue)
diff --git a/tests/functional/scripts/pyi_multiprocess_forking.py b/tests/functional/scripts/pyi_multiprocess_forking.py
--- a/tests/functional/scripts/pyi_multiprocess_forking.py
+++ b/tests/functional/scripts/pyi_multiprocess_forking.py
@@ -7,58 +7,11 @@
 # The full license is in the file COPYING.txt, distributed with this software.
 #-----------------------------------------------------------------------------
 
-
-# Bootloader unsets _MEIPASS2 for child processes to allow running
-# PyInstaller binaries inside pyinstaller binaries.
-# This is ok for mac or unix with fork() system call.
-# But on Windows we need to overcome missing fork() fuction.
-#
-# See http://www.pyinstaller.org/wiki/Recipe/Multiprocessing
-
-
-import os
-
 import sys
 import multiprocessing
 
-# Module multiprocessing is organized differently in Python 3.4+
-try:
-    # Python 3.4+
-    if sys.platform.startswith('win'):
-        import multiprocessing.popen_spawn_win32 as forking
-    else:
-        import multiprocessing.popen_fork as forking
-except ImportError:
-    import multiprocessing.forking as forking
-
-
-class _Popen(forking.Popen):
-    def __init__(self, *args, **kw):
-        if hasattr(sys, 'frozen'):
-            # We have to set original _MEIPASS2 value from sys._MEIPASS
-            # to get --onefile mode working.
-            # Last character is stripped in C-loader. We have to add
-            # '/' or '\\' at the end.
-            os.putenv('_MEIPASS2', sys._MEIPASS + os.sep)
-        try:
-            super(_Popen, self).__init__(*args, **kw)
-        finally:
-            if hasattr(sys, 'frozen'):
-                # On some platforms (e.g. AIX) 'os.unsetenv()' is not
-                # available. In those cases we cannot delete the variable
-                # but only set it to the empty string. The bootloader
-                # can handle this case.
-                if hasattr(os, 'unsetenv'):
-                    os.unsetenv('_MEIPASS2')
-                else:
-                    os.putenv('_MEIPASS2', '')
-
-
-class Process(multiprocessing.Process):
-    _Popen = _Popen
-
 
-class SendeventProcess(Process):
+class SendeventProcess(multiprocessing.Process):
     def __init__(self, resultQueue):
         self.resultQueue = resultQueue
 
@@ -72,9 +25,7 @@ def run(self):
 
 
 if __name__ == '__main__':
-    # On Windows calling this function is necessary.
-    if sys.platform.startswith('win'):
-        multiprocessing.freeze_support()
+    multiprocessing.freeze_support()
     print('main')
     resultQueue = multiprocessing.Queue()
     SendeventProcess(resultQueue)
diff --git a/tests/functional/scripts/pyi_multiprocess_pool.py b/tests/functional/scripts/pyi_multiprocess_pool.py
--- a/tests/functional/scripts/pyi_multiprocess_pool.py
+++ b/tests/functional/scripts/pyi_multiprocess_pool.py
@@ -7,57 +7,10 @@
 # The full license is in the file COPYING.txt, distributed with this software.
 #-----------------------------------------------------------------------------
 
-
-# Bootloader unsets _MEIPASS2 for child processes to allow running
-# PyInstaller binaries inside pyinstaller binaries.
-# This is ok for mac or unix with fork() system call.
-# But on Windows we need to overcome missing fork() fuction.
-#
-# See http://www.pyinstaller.org/wiki/Recipe/Multiprocessing
-
-import os
-import sys
-
-# Module multiprocessing is organized differently in Python 3.4+
-try:
-    # Python 3.4+
-    if sys.platform.startswith('win'):
-        import multiprocessing.popen_spawn_win32 as forking
-    else:
-        import multiprocessing.popen_fork as forking
-except ImportError:
-    import multiprocessing.forking as forking
-
-
-if sys.platform.startswith('win'):
-    # First define a modified version of Popen.
-    class _Popen(forking.Popen):
-        def __init__(self, *args, **kw):
-            if hasattr(sys, 'frozen'):
-                # We have to set original _MEIPASS2 value from sys._MEIPASS
-                # to get --onefile mode working.
-                os.putenv('_MEIPASS2', sys._MEIPASS)
-            try:
-                super(_Popen, self).__init__(*args, **kw)
-            finally:
-                if hasattr(sys, 'frozen'):
-                    # On some platforms (e.g. AIX) 'os.unsetenv()' is not
-                    # available. In those cases we cannot delete the variable
-                    # but only set it to the empty string. The bootloader
-                    # can handle this case.
-                    if hasattr(os, 'unsetenv'):
-                        os.unsetenv('_MEIPASS2')
-                    else:
-                        os.putenv('_MEIPASS2', '')
-
-    # Second override 'Popen' class with our modified version.
-    forking.Popen = _Popen
-
-
 import multiprocessing
 
 
-def  f(x):
+def f(x):
     return x*x
 
 
diff --git a/tests/functional/test_basic.py b/tests/functional/test_basic.py
--- a/tests/functional/test_basic.py
+++ b/tests/functional/test_basic.py
@@ -196,26 +196,6 @@ def test_module_reload(pyi_builder):
     pyi_builder.test_script('pyi_module_reload.py')
 
 
-# TODO move 'multiprocessig' tests into 'test_multiprocess.py.
-
-
-@skipif_win(reason="Issue #2116")
-@importorskip('multiprocessing')
-def test_multiprocess(pyi_builder):
-    pyi_builder.test_script('pyi_multiprocess.py')
-
-
-@xfail(is_darwin, reason="Pull Request #2505")
-@importorskip('multiprocessing')
-def test_multiprocess_forking(pyi_builder):
-    pyi_builder.test_script('pyi_multiprocess_forking.py')
-
-
-@importorskip('multiprocessing')
-def test_multiprocess_pool(pyi_builder):
-    pyi_builder.test_script('pyi_multiprocess_pool.py')
-
-
 # TODO test it on OS X.
 @skipif_no_compiler
 def test_load_dll_using_ctypes(monkeypatch, pyi_builder, compiled_dylib):
diff --git a/tests/functional/test_multiprocess.py b/tests/functional/test_multiprocess.py
new file mode 100644
--- /dev/null
+++ b/tests/functional/test_multiprocess.py
@@ -0,0 +1,173 @@
+# -*- coding: utf-8 -*-
+# ----------------------------------------------------------------------------
+# Copyright (c) 2005-2017, PyInstaller Development Team.
+#
+# Distributed under the terms of the GNU General Public License with exception
+# for distributing bootloader.
+#
+# The full license is in the file COPYING.txt, distributed with this software.
+# ----------------------------------------------------------------------------
+
+# Library imports
+# ---------------
+import os
+import sys
+
+# Local imports
+# -------------
+from PyInstaller.compat import is_py34, is_win
+from PyInstaller.utils.tests import importorskip, skipif
+
+
+@importorskip('multiprocessing')
+def test_multiprocess(pyi_builder):
+    pyi_builder.test_script('pyi_multiprocess.py')
+
+
+@importorskip('multiprocessing')
+def test_multiprocess_forking(pyi_builder):
+    pyi_builder.test_script('pyi_multiprocess_forking.py')
+
+
+@importorskip('multiprocessing')
+def test_multiprocess_pool(pyi_builder):
+    pyi_builder.test_script('pyi_multiprocess_pool.py')
+
+
+@skipif(not is_py34, reason='set_start_method introduced in python 3.4+')
+@importorskip('multiprocessing')
+def test_multiprocess_spawn_semaphore(pyi_builder, capfd):
+    pyi_builder.test_source("""
+        import sys 
+
+        from multiprocessing import set_start_method, Process, Semaphore
+        from multiprocessing import freeze_support
+        from multiprocessing.util import log_to_stderr
+
+        def test(s):
+            s.acquire()
+            print('In subprocess')
+            s.release()
+
+        if __name__ == '__main__':
+            log_to_stderr()
+            freeze_support()
+            set_start_method('spawn')
+
+            print('In main')
+            sys.stdout.flush()
+            s = Semaphore()
+            s.acquire()
+            proc = Process(target=test, args = [s])
+            proc.start()
+            s.release()
+            proc.join()
+        """)
+
+    out, err = capfd.readouterr()
+
+    # Print the captured output and error so that it will show up in the test output.
+    sys.stderr.write(err)
+    sys.stdout.write(out)
+
+    expected = ["In main", "In subprocess"]
+
+    assert os.linesep.join(expected) in out
+    for substring in expected:
+        assert out.count(substring) == 1
+
+
+@skipif(not is_py34, reason='set_start_method introduced in python 3.4+')
+@skipif(is_win, reason='fork is not available on windows')
+@importorskip('multiprocessing')
+def test_multiprocess_fork_semaphore(pyi_builder, capfd):
+    pyi_builder.test_source("""
+        import sys 
+
+        from multiprocessing import set_start_method, Process, Semaphore
+        from multiprocessing import freeze_support
+        from multiprocessing.util import log_to_stderr
+
+        def test(s):
+            s.acquire()
+            print('In subprocess')
+            s.release()
+
+        if __name__ == '__main__':
+            log_to_stderr()
+            freeze_support()
+            set_start_method('fork')
+
+            print('In main')
+            sys.stdout.flush()
+            s = Semaphore()
+            s.acquire()
+            proc = Process(target=test, args = [s])
+            proc.start()
+            s.release()
+            proc.join()
+        """)
+
+    out, err = capfd.readouterr()
+
+    # Print the captured output and error so that it will show up in the test output.
+    sys.stderr.write(err)
+    sys.stdout.write(out)
+
+    expected = ["In main", "In subprocess"]
+
+    assert os.linesep.join(expected) in out
+    for substring in expected:
+        assert out.count(substring) == 1
+
+
+
+
+@skipif(not is_py34, reason='set_start_method introduced in python 3.4+')
+@skipif(is_win, reason='forkserver is not available on windows')
+@importorskip('multiprocessing')
+def test_multiprocess_forkserver_semaphore(pyi_builder, capfd):
+    pyi_builder.test_source("""
+        import sys 
+
+        from multiprocessing import set_start_method, Process, Semaphore
+        from multiprocessing import freeze_support
+        from multiprocessing.util import log_to_stderr
+
+        def test(s):
+            s.acquire()
+            print('In subprocess')
+            s.release()
+
+        if __name__ == '__main__':
+            log_to_stderr()
+            freeze_support()
+            set_start_method('forkserver')
+
+            print('In main')
+            sys.stdout.flush()
+            s = Semaphore()
+            s.acquire()
+            proc = Process(target=test, args = [s])
+            proc.start()
+            s.release()
+            proc.join()
+        """)
+
+    out, err = capfd.readouterr()
+
+    # Print the captured output and error so that it will show up in the test output.
+    sys.stderr.write(err)
+    sys.stdout.write(out)
+
+    expected = ["In main", "In subprocess"]
+
+    assert os.linesep.join(expected) in out
+    for substring in expected:
+        assert out.count(substring) == 1
+
+
+
+
+
+
diff --git a/tests/functional/test_runtime.py b/tests/functional/test_runtime.py
--- a/tests/functional/test_runtime.py
+++ b/tests/functional/test_runtime.py
@@ -22,38 +22,3 @@ def test_ctypes_cdll_unknown_dll(pyi_builder, capfd):
             """)
     out, err = capfd.readouterr()
     assert "Failed to load dynlib/dll" in err
-
-
-@skipif_notosx
-@xfail_py2
-def test_issue_2322(pyi_builder, capfd):
-    pyi_builder.test_source("""
-        from multiprocessing import set_start_method, Process
-        from multiprocessing import freeze_support
-        from multiprocessing.util import log_to_stderr
-
-        def test():
-            print('In subprocess')
-
-        if __name__ == '__main__':
-            log_to_stderr()
-            freeze_support()
-            set_start_method('spawn')
-
-            print('In main')
-            proc = Process(target=test)
-            proc.start()
-            proc.join()
-        """)
-
-    out, err = capfd.readouterr()
-
-    # Print the captured output and error so that it will show up in the test output.
-    sys.stderr.write(err)
-    sys.stdout.write(out)
-
-    expected = ["In main", "In subprocess"]
-
-    assert "\n".join(expected) in out
-    for substring in expected:
-        assert out.count(substring) == 1