Repository: scalableminds/webknossos-libs

Issue:
Warn when multiprocessing fails due to missing if __name__ guard
If wklibs are used with multiprocessing (e.g., calling `downsample` on a layer), python's multiprocessing module will import the main module (if `spawn` is used which is the default on OS  X). If that module has side effects (i.e., it is not guarded with `if __name__ == "__main__":`), weird errors can occur.

On start up, we could spawn a dummy job with the clustertools and if that fails, we could emit

```
            raise AssertionError(
                """
                #########################################
                #########################################
                #########################################
                #########################################
                #########################################
                ## Multiprocessing setup does not work ##
                ## properly. Please check that you are ##
                ## using an if __name__ == "__main__"  ##
                ## guard in your main module.          ##
                #########################################
                #########################################
                #########################################
                #########################################
                #########################################
            """
            ) 
```

to help the user with what's wrong.


Discussion:

Resolving patch:
diff --git a/cluster_tools/cluster_tools/__init__.py b/cluster_tools/cluster_tools/__init__.py
--- a/cluster_tools/cluster_tools/__init__.py
+++ b/cluster_tools/cluster_tools/__init__.py
@@ -286,12 +286,52 @@ def submit(self, _func, *_args, **_kwargs):
         return super().submit(pickle_identity_executor, func, *args, **kwargs)
 
 
+def noop():
+    return True
+
+
+did_start_test_multiprocessing = False
+
+
+def test_valid_multiprocessing():
+
+    msg = """
+    ###############################################################
+    An attempt has been made to start a new process before the
+    current process has finished its bootstrapping phase.
+
+    This probably means that you are not using fork to start your
+    child processes and you have forgotten to use the proper idiom
+    in the main module:
+
+        if __name__ == '__main__':
+            main()
+            ...
+    ###############################################################
+    """
+
+    with get_executor("multiprocessing") as executor:
+        try:
+            res_fut = executor.submit(noop)
+            assert res_fut.result() == True, msg
+        except RuntimeError as exc:
+            raise Exception(msg) from exc
+        except EOFError as exc:
+            raise Exception(msg) from exc
+
+
 def get_executor(environment, **kwargs):
+
     if environment == "slurm":
         return SlurmExecutor(**kwargs)
     elif environment == "pbs":
         return PBSExecutor(**kwargs)
     elif environment == "multiprocessing":
+        global did_start_test_multiprocessing
+        if not did_start_test_multiprocessing:
+            did_start_test_multiprocessing = True
+            test_valid_multiprocessing()
+
         return WrappedProcessPoolExecutor(**kwargs)
     elif environment == "sequential":
         return SequentialExecutor(**kwargs)


Test patch:
diff --git a/cluster_tools/tests/guardless_multiprocessing.py b/cluster_tools/tests/guardless_multiprocessing.py
new file mode 100644
--- /dev/null
+++ b/cluster_tools/tests/guardless_multiprocessing.py
@@ -0,0 +1,30 @@
+import multiprocessing
+
+from cluster_tools import get_executor
+
+"""
+This file is an example of an incorrect script setup.
+The module does not use the
+    if __name__ == "__main__":
+      main()
+pattern which can lead to bugs when using multiprocessing.
+The clustertools will detect the wrong usage and emit a warning.
+This file is used to test the warning mechanism.
+"""
+
+
+multiprocessing.set_start_method("spawn", force=True)
+
+
+def worker_fn() -> bool:
+    return True
+
+
+def main() -> None:
+
+    res_fut = get_executor("multiprocessing").submit(worker_fn)
+    assert res_fut.result() == True, "Function should return True"
+    print("success")
+
+
+main()
diff --git a/cluster_tools/test.py b/cluster_tools/tests/tests.py
similarity index 98%
rename from cluster_tools/test.py
rename to cluster_tools/tests/tests.py
--- a/cluster_tools/test.py
+++ b/cluster_tools/tests/tests.py
@@ -824,6 +824,18 @@ def test_preliminary_file_map():
                 ), "Preliminary output file should not exist anymore"
 
 
+def test_multiprocessing_validation():
+
+    import sys
+    from subprocess import PIPE, STDOUT, Popen
+
+    cmd = [sys.executable, "guardless_multiprocessing.py"]
+    p = Popen(cmd, shell=False, stdin=PIPE, stdout=PIPE, stderr=STDOUT)
+    output = p.stdout.read()
+
+    assert "current process has finished its bootstrapping phase." in str(output), "S"
+
+
 if __name__ == "__main__":
     # Validate that slurm_executor.submit also works when being called from a __main__ module
     test_dereferencing_main()