Repository: ray-project/ray

Issue:
Fatal Python error: PyImport_GetModuleDict: no module dictionary
<!--Please include [tune], [rllib], [autoscaler] etc. in the issue title if relevant-->

### What is the problem?

On actor shutdown, we get a lot of warnings like this:

```
(pid=58704) Current thread 0x00007f61957f3700 (most recent call first):
(pid=58704)   File "/home/ubuntu/ray/python/ray/worker.py", line 433 in main_loop
(pid=58704)   File "/home/ubuntu/ray/python/ray/workers/default_worker.py", line 99 in <module>
(pid=58709) WARNING: Not monitoring node memory since `psutil` is not installed. Install this with `pip install psutil` (or ray[debug]) to enable debugging of memory-related crashes.
(pid=58709) Fatal Python error: PyImport_GetModuleDict: no module dictionary!
(pid=58709) 
(pid=58709) Current thread 0x00007f27c36a4700 (most recent call first):
(pid=58709)   File "/home/ubuntu/ray/python/ray/worker.py", line 433 in main_loop
(pid=58709)   File "/home/ubuntu/ray/python/ray/workers/default_worker.py", line 99 in <module>
(pid=58659) WARNING: Not monitoring node memory since `psutil` is not installed. Install this with `pip install psutil` (or ray[debug]) to enable debugging of memory-related crashes.
(pid=58659) Fatal Python error: PyImport_GetModuleDict: no module dictionary!
(pid=58659) 
(pid=58659) Current thread 0x00007f2b0d61f700 (most recent call first):
(pid=58659)   File "/home/ubuntu/ray/python/ray/worker.py", line 433 in main_loop
(pid=58659)   File "/home/ubuntu/ray/python/ray/workers/default_worker.py", line 99 in <module>
(pid=58693) Fatal Python error: PyImport_GetModuleDict: no module dictionary!
(pid=58693) 
```


### Reproduction
Run any RLlib job.

cc @edoakes 


Discussion:
Manually testing indicates it was introduced by: https://github.com/ray-project/ray/pull/5783
Does it also only happen on CUDA instances? I can't reproduce this locally, only on EC2. The Python versions are both 3.6.
I'm able to reproduce on my laptop
Googling around shows that it might be caused by referencing imported modules inside of threads during shutdown. A few separate issues people have seen similar to:
https://jira.mongodb.org/browse/SERVER-22142

Resolving patch:
diff --git a/python/ray/worker.py b/python/ray/worker.py
--- a/python/ray/worker.py
+++ b/python/ray/worker.py
@@ -431,6 +431,7 @@ def sigterm_handler(signum, frame):
 
         signal.signal(signal.SIGTERM, sigterm_handler)
         self.core_worker.run_task_loop()
+        sys.exit(0)
 
 
 def get_gpu_ids():
@@ -834,6 +835,12 @@ def shutdown(exiting_interpreter=False):
 
     disconnect(exiting_interpreter)
 
+    # We need to destruct the core worker here because after this function,
+    # we will tear down any processes spawned by ray.init() and the background
+    # IO thread in the core worker doesn't currently handle that gracefully.
+    if hasattr(global_worker, "core_worker"):
+        del global_worker.core_worker
+
     # Disconnect global state from GCS.
     ray.state.state.disconnect()
 
@@ -843,7 +850,7 @@ def shutdown(exiting_interpreter=False):
         _global_node.kill_all_processes(check_alive=False, allow_graceful=True)
         _global_node = None
 
-    # TODO(rkn): Instead of manually reseting some of the worker fields, we
+    # TODO(rkn): Instead of manually resetting some of the worker fields, we
     # should simply set "global_worker" to equal "None" or something like that.
     global_worker.set_mode(None)
     global_worker._post_get_hooks = []
@@ -1332,12 +1339,6 @@ def disconnect(exiting_interpreter=False):
     worker.cached_functions_to_run = []
     worker.serialization_context_map.clear()
 
-    # We need to destruct the core worker here because after this function,
-    # we will tear down any processes spawned by ray.init() and the background
-    # threads in the core worker don't currently handle that gracefully.
-    if hasattr(worker, "core_worker"):
-        del worker.core_worker
-
 
 @contextmanager
 def _changeproctitle(title, next_title):


Test patch:
diff --git a/python/ray/tests/test_tempfile.py b/python/ray/tests/test_tempfile.py
--- a/python/ray/tests/test_tempfile.py
+++ b/python/ray/tests/test_tempfile.py
@@ -37,13 +37,12 @@ def test_conn_cluster():
         "temp_dir must not be provided.")
 
 
-def test_tempdir():
+def test_tempdir(shutdown_only):
     shutil.rmtree("/tmp/ray", ignore_errors=True)
     ray.init(temp_dir="/tmp/i_am_a_temp_dir")
     assert os.path.exists(
         "/tmp/i_am_a_temp_dir"), "Specified temp dir not found."
     assert not os.path.exists("/tmp/ray"), "Default temp dir should not exist."
-    ray.shutdown()
     shutil.rmtree("/tmp/i_am_a_temp_dir", ignore_errors=True)
 
 
@@ -57,7 +56,7 @@ def test_tempdir_commandline():
     shutil.rmtree("/tmp/i_am_a_temp_dir2", ignore_errors=True)
 
 
-def test_raylet_socket_name():
+def test_raylet_socket_name(shutdown_only):
     ray.init(raylet_socket_name="/tmp/i_am_a_temp_socket")
     assert os.path.exists(
         "/tmp/i_am_a_temp_socket"), "Specified socket path not found."
@@ -77,7 +76,7 @@ def test_raylet_socket_name():
         pass  # It could have been removed by Ray.
 
 
-def test_temp_plasma_store_socket():
+def test_temp_plasma_store_socket(shutdown_only):
     ray.init(plasma_store_socket_name="/tmp/i_am_a_temp_socket")
     assert os.path.exists(
         "/tmp/i_am_a_temp_socket"), "Specified socket path not found."
@@ -97,7 +96,7 @@ def test_temp_plasma_store_socket():
         pass  # It could have been removed by Ray.
 
 
-def test_raylet_tempfiles():
+def test_raylet_tempfiles(shutdown_only):
     ray.init(num_cpus=0)
     node = ray.worker._global_node
     top_levels = set(os.listdir(node.get_session_dir_path()))
@@ -132,15 +131,13 @@ def test_raylet_tempfiles():
 
     socket_files = set(os.listdir(node.get_sockets_dir_path()))
     assert socket_files == {"plasma_store", "raylet"}
-    ray.shutdown()
 
 
-def test_tempdir_privilege():
+def test_tempdir_privilege(shutdown_only):
     os.chmod("/tmp/ray", 0o000)
     ray.init(num_cpus=1)
     session_dir = ray.worker._global_node.get_session_dir_path()
     assert os.path.exists(session_dir), "Specified socket path not found."
-    ray.shutdown()
 
 
 def test_session_dir_uniqueness():