Repository: eventlet/eventlet

Issue:
os.fork() doesn't work with Eventlet on Python 3.13
While trying to make nova work with python3.13, I've encountered the following bug which I could reduce to this reproducer:
```python3
import eventlet

eventlet.monkey_patch()

import threading

def background_task():
    threading._after_fork()

t = threading.Thread(target=background_task)
t.start()
t.join()
```

Here's the error output:
```shell
$ python3 reproducer.py 
Exception in thread Thread-1 (background_task):
Traceback (most recent call last):
  File "/usr/lib/python3.13/threading.py", line 1566, in _after_fork
    current = _active[get_ident()]
              ~~~~~~~^^^^^^^^^^^^^
KeyError: 128691506491200

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/usr/lib/python3.13/threading.py", line 1041, in _bootstrap_inner
    self.run()
    ~~~~~~~~^^
  File "/usr/lib/python3.13/threading.py", line 992, in run
    self._target(*self._args, **self._kwargs)
    ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/var/lib/nova/reproducer.py", line 9, in background_task
    threading._after_fork()
    ~~~~~~~~~~~~~~~~~~~~~^^
  File "/usr/lib/python3.13/threading.py", line 1571, in _after_fork
    current = _MainThread()
  File "/usr/lib/python3.13/threading.py", line 1354, in __init__
    self._handle = _make_thread_handle(self._ident)
                   ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^
  File "/usr/lib/python3/dist-packages/eventlet/green/thread.py", line 80, in _make_thread_handle
    assert ident == get_ident(greenthread)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
AssertionError
```

The same snippet without eventlet works correctly:
```shell
$ python3 reproducer_threading.py
```

Error from nova that threw me into this race:
```python
Exception ignored in: <function _after_fork at 0x75fbe2136ac0>                                                              
Traceback (most recent call last):                                                                                          
  File "/usr/lib/python3.13/threading.py", line 1571, in _after_fork                                                        
    current = _MainThread()                                                                                                 
  File "/usr/lib/python3.13/threading.py", line 1354, in __init__                                                           
    self._handle = _make_thread_handle(self._ident)                                                                         
  File "/usr/lib/python3/dist-packages/eventlet/green/thread.py", line 80, in _make_thread_handle                           
    assert ident == get_ident(greenthread)                                                                                  
AssertionError:         
```


Discussion:
To make error more apparent:
```python
import io
import sys
if len(sys.argv) == 2 and sys.argv[1] == "eventlet":
    import eventlet
    eventlet.monkey_patch()

import threading

sys.stderr = io.StringIO()

def background_task():
    threading._after_fork()

t = threading.Thread(target=background_task)
t.start()
t.join()
if stderr := sys.stderr.getvalue():
    print(stderr)
    print("fail")
    sys.exit(1)
else:
    print("pass")

```

```python
python3 tests/isolated/patcher_threading_after_fork.py threading
pass
```
```python
python3 tests/isolated/patcher_threading_after_fork.py eventlet
Exception in thread Thread-1 (background_task):
Traceback (most recent call last):
  File "/home/gboutry/.local/share/uv/python/cpython-3.13.2-linux-x86_64-gnu/lib/python3.13/threading.py", line 1566, in _after_fork
    current = _active[get_ident()]
              ~~~~~~~^^^^^^^^^^^^^
KeyError: 140354633324736

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/gboutry/.local/share/uv/python/cpython-3.13.2-linux-x86_64-gnu/lib/python3.13/threading.py", line 1041, in _bootstrap_inner
    self.run()
    ~~~~~~~~^^
  File "/home/gboutry/.local/share/uv/python/cpython-3.13.2-linux-x86_64-gnu/lib/python3.13/threading.py", line 992, in run
    self._target(*self._args, **self._kwargs)
    ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/dev/shm/git/eventlet/tests/isolated/patcher_threading_after_fork.py", line 12, in background_task
    threading._after_fork()
    ~~~~~~~~~~~~~~~~~~~~~^^
  File "/home/gboutry/.local/share/uv/python/cpython-3.13.2-linux-x86_64-gnu/lib/python3.13/threading.py", line 1571, in _after_fork
    current = _MainThread()
  File "/home/gboutry/.local/share/uv/python/cpython-3.13.2-linux-x86_64-gnu/lib/python3.13/threading.py", line 1354, in __init__
    self._handle = _make_thread_handle(self._ident)
                   ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^
  File "/dev/shm/git/eventlet/eventlet/green/thread.py", line 80, in _make_thread_handle
    assert ident == get_ident(greenthread)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
AssertionError

fail
```

@gboutry thanks for your heads up.

With the introduction of [PEP 703](https://peps.python.org/pep-0703/) (No-GIL Mode) and related internal changes:

- Python introduced a more rigorous, internalized notion of thread identifiers, particularly to support "joinable threads" and the new `_thread.ThreadHandle` structure;
- Starting from Python 3.13, thread identifiers (ident) are no longer just simple values returned by `_thread.get_ident()`, instead, they now represent internal native identifiers managed at the interpreter level (C-level), tied directly to real OS-level threads or to new internal Python structures;
- A new internal function, `_get_main_thread_ident()`, was specifically added to retrieve this native internal thread identifier for the main thread, independently of Python-level wrappers or greenlets.

In short, Python has introduced an additional abstraction layer over thread identifiers. This technical shift means:

- ` _get_main_thread_ident()` no longer simply equals `_thread.get_ident()`;
- Using `id(greenlet)` to emulate threads in Eventlet no longer corresponds to Python’s new native internal thread identifier.
The problem is that, to solve that, it will surely require to deeply modify Eventlet to manage native identifiers, to use actual CPython native identifiers (CPython 3.13) rather than Python-level identifiers (id(greenlet))... Unfortunately, this solution is significantly more complex and beyond the scope of a simple compatibility patch.

I made a patch (not yet submitted), but my implementation always mismatch threads ident...

The shortest path would be either to ignore the ident comparison during the calls of `_make_thread_handle(ident)` or to raise a `ValueError` to inform of the Greenlet ident mismatch but I don't think that's really useful. So, if we really want something compatible, it would surely require almost an entire logic of Eventlet due to the GIL/PEP 703 changes introduced within CPython 3.13...
Lets workaround that for now... #1031 
I used your reproducer locally, to check my patch, and it now works with both versions, but yeah, ident comparison is now skipped which is also a path for unforeseen side effects somewhere else...
@4383 Thank you!

There might be something more insidious at play here, I've started a conversation with nova on IRC, but you might have an idea what's happening.

I've tried to use nova with python 3.13 (not yet supported upstream), and I'm getting a strange issue as soon as I try to create a VM. From my observation, `https://opendev.org/openstack/nova/src/commit/98226b60f3fe7b20e8d7f208c12f8d0086cd83d0/nova/network/neutron.py#L1216` the inner `base_client` of `admin_client` gets finalized in the current active greenthread. This leads to nova failing to get a network for the VM.
I've used `weakref.finalize` to add a callback to print the thread ident:
`finalizer called:  <_GreenThread(GreenThread-5, <GreenThread object at 0x718ce8547c00 (otid=0x718cfb6e7db0) current active started>)>`.

I am not sure if the fact that this is happening in the current active greenthread is a red herring or not.

Additional note: If I hold a hard ref on the base_client, it does not get finalized,  And admin_client gets finalized much later... I actually need to hold a hardref to `admin_client.base_client.http_client` to get a VM created...

This looks GIL/eventlet related to me?

For the resulting symptoms: https://bugs.launchpad.net/ubuntu/+source/nova/+bug/2103413
Hello,

Concerning the nova problem you describe I'm sorry, I've no ideas... but, as it is related to the "active thread" notion, and as this behaviour is observed with python 3.13 and as it seems to require the identity of the threads in some ways, then, yeah, maybe that would be related...
GIL and gevent: https://github.com/gevent/gevent/issues/2087
Is there work being done to deal with this issue, or should we just try really hard to move off eventlet before moving to python 3.13?
@cbf123: Well, I proposed a workaround (that is not finished and shown side effects), and in all the case it will remain a crappy workaround. I do not think we will be able to adapt eventlet properly, so, I'd encourage to you to try hard moving off eventlet. In all the case Eventlet is now discouraged since a while and the more wait to migrate the more we will suffer.
@cbf123: You can follow this guide https://removal.eventlet.org/ and find more resources there https://gibizer.github.io/
we are required to start supportign python3.13 in openstack this cycle as a fully supproted runtime https://github.com/openstack/governance/blob/master/reference/runtimes/2025.2.rst

so not fixing this is not really an option or evently will prevent use supprot that properly.
we may be able to avoid adding supprot for 3.14 by the time the eventlet removal is complete but not 3.13
Just to follow up on @SeanMooney 's comment.   Based on that doc it seems like Python 3.13 is not mandatory for Openstack 2025.2, but will be mandatory in "a future release", presumably in 2026.
correct, we typically ask for non voting jobs 1 cycle before requiring them to be voting.
the unfortunate situation we are in is that eventelet is functional enough to pass of of its tests on 3.13 and to pass all of nova unit and functional tests.

however when we actually deploy it with devstack it fails.

what makes this more painful is Ubuntu 25.04 which was intended to ship epoxy 2025.1 uses 3.13
as will debian 13 (trixie) when it release, fedora also already uses 3.13 already so while we can hopefully move off eventlet before we need to support 3.14 we are already late with support for 3.13.
So here is the thing: fork() without exec() is _fundamentally broken_. See e.g. writeup I made about `multiprocessing`: https://pythonspeed.com/articles/python-multiprocessing/ - I have successfully convinced CPython devs to switch `multiprocessing` to not use fork() backend as of Python 3.14.

If you are seeing problems with fork() you should _stop using fork()_. Even if we "fix" eventlet there will be other mysterious intermittent bugs.

So separate from any fix I would treat any use of fork() on your side as something that needs to be ripped out ASAP. It's just broken. There is no way to make it not broken.
right so as i mentioned elsewhere to stop using fork we need to fix oslo's eventlet backend  because most opnstack proccess are not suing fork intentionally.

its happening in oslo.service and potentially in other other libs that service project cant control.

this change woudl also have to be backported in to the epoxy version of oslo libs so that debian and ubuntu coudl actully package opensack on 3.13 in trixe and 25.04

so yes we shoudl absolutely stop usign fork. but in the interim we need to have a paractical solution for that in our existing code bases or that distos can backport to resolve the current 3.13 compaitblity issues.
My suggestion:

1. I will see what it takes to get `oslo.service` (or relevant upstream projects) to stop using fork(). It's just bad, separately from eventlet.
2. Until we have a good fix for this (and I'd like a real test, the reproducer above is not a thing anyone would run), it might be better to make fork()-without-exec() blow up on Python 3.13 with a useful explanation, so at least it's clear that's not currently supported.
Here is a better reproducer:

```python
import os
import eventlet

eventlet.monkey_patch()

import threading

def background_task():
    os.fork()

threading.Thread(target=background_task).start()
```



Patch:
diff --git a/doc/source/fork.rst b/doc/source/fork.rst
new file mode 100644
index 000000000..91b3c9851
--- /dev/null
+++ b/doc/source/fork.rst
@@ -0,0 +1,30 @@
+fork() without execve()
+=======================
+
+``fork()`` is a way to make a clone of a process.
+Most subprocesses replace the child process with a new executable, wiping out all memory from the parent process.
+A ``fork()ed`` subprocess can choose not to do this, and preserve data from the parent process.
+(Technically this is ``fork()`` without ``execve()``.)
+
+This is a terrible idea, as it can cause deadlocks, memory corruption, and crashes.
+
+For backwards compatibility, Eventlet *tries* to work in this case, but this is very much not guaranteed and your program may suffer from deadlocks, memory corruption, and crashes.
+This is even more so when using the ``asyncio`` hub, as that requires even more questionable interventions to "work".
+
+This is not a problem with Eventlet.
+It's a fundamental problem with ``fork()`` applicable to pretty much every Python program.
+
+Below are some common reasons you might be using ``fork()`` without knowing it, and what you can do about it.
+
+``multiprocessing``
+------------------
+
+On Linux, on Python 3.13 earlier, the standard library ``multiprocessing`` library uses ``fork()`` by default.
+To fix this, switch to the ``"spawn"`` method (the default in Python 3.14 and later) as `documented here <https://docs.python.org/3.13/library/multiprocessing.html#contexts-and-start-methods>`.
+
+
+``oslo.service``
+----------------
+
+There are alternative ways of running services that do not use ``fork()``.
+See the documentation.
diff --git a/doc/source/index.rst b/doc/source/index.rst
index 96b4de90b..923d7e6c6 100644
--- a/doc/source/index.rst
+++ b/doc/source/index.rst
@@ -93,6 +93,7 @@ Concepts & References
    zeromq
    hubs
    environment
+   fork
    modules
 
 Want to contribute?
diff --git a/eventlet/__init__.py b/eventlet/__init__.py
index 318ee0d69..01773c52b 100644
--- a/eventlet/__init__.py
+++ b/eventlet/__init__.py
@@ -76,4 +76,13 @@
         ('call_after_global', 'greenthread.call_after_global', greenthread.call_after_global),
     ))
 
-os
+
+if hasattr(os, "register_at_fork"):
+    def _warn_on_fork():
+        import warnings
+        warnings.warn(
+            "Using fork() is a bad idea, and there is no guarantee eventlet will work." +
+            " See https://eventlet.readthedocs.io/en/latest/fork.html for more details.",
+            DeprecationWarning
+        )
+    os.register_at_fork(before=_warn_on_fork)
diff --git a/eventlet/green/threading.py b/eventlet/green/threading.py
index 4be776682..ae01a5bb3 100644
--- a/eventlet/green/threading.py
+++ b/eventlet/green/threading.py
@@ -4,10 +4,11 @@
 from eventlet.green import time
 from eventlet.support import greenlets as greenlet
 
-__patched__ = ['Lock', '_after_fork', '_allocate_lock', '_get_main_thread_ident',
+__patched__ = ['Lock', '_allocate_lock', '_get_main_thread_ident',
                '_make_thread_handle', '_shutdown', '_sleep',
                '_start_joinable_thread', '_start_new_thread', '_ThreadHandle',
-               'currentThread', 'current_thread', 'local', 'stack_size']
+               'currentThread', 'current_thread', 'local', 'stack_size',
+               "_active", "_limbo"]
 
 __patched__ += ['get_ident', '_set_sentinel']
 
diff --git a/eventlet/hubs/asyncio.py b/eventlet/hubs/asyncio.py
index 501fdf686..2b9b7e57a 100644
--- a/eventlet/hubs/asyncio.py
+++ b/eventlet/hubs/asyncio.py
@@ -44,6 +44,19 @@ def __init__(self):
         asyncio.set_event_loop(self.loop)
         self.sleep_event = asyncio.Event()
 
+        import asyncio.events
+        if hasattr(asyncio.events, "on_fork"):
+            # Allow post-fork() child to continue using the same event loop.
+            # This is a terrible idea.
+            asyncio.events.on_fork.__code__ = (lambda: None).__code__
+        else:
+            # On Python 3.9-3.11, there's a thread local we need to reset.
+            # Also a terrible idea.
+            def re_register_loop(loop=self.loop):
+                asyncio.events._set_running_loop(loop)
+
+            os.register_at_fork(after_in_child=re_register_loop)
+
     def add_timer(self, timer):
         """
         Register a ``Timer``.
diff --git a/eventlet/patcher.py b/eventlet/patcher.py
index c3c4a8f14..b3901338b 100644
--- a/eventlet/patcher.py
+++ b/eventlet/patcher.py
@@ -432,27 +432,21 @@ def monkey_patch(**on):
                 if hasattr(orig_mod, attr_name):
                     delattr(orig_mod, attr_name)
 
-            # https://github.com/eventlet/eventlet/issues/592
             if name == "threading" and register_at_fork:
-
-                def fix_threading_active(
-                    _global_dict=_threading.current_thread.__globals__,
-                    # alias orig_mod as patched to reflect its new state
-                    # https://github.com/eventlet/eventlet/pull/661#discussion_r509877481
-                    _patched=orig_mod,
-                ):
-                    _prefork_active = [None]
-
-                    def before_fork():
-                        _prefork_active[0] = _global_dict["_active"]
-                        _global_dict["_active"] = _patched._active
-
-                    def after_fork():
-                        _global_dict["_active"] = _prefork_active[0]
-
-                    register_at_fork(before=before_fork, after_in_parent=after_fork)
-
-                fix_threading_active()
+                # The whole post-fork processing in stdlib threading.py,
+                # implemented in threading._after_fork(), is based on the
+                # assumption that threads don't survive fork(). However, green
+                # threads do survive fork, and that's what threading.py is
+                # tracking when using eventlet, so there's no need to do any
+                # post-fork cleanup in this case.
+                #
+                # So, we wipe out _after_fork()'s code so it does nothing. We
+                # can't just override it because it has already been registered
+                # with os.register_after_fork().
+                def noop():
+                    pass
+                orig_mod._after_fork.__code__ = noop.__code__
+                inject("threading", {})._after_fork.__code__ = noop.__code__
     finally:
         imp.release_lock()
 


Test patch:
diff --git a/tests/isolated/fork_in_main_thread.py b/tests/isolated/fork_in_main_thread.py
new file mode 100644
index 000000000..05f741567
--- /dev/null
+++ b/tests/isolated/fork_in_main_thread.py
@@ -0,0 +1,47 @@
+import eventlet
+
+eventlet.monkey_patch()
+
+import os
+import time
+import threading
+
+results = set()
+parent = True
+
+
+def check_current():
+    if threading.current_thread() not in threading.enumerate():
+        raise SystemExit(17)
+
+
+def background():
+    time.sleep(1)
+    check_current()
+    results.add("background")
+
+
+def forker():
+    pid = os.fork()
+    check_current()
+    if pid != 0:
+        # We're in the parent. Wait for child to die.
+        wait_pid, status = os.wait()
+        exit_code = os.waitstatus_to_exitcode(status)
+        assert wait_pid == pid
+        assert exit_code == 0, exit_code
+    else:
+        global parent
+        parent = False
+    results.add("forker")
+
+
+t = threading.Thread(target=background)
+t.start()
+forker()
+t.join()
+
+check_current()
+assert results == {"background", "forker"}, results
+if parent:
+    print("pass")
diff --git a/tests/isolated/fork_in_thread.py b/tests/isolated/fork_in_thread.py
new file mode 100644
index 000000000..c78fa7547
--- /dev/null
+++ b/tests/isolated/fork_in_thread.py
@@ -0,0 +1,49 @@
+import eventlet
+
+eventlet.monkey_patch()
+
+import os
+import time
+import threading
+
+results = []
+parent = True
+
+
+def check_current():
+    if threading.current_thread() not in threading.enumerate():
+        raise SystemExit(17)
+
+
+def background():
+    time.sleep(1)
+    check_current()
+    results.append(True)
+
+
+def forker():
+    pid = os.fork()
+    check_current()
+    if pid != 0:
+        # We're in the parent. Wait for child to die.
+        wait_pid, status = os.wait()
+        exit_code = os.waitstatus_to_exitcode(status)
+        assert wait_pid == pid
+        assert exit_code == 0, exit_code
+    else:
+        global parent
+        parent = False
+    results.append(True)
+
+
+t = threading.Thread(target=background)
+t.start()
+t2 = threading.Thread(target=forker)
+t2.start()
+t2.join()
+t.join()
+
+check_current()
+assert results == [True, True]
+if parent:
+    print("pass")
diff --git a/tests/isolated/patcher_fork_after_monkey_patch.py b/tests/isolated/patcher_fork_after_monkey_patch.py
index c8ee37683..dc9b8ee60 100644
--- a/tests/isolated/patcher_fork_after_monkey_patch.py
+++ b/tests/isolated/patcher_fork_after_monkey_patch.py
@@ -17,8 +17,12 @@ def check(n, mod, tag):
     _threading = eventlet.patcher.original('threading')
     import eventlet.green.threading
 
+    global threads_keep_running
+    threads_keep_running = True
+
     def target():
-        eventlet.sleep(0.1)
+        while threads_keep_running:
+            eventlet.sleep(0.001)
 
     threads = [
         threading.Thread(target=target, name='patched'),
@@ -31,23 +35,31 @@ def target():
     for t in threads:
         t.start()
 
-    check(2, threading, 'pre-fork patched')
+    check(5, threading, 'pre-fork patched')
     check(3, _threading, 'pre-fork original')
-    check(4, eventlet.green.threading, 'pre-fork green')
+    check(5, eventlet.green.threading, 'pre-fork green')
 
-    if os.fork() == 0:
-        # Inside the child, we should only have a main thread,
-        # but old pythons make it difficult to ensure
-        check(1, threading, 'child post-fork patched')
+    pid = os.fork()
+    if pid == 0:
+        # Inside the child, we should only have a main _OS_ thread,
+        # but green threads should survive.
+        check(5, threading, 'child post-fork patched')
         check(1, _threading, 'child post-fork original')
-        check(1, eventlet.green.threading, 'child post-fork green')
+        check(5, eventlet.green.threading, 'child post-fork green')
+        threads_keep_running = False
         sys.exit()
     else:
-        os.wait()
+        wait_pid, status = os.wait()
+        exit_code = os.waitstatus_to_exitcode(status)
+        assert wait_pid == pid
+        assert exit_code == 0, exit_code
 
-    check(2, threading, 'post-fork patched')
+    # We're in the parent now; all threads should survive:
+    check(5, threading, 'post-fork patched')
     check(3, _threading, 'post-fork original')
-    check(4, eventlet.green.threading, 'post-fork green')
+    check(5, eventlet.green.threading, 'post-fork green')
+
+    threads_keep_running = False
 
     for t in threads:
         t.join()
diff --git a/tests/patcher_test.py b/tests/patcher_test.py
index 867a75361..1a988f47d 100644
--- a/tests/patcher_test.py
+++ b/tests/patcher_test.py
@@ -514,14 +514,33 @@ def test_threadpoolexecutor():
     tests.run_isolated('patcher_threadpoolexecutor.py')
 
 
+FORK_REASON = "fork() doesn't work well on macOS, and definitely doesn't work on Windows"
+
+
 @pytest.mark.skipif(
     not sys.platform.startswith("linux"),
-    reason="fork() doesn't work well on macOS, and definitely doesn't work on Windows"
+    reason=FORK_REASON
 )
 def test_fork_after_monkey_patch():
     tests.run_isolated('patcher_fork_after_monkey_patch.py')
 
 
+@pytest.mark.skipif(
+    not sys.platform.startswith("linux"),
+    reason=FORK_REASON
+)
+def test_fork_after_monkey_patch_threading():
+    tests.run_isolated('fork_in_main_thread.py')
+
+
+@pytest.mark.skipif(
+    not sys.platform.startswith("linux"),
+    reason=FORK_REASON
+)
+def test_fork_in_thread_after_monkey_patch_threading():
+    tests.run_isolated('fork_in_thread.py')
+
+
 def test_builtin():
     tests.run_isolated('patcher_builtin.py')