Repository: pypa/setuptools

Issue:
[BUG] 60.7.0 breaks gevent monkeypatching
### setuptools version

setuptools>=60.7.0

### Python version

Python 3.x (tested on 3.9 & 3.10)

### OS

Any (tested on Linux and MacOS)

### Additional environment information

_No response_

### Description

Recent changes in Setuptools, introduced in version 60.7.0, break `gevent` concurrent networking library. When `setuptools>=60.7.0` is installed, `gevent` is unable to monkey-patch the standard library and freezes in runtime.

There are more detail in https://github.com/gevent/gevent/issues/1865 and the script used in reproduction steps comes from that issue.

Gevent monkey-patches networking and concurrency functions in standard library to replace them with versions that allow other green threads to run when the function is waiting for I/O. This monkey-patching needs to happen before any of the patched modules are loaded.

One of the changes in Setuptools 60.7.0 was using a vendored `more_itertools` library. This library, in turn, imports `concurrent.futures.threading`, which is one of the modules monkey-patched by gevent. Gevent code includes a `from pkg_resources import iter_entry_points` line, which triggers `more_itertools` import when gevent is imported, which creates a chicken-and-egg problem.

This is fixed by a following quick-and-dirty patch to `setuptools`, from which I will prepare a proper pull request. I'm sending a bug report first to provide context, because it will require a change in `jaraco.text` too. Note that this is a proof of concept, not a final proposed change.

<details>
<summary><kbd>git diff</kbd></summary>

```diff
diff --git i/pkg_resources/_vendor/jaraco/functools.py w/pkg_resources/_vendor/jaraco/functools.py
index a3fea3a1..e8f47eb8 100644
--- i/pkg_resources/_vendor/jaraco/functools.py
+++ w/pkg_resources/_vendor/jaraco/functools.py
@@ -5,14 +5,18 @@ import collections
 import types
 import itertools
 
-import pkg_resources.extern.more_itertools
-
 from typing import Callable, TypeVar
 
 
 CallableT = TypeVar("CallableT", bound=Callable[..., object])
 
 
+def _consume(iterator):
+    """Consume iterator (itertools recipe)"""
+    # feed the entire iterator into a zero-length deque
+    collections.deque(iterator, maxlen=0)
+
+
 def compose(*funcs):
     """
     Compose any number of unary functions into a single unary function.
@@ -385,7 +389,7 @@ def print_yielded(func):
     None
     """
     print_all = functools.partial(map, print)
-    print_results = compose(more_itertools.consume, print_all, func)
+    print_results = compose(_consume, print_all, func)
     return functools.wraps(func)(print_results)
 
 
diff --git i/setuptools/_vendor/jaraco/functools.py w/setuptools/_vendor/jaraco/functools.py
index bbd8b29f..e8f47eb8 100644
--- i/setuptools/_vendor/jaraco/functools.py
+++ w/setuptools/_vendor/jaraco/functools.py
@@ -5,14 +5,18 @@ import collections
 import types
 import itertools
 
-import setuptools.extern.more_itertools
-
 from typing import Callable, TypeVar
 
 
 CallableT = TypeVar("CallableT", bound=Callable[..., object])
 
 
+def _consume(iterator):
+    """Consume iterator (itertools recipe)"""
+    # feed the entire iterator into a zero-length deque
+    collections.deque(iterator, maxlen=0)
+
+
 def compose(*funcs):
     """
     Compose any number of unary functions into a single unary function.
@@ -385,7 +389,7 @@ def print_yielded(func):
     None
     """
     print_all = functools.partial(map, print)
-    print_results = compose(more_itertools.consume, print_all, func)
+    print_results = compose(_consume, print_all, func)
     return functools.wraps(func)(print_results)
 
 
diff --git i/setuptools/command/build_py.py w/setuptools/command/build_py.py
index c3fdc092..a946d1fd 100644
--- i/setuptools/command/build_py.py
+++ w/setuptools/command/build_py.py
@@ -8,7 +8,7 @@ import io
 import distutils.errors
 import itertools
 import stat
-from setuptools.extern.more_itertools import unique_everseen
+from setuptools.extern.more_itertools.recipes import unique_everseen
 
 
 def make_writable(target):
diff --git i/setuptools/command/test.py w/setuptools/command/test.py
index 4a389e4d..17bc69ee 100644
--- i/setuptools/command/test.py
+++ w/setuptools/command/test.py
@@ -19,7 +19,7 @@ from pkg_resources import (
     EntryPoint,
 )
 from setuptools import Command
-from setuptools.extern.more_itertools import unique_everseen
+from setuptools.extern.more_itertools.recipes import unique_everseen
 
 
 class ScanningLoader(TestLoader):
diff --git i/setuptools/dist.py w/setuptools/dist.py
index 733ae14f..ef00e417 100644
--- i/setuptools/dist.py
+++ w/setuptools/dist.py
@@ -28,7 +28,7 @@ from distutils.util import rfc822_escape
 
 from setuptools.extern import packaging
 from setuptools.extern import ordered_set
-from setuptools.extern.more_itertools import unique_everseen
+from setuptools.extern.more_itertools.recipes import unique_everseen
 
 from . import SetuptoolsDeprecationWarning
 
diff --git i/setuptools/msvc.py w/setuptools/msvc.py
index 281ea1c2..5908a06e 100644
--- i/setuptools/msvc.py
+++ w/setuptools/msvc.py
@@ -30,7 +30,7 @@ import itertools
 import subprocess
 import distutils.errors
 from setuptools.extern.packaging.version import LegacyVersion
-from setuptools.extern.more_itertools import unique_everseen
+from setuptools.extern.more_itertools.recipes import unique_everseen
 
 from .monkey import get_unpatched
 
diff --git i/setuptools/package_index.py w/setuptools/package_index.py
index 051e523a..f85a4b5e 100644
--- i/setuptools/package_index.py
+++ w/setuptools/package_index.py
@@ -27,7 +27,7 @@ from distutils import log
 from distutils.errors import DistutilsError
 from fnmatch import translate
 from setuptools.wheel import Wheel
-from setuptools.extern.more_itertools import unique_everseen
+from setuptools.extern.more_itertools.recipes import unique_everseen
 
 
 EGG_FRAGMENT = re.compile(r'^egg=([-A-Za-z0-9_.+!]+)$')
``` 

</details>

- `unique_everseen` is imported directly from the `recipes` submodule which doesn't import `concurrency`. I'm not sure it's necessary or that it doesn't import `more_itertools.more` via parent module anyway. This function is 16 lines copied from stdlib documentation, so in final PR I'll probably consider vendoring just this one function
- `consume` is one-liner from stdlib documentation (only this branch of upstream implementation's `if` is used), so it was pasted directly into source
- The `more_itertools` library is the largest part of 60.7.0 diff. It's about 5KLOC, vendored twice (in `setuptools` and in `pkg_resources`). Out of this amount of code, only two functions are used; one is a one-liner `consume` and the other is 16 lines of code.

The freezes can and should be fixed in `gevent` (by using `importlib.metadata` / `importlib_metadata` on Python 3; if I'm right, this version of Setuptools is Python 3 only, so `pkg_resources` on Python 2.7 should be safe), but I believe that at the same time it requires a fix on Setuptools side, for the following reasons:

- Minimize side effects of simply importing a library as low-level and fundamental as `setuptools` / `pkg_resources`
- Reduce size of codebase by removing second largest vendored dependency `more_itertools` (or postprocessing it in `tools/vendored.py` to remove the problematic `more_itertools.more` and keeping only `more_itertools.recipes`)

### Expected behavior

I expected that the test code from "How to Reproduce" section will run.

### How to Reproduce

1. Install `gevent` and `setuptools>=60.7.0`
2. Run the following code

```python
from gevent import monkey
import gevent

if __name__ == "__main__":
    monkey.patch_thread()
    monkey.patch_subprocess()

    import subprocess
    from concurrent.futures import ThreadPoolExecutor

    def func():
        # gevent.sleep(0) # uncomment to fail
        return subprocess.run(["ls", "-l", "/dev/null"], capture_output=True)

    with ThreadPoolExecutor(max_workers=5) as executor:
        fut = executor.submit(func)
        print(fut.result())
```

The script will freeze.

### Output

```console
% python3 -m venv ./venv
% . ./venv/bin/activate
(venv) % pip install -U pip gevent setuptools==60.6.0
Requirement already satisfied: pip in ./venv/lib/python3.9/site-packages (21.3.1)
Collecting pip
  Using cached pip-22.0.3-py3-none-any.whl (2.1 MB)
Collecting gevent
  Using cached gevent-21.12.0-cp39-cp39-macosx_12_0_arm64.whl
Collecting setuptools==60.6.0
  Using cached setuptools-60.6.0-py3-none-any.whl (953 kB)
Collecting greenlet<2.0,>=1.1.0
  Using cached greenlet-1.1.2-cp39-cp39-macosx_12_0_arm64.whl
Collecting zope.interface
  Using cached zope.interface-5.4.0-cp39-cp39-macosx_12_0_arm64.whl
Collecting zope.event
  Using cached zope.event-4.5.0-py2.py3-none-any.whl (6.8 kB)
Installing collected packages: setuptools, zope.interface, zope.event, greenlet, pip, gevent
  Attempting uninstall: setuptools
    Found existing installation: setuptools 60.5.0
    Uninstalling setuptools-60.5.0:
      Successfully uninstalled setuptools-60.5.0
  Attempting uninstall: pip
    Found existing installation: pip 21.3.1
    Uninstalling pip-21.3.1:
      Successfully uninstalled pip-21.3.1
Successfully installed gevent-21.12.0 greenlet-1.1.2 pip-22.0.3 setuptools-60.6.0 zope.event-4.5.0 zope.interface-5.4.0
(venv) % cat test.py
from gevent import monkey
import gevent

if __name__ == "__main__":
    monkey.patch_thread()
    monkey.patch_subprocess()

    import subprocess
    from concurrent.futures import ThreadPoolExecutor

    def func():
        # gevent.sleep(0) # uncomment to fail
        return subprocess.run(["ls", "-l", "/dev/null"], capture_output=True)

    with ThreadPoolExecutor(max_workers=5) as executor:
        fut = executor.submit(func)
        print(fut.result())
(venv) % python test.py
CompletedProcess(args=['ls', '-l', '/dev/null'], returncode=0, stdout=b'crw-rw-rw-  1 root  wheel  0x3000002 Feb  8 17:38 /dev/null\n', stderr=b'')
(venv) % pip install -U setuptools
Requirement already satisfied: setuptools in ./venv/lib/python3.9/site-packages (60.6.0)
Collecting setuptools
  Using cached setuptools-60.8.1-py3-none-any.whl (1.1 MB)
Installing collected packages: setuptools
  Attempting uninstall: setuptools
    Found existing installation: setuptools 60.6.0
    Uninstalling setuptools-60.6.0:
      Successfully uninstalled setuptools-60.6.0
Successfully installed setuptools-60.8.1
(venv) % python test.py
# Script freezes, interrupted with Ctrl+C
^CException ignored in: <built-in method acquire of _thread.lock object at 0x1037e26f0>
Traceback (most recent call last):
  File "/Users/maciejp/Src/github.com/HealthByRo/tmp/testink/venv/lib/python3.9/site-packages/gevent/os.py", line 431, in fork_and_watch
    pid = fork()
KeyboardInterrupt:
Traceback (most recent call last):
  File "/Users/maciejp/Src/github.com/HealthByRo/tmp/testink/test.py", line 16, in <module>
    fut = executor.submit(func)
  File "/opt/homebrew/Cellar/python@3.9/3.9.10/Frameworks/Python.framework/Versions/3.9/lib/python3.9/concurrent/futures/thread.py", line 177, in submit
    return f
RuntimeError: release unlocked lock
(venv) %

```



Discussion:

Resolving patch:
diff --git a/pkg_resources/_vendor/more_itertools/more.py b/pkg_resources/_vendor/more_itertools/more.py
--- a/pkg_resources/_vendor/more_itertools/more.py
+++ b/pkg_resources/_vendor/more_itertools/more.py
@@ -2,7 +2,6 @@
 
 from collections import Counter, defaultdict, deque, abc
 from collections.abc import Sequence
-from concurrent.futures import ThreadPoolExecutor
 from functools import partial, reduce, wraps
 from heapq import merge, heapify, heapreplace, heappop
 from itertools import (
@@ -3656,7 +3655,7 @@ def __init__(self, func, callback_kwd='callback', wait_seconds=0.1):
         self._aborted = False
         self._future = None
         self._wait_seconds = wait_seconds
-        self._executor = ThreadPoolExecutor(max_workers=1)
+        self._executor = __import__("concurrent.futures").futures.ThreadPoolExecutor(max_workers=1)
         self._iterator = self._reader()
 
     def __enter__(self):
diff --git a/setuptools/_vendor/more_itertools/more.py b/setuptools/_vendor/more_itertools/more.py
--- a/setuptools/_vendor/more_itertools/more.py
+++ b/setuptools/_vendor/more_itertools/more.py
@@ -2,7 +2,6 @@
 
 from collections import Counter, defaultdict, deque, abc
 from collections.abc import Sequence
-from concurrent.futures import ThreadPoolExecutor
 from functools import partial, reduce, wraps
 from heapq import merge, heapify, heapreplace, heappop
 from itertools import (
@@ -3454,7 +3453,7 @@ def __init__(self, func, callback_kwd='callback', wait_seconds=0.1):
         self._aborted = False
         self._future = None
         self._wait_seconds = wait_seconds
-        self._executor = ThreadPoolExecutor(max_workers=1)
+        self._executor = __import__("concurrent.futures").futures.ThreadPoolExecutor(max_workers=1)
         self._iterator = self._reader()
 
     def __enter__(self):
diff --git a/tools/vendored.py b/tools/vendored.py
--- a/tools/vendored.py
+++ b/tools/vendored.py
@@ -64,6 +64,21 @@ def rewrite_importlib_resources(pkg_files, new_root):
         file.write_text(text)
 
 
+def rewrite_more_itertools(pkg_files: Path):
+    """
+    Defer import of concurrent.futures. Workaround for #3090.
+    """
+    more_file = pkg_files.joinpath('more.py')
+    text = more_file.read_text()
+    text = re.sub(r'^.*concurrent.futures.*?\n', '', text, flags=re.MULTILINE)
+    text = re.sub(
+        'ThreadPoolExecutor',
+        '__import__("concurrent.futures").futures.ThreadPoolExecutor',
+        text,
+    )
+    more_file.write_text(text)
+
+
 def clean(vendor):
     """
     Remove all files out of the vendor directory except the meta
@@ -96,6 +111,7 @@ def update_pkg_resources():
     rewrite_jaraco_text(vendor / 'jaraco/text', 'pkg_resources.extern')
     rewrite_jaraco(vendor / 'jaraco', 'pkg_resources.extern')
     rewrite_importlib_resources(vendor / 'importlib_resources', 'pkg_resources.extern')
+    rewrite_more_itertools(vendor / "more_itertools")
 
 
 def update_setuptools():
@@ -105,6 +121,7 @@ def update_setuptools():
     rewrite_jaraco_text(vendor / 'jaraco/text', 'setuptools.extern')
     rewrite_jaraco(vendor / 'jaraco', 'setuptools.extern')
     rewrite_importlib_resources(vendor / 'importlib_resources', 'setuptools.extern')
+    rewrite_more_itertools(vendor / "more_itertools")
 
 
 __name__ == '__main__' and update_vendored()


Test patch: