Repository: pytest-dev/pytest-xdist
Pull request #1027 (2024-02-25 21:17:45)

Use execnet main_thread_only execmodel
Use the execnet `main_thread_only `execmodel so that code which expects to run in the main thread will just work.  This execmodel has been merged to the execnet master branch via pytest-dev/execnet#243, so this patch should not be merged until there is a released version of execnet supporting the `main_thread_only` execmodel.

Also increase minimum python version to 3.8 since execnet dropped 3.7 support in pytest-dev/execnet#245.

Closes: pytest-dev/pytest-xdist#620

bluetech: @zmedico we now depend on execnet>=2.1.0 in master so I think this unblocked. @zmedico can you rebase? Note that master already dropped py3.7 as well so you can remove these changes from the PR changelog notes.
bluetech: @zmedico The just-merged typing PR created other conflicts (sorry about that, I forgot). Can you rebase again? (if you want I can do it as well).

@nicoddemus You marked "requested changes" but now that we depend on new execnet I assume you're good with merging?

After this is merged I can do the 3.6.0 release.

Patch:
diff --git a/changelog/1027.improvement.rst b/changelog/1027.improvement.rst
new file mode 100644
index 0000000..2e6dca2
--- /dev/null
+++ b/changelog/1027.improvement.rst
@@ -0,0 +1,3 @@
+``pytest-xdist`` workers now always execute the tests in the main thread.
+
+Previously some tests might end up executing in a separate thread other than ``main`` in the workers, due to some internal `èxecnet`` details. This can cause problems specially with async frameworks where the event loop is running in the ``main`` thread (for example `#620 <https://github.com/pytest-dev/pytest-xdist/issues/620>`__).
diff --git a/changelog/620.bugfix b/changelog/620.bugfix
new file mode 100644
index 0000000..81b3378
--- /dev/null
+++ b/changelog/620.bugfix
@@ -0,0 +1,1 @@
+Use the ``execnet`` new ``main_thread_only`` "execmodel" so that code which expects to only run in the main thread will now work as expected.
diff --git a/src/xdist/looponfail.py b/src/xdist/looponfail.py
index 7e30e4c..4826c70 100644
--- a/src/xdist/looponfail.py
+++ b/src/xdist/looponfail.py
@@ -82,7 +82,7 @@ class RemoteControl:
             print("RemoteControl:", msg)
 
     def initgateway(self) -> execnet.Gateway:
-        return execnet.makegateway("popen")
+        return execnet.makegateway("execmodel=main_thread_only//popen")
 
     def setup(self) -> None:
         if hasattr(self, "gateway"):
diff --git a/src/xdist/workermanage.py b/src/xdist/workermanage.py
index 70d9597..5570e1d 100644
--- a/src/xdist/workermanage.py
+++ b/src/xdist/workermanage.py
@@ -56,13 +56,15 @@ class NodeManager:
         self.testrunuid = self.config.getoption("testrunuid")
         if self.testrunuid is None:
             self.testrunuid = uuid.uuid4().hex
-        self.group = execnet.Group()
+        self.group = execnet.Group(execmodel="main_thread_only")
         if specs is None:
             specs = self._getxspecs()
         self.specs: list[execnet.XSpec] = []
         for spec in specs:
             if not isinstance(spec, execnet.XSpec):
                 spec = execnet.XSpec(spec)
+            if getattr(spec, "execmodel", None) != "main_thread_only":
+                spec = execnet.XSpec(f"execmodel=main_thread_only//{spec}")
             if not spec.chdir and not spec.popen:
                 spec.chdir = defaultchdir
             self.group.allocate_id(spec)
@@ -90,6 +92,8 @@ class NodeManager:
         spec: execnet.XSpec,
         putevent: Callable[[tuple[str, dict[str, Any]]], None],
     ) -> WorkerController:
+        if getattr(spec, "execmodel", None) != "main_thread_only":
+            spec = execnet.XSpec(f"execmodel=main_thread_only//{spec}")
         gw = self.group.makegateway(spec)
         self.config.hook.pytest_xdist_newgateway(gateway=gw)
         self.rsync_roots(gw)


Test patch:
diff --git a/testing/test_remote.py b/testing/test_remote.py
index cbbf758..0b0334d 100644
--- a/testing/test_remote.py
+++ b/testing/test_remote.py
@@ -49,7 +49,7 @@ class WorkerSetup:
     def setup(self) -> None:
         self.pytester.chdir()
         # import os ; os.environ['EXECNET_DEBUG'] = "2"
-        self.gateway = execnet.makegateway()
+        self.gateway = execnet.makegateway("execmodel=main_thread_only//popen")
         self.config = config = self.pytester.parseconfigure()
         putevent = self.events.put if self.use_callback else None
 
diff --git a/testing/test_workermanage.py b/testing/test_workermanage.py
index 08b3885..491d042 100644
--- a/testing/test_workermanage.py
+++ b/testing/test_workermanage.py
@@ -83,7 +83,7 @@ class TestNodeManagerPopen:
         assert len(call.specs) == 2
 
         call = hookrecorder.popcall("pytest_xdist_newgateway")
-        assert call.gateway.spec == execnet.XSpec("popen")
+        assert call.gateway.spec == execnet.XSpec("execmodel=main_thread_only//popen")
         assert call.gateway.id == "gw0"
         call = hookrecorder.popcall("pytest_xdist_newgateway")
         assert call.gateway.id == "gw1"
@@ -177,7 +177,7 @@ class TestHRSync:
         assert names == {"dir", "file.txt", "somedir"}
 
     def test_hrsync_one_host(self, source: Path, dest: Path) -> None:
-        gw = execnet.makegateway("popen//chdir=%s" % dest)
+        gw = execnet.makegateway("execmodel=main_thread_only//popen//chdir=%s" % dest)
         finished = []
         rsync = HostRSync(source)
         rsync.add_target_host(gw, finished=lambda: finished.append(1))