Repository: PrefectHQ/prefect

Issue:
Importing prefect fails in a thread in python <=3.9
### First check

- [X] I added a descriptive title to this issue.
- [X] I used the GitHub search to find a similar issue and didn't find it.
- [X] I searched the Prefect documentation for this issue.
- [X] I checked that this issue is related to Prefect and not one of its dependencies.

### Bug summary

In python 3.8-3.9, we can't import prefect from a thread other than the main thread.

In our production code we use flask to create our app, which imports the rest of the code in a worker thread. One of our modules uses `from prefect.runtime import flow_run` to understand whether we're currently in a flow run or not.

The problem seems to happen because prefect creates an asyncio.Semaphore at import time in prefect/deployments/base.py:485
```python
OPEN_FILE_SEMAPHORE = asyncio.Semaphore(math.floor(get_open_file_limit() * 0.5))
```

Internally this leads to a `.get_event_loop()` in python <=3.9, which raises exception. It seems to be fixed in later versions of python.

### Reproduction

```python3
from concurrent.futures import ThreadPoolExecutor

def work():
    import prefect

with ThreadPoolExecutor() as pool:
    pool.submit(work).result()
```


### Error

```python3
Traceback (most recent call last):
  File "/tmp/main.py", line 9, in <module>
    pool.submit(work).result()
  File "/opt/homebrew/Cellar/python@3.9/3.9.18/Frameworks/Python.framework/Versions/3.9/lib/python3.9/concurrent/futures/_base.py", line 446, in result
    return self.__get_result()
  File "/opt/homebrew/Cellar/python@3.9/3.9.18/Frameworks/Python.framework/Versions/3.9/lib/python3.9/concurrent/futures/_base.py", line 391, in __get_result
    raise self._exception
  File "/opt/homebrew/Cellar/python@3.9/3.9.18/Frameworks/Python.framework/Versions/3.9/lib/python3.9/concurrent/futures/thread.py", line 58, in run
    result = self.fn(*self.args, **self.kwargs)
  File "/tmp/main.py", line 5, in work
    import prefect  # noqa
  File "/tmp/.venv/lib/python3.9/site-packages/prefect/__init__.py", line 37, in <module>
    from prefect.runner import Runner, serve
  File "/tmp/.venv/lib/python3.9/site-packages/prefect/runner.py", line 56, in <module>
    from prefect.deployments.runner import RunnerDeployment
  File "/tmp/.venv/lib/python3.9/site-packages/prefect/deployments/__init__.py", line 1, in <module>
    import prefect.deployments.base
  File "/tmp/.venv/lib/python3.9/site-packages/prefect/deployments/base.py", line 485, in <module>
    OPEN_FILE_SEMAPHORE = asyncio.Semaphore(math.floor(get_open_file_limit() * 0.5))
  File "/opt/homebrew/Cellar/python@3.9/3.9.18/Frameworks/Python.framework/Versions/3.9/lib/python3.9/asyncio/locks.py", line 375, in __init__
    self._loop = events.get_event_loop()
  File "/opt/homebrew/Cellar/python@3.9/3.9.18/Frameworks/Python.framework/Versions/3.9/lib/python3.9/asyncio/events.py", line 642, in get_event_loop
    raise RuntimeError('There is no current event loop in thread %r.'
RuntimeError: There is no current event loop in thread 'ThreadPoolExecutor-0_0'.
```


### Versions

```Text
Version:             2.13.4
API version:         0.8.4
Python version:      3.9.18
Git commit:          48d9d9a2
Built:               Fri, Sep 29, 2023 11:26 AM
OS/Arch:             darwin/arm64
Profile:             default
Server type:         ephemeral
Server:
  Database:          sqlite
  SQLite version:    3.43.0
```


### Additional context

_No response_


Discussion:
Hi @ashtuchkin thanks for filing this issue! I was able to reproduce what you are seeing. We'll take a look at this.

Resolving patch:
diff --git a/src/prefect/deployments/base.py b/src/prefect/deployments/base.py
--- a/src/prefect/deployments/base.py
+++ b/src/prefect/deployments/base.py
@@ -31,7 +31,7 @@
 from prefect.flows import load_flow_from_entrypoint
 from prefect.logging import get_logger
 from prefect.settings import PREFECT_DEBUG_MODE
-from prefect.utilities.asyncutils import run_sync_in_worker_thread
+from prefect.utilities.asyncutils import LazySemaphore, run_sync_in_worker_thread
 from prefect.utilities.filesystem import create_default_ignore_file, get_open_file_limit
 from prefect.utilities.templating import apply_values
 
@@ -489,7 +489,7 @@ def _save_deployment_to_prefect_file(
 
 # Only allow half of the open file limit to be open at once to allow for other
 # actors to open files.
-OPEN_FILE_SEMAPHORE = asyncio.Semaphore(math.floor(get_open_file_limit() * 0.5))
+OPEN_FILE_SEMAPHORE = LazySemaphore(lambda: math.floor(get_open_file_limit() * 0.5))
 
 
 async def _find_flow_functions_in_file(filename: str) -> List[Dict]:
diff --git a/src/prefect/utilities/asyncutils.py b/src/prefect/utilities/asyncutils.py
--- a/src/prefect/utilities/asyncutils.py
+++ b/src/prefect/utilities/asyncutils.py
@@ -416,3 +416,22 @@ async def gather(*calls: Callable[[], Coroutine[Any, Any, T]]) -> List[T]:
         for call in calls:
             keys.append(tg.start_soon(call))
     return [tg.get_result(key) for key in keys]
+
+
+class LazySemaphore:
+    def __init__(self, initial_value_func):
+        self._semaphore = None
+        self._initial_value_func = initial_value_func
+
+    async def __aenter__(self):
+        self._initialize_semaphore()
+        await self._semaphore.__aenter__()
+        return self._semaphore
+
+    async def __aexit__(self, exc_type, exc, tb):
+        await self._semaphore.__aexit__(exc_type, exc, tb)
+
+    def _initialize_semaphore(self):
+        if self._semaphore is None:
+            initial_value = self._initial_value_func()
+            self._semaphore = asyncio.Semaphore(initial_value)


Test patch:
diff --git a/tests/deployment/test_base.py b/tests/deployment/test_base.py
--- a/tests/deployment/test_base.py
+++ b/tests/deployment/test_base.py
@@ -17,6 +17,7 @@
     register_flow,
 )
 from prefect.settings import PREFECT_DEBUG_MODE, temporary_settings
+from prefect.utilities.asyncutils import run_sync_in_worker_thread
 
 TEST_PROJECTS_DIR = prefect.__development_base_path__ / "tests" / "test-projects"
 
@@ -280,3 +281,13 @@ async def test_find_flow_functions_in_file_returns_empty_list_on_file_error(
         with temporary_settings({PREFECT_DEBUG_MODE: True}):
             assert await _find_flow_functions_in_file("foo.py") == []
             assert "Could not open foo.py" in caplog.text
+
+    async def test_prefect_can_be_imported_from_non_main_thread(self):
+        """testing due to `asyncio.Semaphore` error when importing prefect from a worker thread
+        in python <= 3.9
+        """
+
+        def import_prefect():
+            import prefect  # noqa: F401
+
+        await run_sync_in_worker_thread(import_prefect)
diff --git a/tests/utilities/test_asyncutils.py b/tests/utilities/test_asyncutils.py
--- a/tests/utilities/test_asyncutils.py
+++ b/tests/utilities/test_asyncutils.py
@@ -10,6 +10,7 @@
 
 from prefect.utilities.asyncutils import (
     GatherIncomplete,
+    LazySemaphore,
     add_event_loop_shutdown_callback,
     create_gather_task_group,
     gather,
@@ -431,3 +432,21 @@ async def test_gather_task_group_get_result_bad_uuid():
 
     with pytest.raises(KeyError):
         tg.get_result(uuid.uuid4())
+
+
+async def test_lazy_semaphore_initialization():
+    initial_value = 5
+    lazy_semaphore = LazySemaphore(lambda: initial_value)
+
+    assert lazy_semaphore._semaphore is None
+
+    lazy_semaphore._initialize_semaphore()
+
+    assert lazy_semaphore._semaphore._value == initial_value
+
+    async with lazy_semaphore as semaphore:
+        assert lazy_semaphore._semaphore is not None
+        assert isinstance(semaphore, asyncio.Semaphore)
+        assert lazy_semaphore._semaphore._value == initial_value - 1
+
+    assert lazy_semaphore._semaphore._value == initial_value