Repository: cocotb/cocotb

Issue:
Stack overflow issue with lots of coroutines waiting on queues
I'm running in to an issue where I have a bunch of coroutines that are blocked on queue.get() and for some reason this is causing a stack overflow in the scheduler.  Minimal example:

    async def test_cr(queue):
        item = await queue.get()


    async def run_test(dut):
        k = 1024

        queue_list = [Queue() for k in range(k)]
        cr_list = [cocotb.scheduler.start_soon(test_cr(q)) for q in queue_list]

        await Timer(10, 'ns')

If I run that, I get something like this:

                                         
                                                                                                                         
         0.00ns ERROR    ..coroutine.Task 491.0x7fd8bc463ee0        scheduler.py:523  in _unschedule                     Exception raised by this forked coroutine
    Fatal Python error: _Py_CheckRecursiveCall: Cannot recover from stack overflow.
    Python runtime state: initialized

    Current thread 0x00007fd8bea76d00 (most recent call first):
      File "/home/alex/.local/lib/python3.9/site-packages/cocotb/triggers.py", line 396 in __init__
      File "/home/alex/.local/lib/python3.9/site-packages/cocotb/triggers.py", line 449 in wait
      File "/home/alex/.local/lib/python3.9/site-packages/cocotb/queue.py", line 126 in get
      File "/home/alex/Projects/cocotb/stack_overflow_test/test.py", line 39 in test_cr
      File "/home/alex/.local/lib/python3.9/site-packages/cocotb/outcomes.py", line 36 in send
      File "/home/alex/.local/lib/python3.9/site-packages/cocotb/decorators.py", line 201 in _advance
      File "/home/alex/.local/lib/python3.9/site-packages/cocotb/scheduler.py", line 857 in _schedule
      File "/home/alex/.local/lib/python3.9/site-packages/cocotb/scheduler.py", line 719 in add
      File "/home/alex/.local/lib/python3.9/site-packages/cocotb/scheduler.py", line 909 in _schedule
      File "/home/alex/.local/lib/python3.9/site-packages/cocotb/scheduler.py", line 719 in add
      File "/home/alex/.local/lib/python3.9/site-packages/cocotb/scheduler.py", line 909 in _schedule
      File "/home/alex/.local/lib/python3.9/site-packages/cocotb/scheduler.py", line 719 in add
      File "/home/alex/.local/lib/python3.9/site-packages/cocotb/scheduler.py", line 909 in _schedule
      File "/home/alex/.local/lib/python3.9/site-packages/cocotb/scheduler.py", line 719 in add
      File "/home/alex/.local/lib/python3.9/site-packages/cocotb/scheduler.py", line 909 in _schedule
      File "/home/alex/.local/lib/python3.9/site-packages/cocotb/scheduler.py", line 719 in add
      File "/home/alex/.local/lib/python3.9/site-packages/cocotb/scheduler.py", line 909 in _schedule
      File "/home/alex/.local/lib/python3.9/site-packages/cocotb/scheduler.py", line 719 in add
      File "/home/alex/.local/lib/python3.9/site-packages/cocotb/scheduler.py", line 909 in _schedule
      File "/home/alex/.local/lib/python3.9/site-packages/cocotb/scheduler.py", line 719 in add
      File "/home/alex/.local/lib/python3.9/site-packages/cocotb/scheduler.py", line 909 in _schedule
      File "/home/alex/.local/lib/python3.9/site-packages/cocotb/scheduler.py", line 719 in add
      File "/home/alex/.local/lib/python3.9/site-packages/cocotb/scheduler.py", line 909 in _schedule
      File "/home/alex/.local/lib/python3.9/site-packages/cocotb/scheduler.py", line 719 in add
      File "/home/alex/.local/lib/python3.9/site-packages/cocotb/scheduler.py", line 909 in _schedule
      File "/home/alex/.local/lib/python3.9/site-packages/cocotb/scheduler.py", line 719 in add

If I set k = 512, I get a different stack trace:

     0.00ns ERROR    ..coroutine.Task 491.0x7feba324e7f0        scheduler.py:523  in _unschedule                     Exception raised by this forked coroutine
     0.00ns ERROR    cocotb.regression                         regression.py:408  in _score_test                     Test Failed: run_test_001 (result was RecursionError)
                                                                                                                     Traceback (most recent call last):
                                                                                                                       File "/home/alex/Projects/cocotb/stack_overflow_test/test.py", line 39, in test_cr
                                                                                                                         item = await queue.get()
                                                                                                                       File "/home/alex/.local/lib/python3.9/site-packages/cocotb/queue.py", line 126, in get
                                                                                                                         await event.wait()
                                                                                                                       File "/home/alex/.local/lib/python3.9/site-packages/cocotb/triggers.py", line 449, in wait
                                                                                                                         return _Event(self)
                                                                                                                       File "/home/alex/.local/lib/python3.9/site-packages/cocotb/triggers.py", line 396, in __init__
                                                                                                                         PythonTrigger.__init__(self)
                                                                                                                     RecursionError: maximum recursion depth exceeded


If I set k = 256, it does not crash.  

I'm running the latest version of cocotb from git.  I ran this with icarus verilog.  I think the max recursion depth is set to 1000.  



Discussion:
This is a consequence of how the scheduler handles queued coroutines, causing recursion instead of correctly using a loop.

`_schedule()` pops the first pending coroutine and calls itself, recursively, blowing the stack if there are enough in the queue.
Is there a workaround for now?
We could move handling the pending coroutines to the event loop, after schedule and before handling pending events. I think that would be equivalent scheduling order.
I mean for @alexforencich who is using 1.5 right now. Would `fork` avoid this issue?

EDIT: Switching to `cocotb.fork` works. This is likely because it runs the coroutine until the first `await` moving the pending coroutine off the queue and onto waiting for some event. Another solution would be to run `await NullTrigger()` after every `start_soon`. This would be similar to the `await cocotb.scheduler.start(coro)` function we had talked about introducing to accompany `start_soon`.
I moved from fork to start_soon as you guys told me that fork is going to be deprecated.  Anyway, it's not a major show stopper right now, I'll just hold off these changes until this is fixed and back-ported to 1.5.x.  

Resolving patch:
diff --git a/cocotb/scheduler.py b/cocotb/scheduler.py
--- a/cocotb/scheduler.py
+++ b/cocotb/scheduler.py
@@ -458,6 +458,17 @@ def _event_loop(self, trigger):
                     # happened in gh-957)
                     del coro
 
+                # Handle any newly queued coroutines that need to be scheduled
+                while self._pending_coros:
+                    task = self._pending_coros.pop(0)
+                    if _debug:
+                        self.log.debug("Scheduling queued coroutine %s" % (task._coro.__qualname__))
+                    self._schedule(task)
+                    if _debug:
+                        self.log.debug("Scheduled queued coroutine %s" % (task._coro.__qualname__))
+
+                    del task
+
                 # Schedule may have queued up some events so we'll burn through those
                 while self._pending_events:
                     if _debug:
@@ -487,6 +498,12 @@ def unschedule(self, coro):
 
     def _unschedule(self, coro):
         """Unschedule a coroutine.  Unprime any pending triggers"""
+        if coro in self._pending_coros:
+            assert not coro.has_started()
+            self._pending_coros.remove(coro)
+            # Close coroutine so there is no RuntimeWarning that it was never awaited
+            coro.close()
+            return
 
         # Unprime the trigger this coroutine is waiting on
         trigger = coro._trigger
@@ -580,7 +597,9 @@ def queue(self, coroutine):
 
     def _queue(self, coroutine):
         """Queue a coroutine for execution"""
-        self._pending_coros.append(coroutine)
+        # Don't queue the same coroutine more than once (gh-2503)
+        if coroutine not in self._pending_coros:
+            self._pending_coros.append(coroutine)
 
     def queue_function(self, coro):
         """
@@ -904,10 +923,6 @@ def _schedule(self, coroutine, trigger=None):
                         self._pending_threads.remove(ext)
                         self._pending_events.append(ext.event)
 
-            # Handle any newly queued coroutines that need to be scheduled
-            while self._pending_coros:
-                self.add(self._pending_coros.pop(0))
-
     def finish_test(self, exc):
         """
         .. deprecated:: 1.5
@@ -964,6 +979,10 @@ def _cleanup(self):
                     self.log.debug("Killing %s" % str(coro))
                 coro.kill()
 
+        # kill any queued coroutines
+        for task in self._pending_coros:
+            task.kill()
+
         if self._main_thread is not threading.current_thread():
             raise Exception("Cleanup() called outside of the main thread")
 


Test patch:
diff --git a/tests/test_cases/test_cocotb/test_concurrency_primitives.py b/tests/test_cases/test_cocotb/test_concurrency_primitives.py
--- a/tests/test_cases/test_cocotb/test_concurrency_primitives.py
+++ b/tests/test_cases/test_cocotb/test_concurrency_primitives.py
@@ -149,3 +149,19 @@ async def test_event_is_set(dut):
     assert e.is_set()
     e.clear()
     assert not e.is_set()
+
+
+@cocotb.test()
+async def test_combine_start_soon(_):
+    async def coro(delay):
+        start_time = cocotb.utils.get_sim_time(units="ns")
+        await Timer(delay, "ns")
+        assert cocotb.utils.get_sim_time(units="ns") == start_time + delay
+
+    max_delay = 10
+
+    coros = [cocotb.scheduler.start_soon(coro(d)) for d in range(1, max_delay + 1)]
+
+    test_start = cocotb.utils.get_sim_time(units="ns")
+    await Combine(*coros)
+    assert cocotb.utils.get_sim_time(units="ns") == test_start + max_delay
diff --git a/tests/test_cases/test_cocotb/test_scheduler.py b/tests/test_cases/test_cocotb/test_scheduler.py
--- a/tests/test_cases/test_cocotb/test_scheduler.py
+++ b/tests/test_cases/test_cocotb/test_scheduler.py
@@ -174,6 +174,21 @@ async def null_coroutine():
     await Timer(100, "ns")
 
 
+@cocotb.test()
+async def test_stack_overflow_pending_coros(dut):
+    """
+    Test against stack overflow when queueing many pending coroutines
+    before yielding to scheduler.
+    """
+    # gh-2489
+    async def simple_coroutine():
+        await Timer(10, "step")
+
+    coros = [cocotb.scheduler.start_soon(simple_coroutine()) for _ in range(1024)]
+
+    await Combine(*coros)
+
+
 @cocotb.test()
 async def test_kill_coroutine_waiting_on_the_same_trigger(dut):
     # gh-1348
@@ -426,3 +441,82 @@ async def example():
     assert a == 0
     await NullTrigger()
     assert a == 1
+
+
+@cocotb.test()
+async def test_start_soon_scheduling(dut):
+    """Test order of scheduling when using start_soon."""
+    coro_scheduled = False
+
+    def react_wrapper(trigger):
+        """Function to prime trigger with."""
+        log = logging.getLogger("cocotb.test")
+        log.debug("react_wrapper start")
+        assert coro_scheduled is False
+        cocotb.scheduler._react(trigger)
+        assert coro_scheduled is True
+        log.debug("react_wrapper end")
+
+    async def coro():
+        nonlocal coro_scheduled
+        coro_scheduled = True
+
+    t = Timer(1, 'step')
+    # pre-prime with wrapper function instead of letting scheduler prime it normally
+    t.prime(react_wrapper)
+    await t
+    cocotb.scheduler.start_soon(coro())
+    await Timer(1, 'step')  # await a GPITrigger to ensure control returns to simulator
+    assert coro_scheduled is True
+
+
+@cocotb.test()
+async def test_await_start_soon(_):
+    """Test awaiting start_soon queued coroutine before it starts."""
+    async def coro():
+        start_time = cocotb.utils.get_sim_time(units="ns")
+        await Timer(1, "ns")
+        assert cocotb.utils.get_sim_time(units="ns") == start_time + 1
+
+    coro = cocotb.scheduler.start_soon(coro())
+
+    await coro
+
+
+@cocotb.test()
+async def test_kill_start_soon_task(_):
+    """Test killing task queued by start_soon."""
+    coro_scheduled = False
+
+    async def coro():
+        nonlocal coro_scheduled
+        coro_scheduled = True
+
+    task = cocotb.scheduler.start_soon(coro())
+    task.kill()
+
+    await NullTrigger()
+    assert coro_scheduled is False
+    assert task._finished
+
+
+start_soon_started = False
+
+
+@cocotb.test()
+async def test_test_end_after_start_soon(_):
+    """Test ending test before start_soon queued coroutine starts."""
+    async def coro():
+        global start_soon_started
+        start_soon_started = True
+
+    cocotb.scheduler.start_soon(coro())
+
+
+@cocotb.test()
+async def test_previous_start_soon_not_scheduled(_):
+    """Test that queued coroutine from previous test did not run.
+
+    NOTE: This test must be after test_test_end_after_start_soon.
+    """
+    assert start_soon_started is False