# Buggy code
async def add(self, tasks: Tuple[TTask, ...]) -> None:
    """
    add() will insert as many tasks as can be inserted until the queue fills up.
    Then it will pause until the queue is no longer full, and continue adding tasks.
    It will finally return when all tasks have been inserted.
    """
    if not isinstance(tasks, tuple):
        raise ValidationError(f"must pass a tuple of tasks to add(), but got {tasks!r}")

    already_pending = self._tasks.intersection(tasks)
    if already_pending:
        raise ValidationError(
            f"Duplicate tasks detected: {already_pending!r} are already present in the queue"
        )

    # make sure to insert the highest-priority items first, in case queue fills up
    remaining = tuple(sorted(map(self._task_wrapper, tasks)))

    while remaining:
        num_tasks = len(self._tasks)

        if self._maxsize <= 0:
            # no cap at all, immediately insert all tasks
            open_slots = len(remaining)
        elif num_tasks < self._maxsize:
            # there is room to add at least one more task
            open_slots = self._maxsize - num_tasks
        else:
            # wait until there is room in the queue
            await self._full_lock.acquire()

            # the current number of tasks has changed, restart attempt
            continue

        queueing, remaining = remaining[:open_slots], remaining[open_slots:]

        for task in queueing:
            # There will always be room in _open_queue until _maxsize is reached
            try:
                self._open_queue.put_nowait(task)
            except asyncio.QueueFull as exc:
                task_idx = queueing.index(task)
                qsize = self._open_queue.qsize()
                raise asyncio.QueueFull(
                    f"TaskQueue unsuccessful in adding task {task.original!r} ",
                    f"because qsize={qsize}, "
                    f"num_tasks={num_tasks}, maxsize={self._maxsize}, open_slots={open_slots}, "
                    f"num queueing={len(queueing)}, len(_tasks)={len(self._tasks)}, task_idx="
                    f"{task_idx}, queuing={queueing}, original msg: {exc}",
                )

        original_queued = tuple(task.original for task in queueing)
        self._tasks.update(original_queued)

        if self._full_lock.locked() and len(self._tasks) < self._maxsize:
            self._full_lock.release()


# Traceback (trio.MultiError)
DEBUG  2020-09-03 21:51:33,790        SkeletonSyncer  Skeleton sync with ETHPeer (eth, 63) <Session <Node(0x1fbb02@34.239.157.14)> a55e806f-fd96-4963-8e5f-73243d2a6820> ended
DEBUG  2020-09-03 21:51:33,791        SkeletonSyncer  Skeleton syncer had 0 pending headers when it was cancelled
DEBUG  2020-09-03 21:51:33,789         MuirGlacierVM  Beam pivot over 1 txn preview for <BlockHeader #10790870 55f0d7cd> b/c StateUnretrievable('No servers for CollectMissingAccount') after 0.0s, %exec 100, stats: BeamStat: accts=0, a_nodes=0, codes=0, strg=0, s_nodes=0, nodes=0, rtt=0.000s, wait=0s
�[1m�[31m   ERROR  2020-09-03 21:51:33,796               asyncio  Exception in callback <TaskWakeupMethWrapper object at 0x7fa4481cd710>
handle: <Handle <TaskWakeupMethWrapper object at 0x7fa4481cd710>>
Traceback (most recent call last):
File "uvloop/cbhandles.pyx", line 70, in uvloop.loop.Handle._run
File "/usr/local/lib/python3.7/site-packages/async_service/asyncio.py", line 122, in _handle_cancelled
await self._real_handle_cancelled()
File "/usr/local/lib/python3.7/site-packages/async_service/asyncio.py", line 173, in _real_handle_cancelled
await asyncio_task
File "uvloop/cbhandles.pyx", line 70, in uvloop.loop.Handle._run
File "/usr/local/lib/python3.7/site-packages/async_service/base.py", line 300, in _run_and_manage_task
await task.run()
File "/usr/local/lib/python3.7/site-packages/async_service/base.py", line 165, in run
await self.child_manager.run()
File "/usr/local/lib/python3.7/site-packages/async_service/asyncio.py", line 246, in run
for _, exc_value, exc_tb in self._errors
trio.MultiError: QueueFull("TaskQueue unsuccessful in adding task HexBytes('0xc20b1fd0d561fd36fc89a13e3c43454c7056ac697ffacd1443a0b2cec5a1a365') ", 'because qsize=6144, num_tasks=6114, maxsize=6144, open_slots=30, num queueing=1, len(_tasks)=6114, task_idx=0, queuing=(<trinity._utils.datastructures.PredefinedSortableTask object at 0x7fa4281be9d0>,), original msg: '), QueueFull("TaskQueue unsuccessful in adding task HexBytes('0xc20b1fd0d561fd36fc89a13e3c43454c7056ac697ffacd1443a0b2cec5a1a365') ", 'because qsize=6144, num_tasks=6114, maxsize=6144, open_slots=30, num queueing=1, len(_tasks)=6114, task_idx=0, queuing=(<trinity._utils.datastructures.PredefinedSortableTask object at 0x7fa37b28cd10>,), original msg: ')

Details of embedded exception 1:

Traceback (most recent call last):
File "/usr/src/app/trinity/trinity/_utils/datastructures.py", line 181, in add
self._open_queue.put_nowait(task)
File "/usr/local/lib/python3.7/asyncio/queues.py", line 144, in put_nowait
raise QueueFull
asyncio.queues.QueueFull

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/local/lib/python3.7/site-packages/async_service/base.py", line 300, in _run_and_manage_task
await task.run()
File "/usr/local/lib/python3.7/site-packages/async_service/asyncio.py", line 35, in run
await self._async_fn(*self._async_fn_args)
File "/usr/src/app/trinity/trinity/sync/beam/chain.py", line 1002, in _hang_until_storage_served
await self._serve_storage(event)
File "/usr/src/app/trinity/trinity/sync/beam/chain.py", line 1034, in _serve_storage
event.urgent,
File "/usr/src/app/trinity/trinity/sync/beam/state.py", line 368, in download_storage
await self.ensure_nodes_present(need_nodes, urgent)
File "/usr/src/app/trinity/trinity/sync/beam/state.py", line 179, in ensure_nodes_present
BLOCK_IMPORT_MISSING_STATE_TIMEOUT,
File "/usr/src/app/trinity/trinity/sync/beam/state.py", line 221, in _wait_for_nodes
await queue.add(unrequested_nodes)
File "/usr/src/app/trinity/trinity/_utils/datastructures.py", line 187, in add
f'because qsize={qsize}, '
asyncio.queues.QueueFull: ("TaskQueue unsuccessful in adding task HexBytes('0xc20b1fd0d561fd36fc89a13e3c43454c7056ac697ffacd1443a0b2cec5a1a365') ", 'because qsize=6144, num_tasks=6114, maxsize=6144, open_slots=30, num queueing=1, len(_tasks)=6114, task_idx=0, queuing=(<trinity._utils.datastructures.PredefinedSortableTask object at 0x7fa4281be9d0>,), original msg: ')

Details of embedded exception 2:

Traceback (most recent call last):
File "/usr/src/app/trinity/trinity/_utils/datastructures.py", line 181, in add
self._open_queue.put_nowait(task)
File "/usr/local/lib/python3.7/asyncio/queues.py", line 144, in put_nowait
raise QueueFull
asyncio.queues.QueueFull

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/local/lib/python3.7/site-packages/async_service/base.py", line 300, in _run_and_manage_task
await task.run()
File "/usr/local/lib/python3.7/site-packages/async_service/asyncio.py", line 35, in run
await self._async_fn(*self._async_fn_args)
File "/usr/src/app/trinity/trinity/sync/beam/chain.py", line 1002, in _hang_until_storage_served
await self._serve_storage(event)
File "/usr/src/app/trinity/trinity/sync/beam/chain.py", line 1034, in _serve_storage
event.urgent,
File "/usr/src/app/trinity/trinity/sync/beam/state.py", line 368, in download_storage
await self.ensure_nodes_present(need_nodes, urgent)
File "/usr/src/app/trinity/trinity/sync/beam/state.py", line 179, in ensure_nodes_present
BLOCK_IMPORT_MISSING_STATE_TIMEOUT,
File "/usr/src/app/trinity/trinity/sync/beam/state.py", line 221, in _wait_for_nodes
await queue.add(unrequested_nodes)
File "/usr/src/app/trinity/trinity/_utils/datastructures.py", line 187, in add
f'because qsize={qsize}, '
asyncio.queues.QueueFull: ("TaskQueue unsuccessful in adding task HexBytes('0xc20b1fd0d561fd36fc89a13e3c43454c7056ac697ffacd1443a0b2cec5a1a365') ", 'because qsize=6144, num_tasks=6114, maxsize=6144, open_slots=30, num queueing=1, len(_tasks)=6114, task_idx=0, queuing=(<trinity._utils.datastructures.PredefinedSortableTask object at 0x7fa37b28cd10>,), original msg: ')�[0m
�[1m�[31m   ERROR  2020-09-03 21:51:33,798               asyncio  Exception in callback <TaskWakeupMethWrapper object at 0x7fa35dc06d50>
handle: <Handle <TaskWakeupMethWrapper object at 0x7fa35dc06d50>>
Traceback (most recent call last):
File "uvloop/cbhandles.pyx", line 70, in uvloop.loop.Handle._run
File "/usr/local/lib/python3.7/site-packages/async_service/base.py", line 300, in _run_and_manage_task
await task.run()
File "/usr/local/lib/python3.7/site-packages/async_service/base.py", line 165, in run
await self.child_manager.run()
File "/usr/local/lib/python3.7/site-packages/async_service/asyncio.py", line 234, in run
await self._wait_all_tasks_done()
File "/usr/local/lib/python3.7/site-packages/async_service/asyncio.py", line 195, in _wait_all_tasks_done
await task
File "uvloop/cbhandles.pyx", line 70, in uvloop.loop.Handle._run
File "/usr/local/lib/python3.7/site-packages/async_service/asyncio.py", line 122, in _handle_cancelled
await self._real_handle_cancelled()
File "/usr/local/lib/python3.7/site-packages/async_service/asyncio.py", line 173, in _real_handle_cancelled
await asyncio_task
File "uvloop/cbhandles.pyx", line 70, in uvloop.loop.Handle._run
File "/usr/local/lib/python3.7/site-packages/async_service/base.py", line 300, in _run_and_manage_task
await task.run()
File "/usr/local/lib/python3.7/site-packages/async_service/base.py", line 165, in run
await self.child_manager.run()
File "/usr/local/lib/python3.7/site-packages/async_service/asyncio.py", line 246, in run
for _, exc_value, exc_tb in self._errors
trio.MultiError: QueueFull("TaskQueue unsuccessful in adding task HexBytes('0xc20b1fd0d561fd36fc89a13e3c43454c7056ac697ffacd1443a0b2cec5a1a365') ", 'because qsize=6144, num_tasks=6114, maxsize=6144, open_slots=30, num queueing=1, len(_tasks)=6114, task_idx=0, queuing=(<trinity._utils.datastructures.PredefinedSortableTask object at 0x7fa4281be9d0>,), original msg: '), QueueFull("TaskQueue unsuccessful in adding task HexBytes('0xc20b1fd0d561fd36fc89a13e3c43454c7056ac697ffacd1443a0b2cec5a1a365') ", 'because qsize=6144, num_tasks=6114, maxsize=6144, open_slots=30, num queueing=1, len(_tasks)=6114, task_idx=0, queuing=(<trinity._utils.datastructures.PredefinedSortableTask object at 0x7fa37b28cd10>,), original msg: ')

Details of embedded exception 1:

Traceback (most recent call last):
File "/usr/src/app/trinity/trinity/_utils/datastructures.py", line 181, in add
self._open_queue.put_nowait(task)
File "/usr/local/lib/python3.7/asyncio/queues.py", line 144, in put_nowait
raise QueueFull
asyncio.queues.QueueFull

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/local/lib/python3.7/site-packages/async_service/base.py", line 300, in _run_and_manage_task
await task.run()
File "/usr/local/lib/python3.7/site-packages/async_service/asyncio.py", line 35, in run
await self._async_fn(*self._async_fn_args)
File "/usr/src/app/trinity/trinity/sync/beam/chain.py", line 1002, in _hang_until_storage_served
await self._serve_storage(event)
File "/usr/src/app/trinity/trinity/sync/beam/chain.py", line 1034, in _serve_storage
event.urgent,
File "/usr/src/app/trinity/trinity/sync/beam/state.py", line 368, in download_storage
await self.ensure_nodes_present(need_nodes, urgent)
File "/usr/src/app/trinity/trinity/sync/beam/state.py", line 179, in ensure_nodes_present
BLOCK_IMPORT_MISSING_STATE_TIMEOUT,
File "/usr/src/app/trinity/trinity/sync/beam/state.py", line 221, in _wait_for_nodes
await queue.add(unrequested_nodes)
File "/usr/src/app/trinity/trinity/_utils/datastructures.py", line 187, in add
f'because qsize={qsize}, '
asyncio.queues.QueueFull: ("TaskQueue unsuccessful in adding task HexBytes('0xc20b1fd0d561fd36fc89a13e3c43454c7056ac697ffacd1443a0b2cec5a1a365') ", 'because qsize=6144, num_tasks=6114, maxsize=6144, open_slots=30, num queueing=1, len(_tasks)=6114, task_idx=0, queuing=(<trinity._utils.datastructures.PredefinedSortableTask object at 0x7fa4281be9d0>,), original msg: ')

Details of embedded exception 2:

Traceback (most recent call last):
File "/usr/src/app/trinity/trinity/_utils/datastructures.py", line 181, in add
self._open_queue.put_nowait(task)
File "/usr/local/lib/python3.7/asyncio/queues.py", line 144, in put_nowait
raise QueueFull
asyncio.queues.QueueFull

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/local/lib/python3.7/site-packages/async_service/base.py", line 300, in _run_and_manage_task
await task.run()
File "/usr/local/lib/python3.7/site-packages/async_service/asyncio.py", line 35, in run
await self._async_fn(*self._async_fn_args)
File "/usr/src/app/trinity/trinity/sync/beam/chain.py", line 1002, in _hang_until_storage_served
await self._serve_storage(event)
File "/usr/src/app/trinity/trinity/sync/beam/chain.py", line 1034, in _serve_storage
event.urgent,
File "/usr/src/app/trinity/trinity/sync/beam/state.py", line 368, in download_storage
await self.ensure_nodes_present(need_nodes, urgent)
File "/usr/src/app/trinity/trinity/sync/beam/state.py", line 179, in ensure_nodes_present
BLOCK_IMPORT_MISSING_STATE_TIMEOUT,
File "/usr/src/app/trinity/trinity/sync/beam/state.py", line 221, in _wait_for_nodes
await queue.add(unrequested_nodes)
File "/usr/src/app/trinity/trinity/_utils/datastructures.py", line 187, in add
f'because qsize={qsize}, '
asyncio.queues.QueueFull: ("TaskQueue unsuccessful in adding task HexBytes('0xc20b1fd0d561fd36fc89a13e3c43454c7056ac697ffacd1443a0b2cec5a1a365') ", 'because qsize=6144, num_tasks=6114, maxsize=6144, open_slots=30, num queueing=1, len(_tasks)=6114, task_idx=0, queuing=(<trinity._utils.datastructures.PredefinedSortableTask object at 0x7fa37b28cd10>,), original msg: ')�[0m
DEBUG  2020-09-03 21:51:33,798         MuirGlacierVM  Beam pivot over 1 txn preview for <BlockHeader #10790870 55f0d7cd> b/c StateUnretrievable('No servers for CollectMissingAccount') after 0.0s, %exec 100, stats: BeamStat: accts=0, a_nodes=0, codes=0, strg=0, s_nodes=0, nodes=0, rtt=0.000s, wait=0s

# Fixed code
async def add(self, tasks: Tuple[TTask, ...]) -> None:
    """
    add() will insert as many tasks as can be inserted until the queue fills up.
    Then it will pause until the queue is no longer full, and continue adding tasks.
    It will finally return when all tasks have been inserted.
    """
    if not isinstance(tasks, tuple):
        raise ValidationError(f"must pass a tuple of tasks to add(), but got {tasks!r}")

    already_pending = self._tasks.intersection(tasks)
    if already_pending:
        raise ValidationError(
            f"Duplicate tasks detected: {already_pending!r} are already present in the queue"
        )

    # make sure to insert the highest-priority items first, in case queue fills up
    remaining = tuple(sorted(map(self._task_wrapper, tasks)))

    while remaining:
        num_tasks = len(self._tasks)

        if self._maxsize <= 0:
            # no cap at all, immediately insert all tasks
            open_slots = len(remaining)
        elif num_tasks < self._maxsize:
            # there is room to add at least one more task
            open_slots = self._maxsize - num_tasks
        else:
            # wait until there is room in the queue
            await self._full_lock.acquire()

            # the current number of tasks has changed, restart attempt
            continue

        queueing, remaining = remaining[:open_slots], remaining[open_slots:]

        for task in queueing:
            await self._open_queue.put(task)

        original_queued = tuple(task.original for task in queueing)
        self._tasks.update(original_queued)

        if self._full_lock.locked() and len(self._tasks) < self._maxsize:
            self._full_lock.release()