Repository: edgedb/edgedb

Issue:
Mysterious test hangs in `start_edgedb_server`
Fantix and I have both been able to replicate this, at least sometimes.
`edb test -j32 -k test_server_adjacent -k test_server_proto  -v` fairly consistently has a hang on @fantix's machine.

I've managed to reproduce it with `taskset -c 0-15 edb test -j32 -k test_server_adjacent -k test_server_proto  -v` on my test machine, but not consistently.

I discovered this while testing https://github.com/edgedb/edgedb/pull/6782, and with https://github.com/edgedb/edgedb/pull/6782 there seemed to also be knock-on effects where some other DROP DATABASE commands hung, which needs to be tracked down.


Discussion:
Looks like the hanging happens during server start, when the pool failed to complete a connection acquisition when it's exhausted by concurrent database (early) introspection due to test database clones, like shown in my debug:

```
413955 EARLY INTROSPECT __edgedbsys__
413955 EARLY INTROSPECT edgedb
413955 EARLY INTROSPECT serverprotoconcurrentddl
413955 EARLY INTROSPECT serverconfig
413955 EARLY INTROSPECT serverproto
413955 EARLY INTROSPECT serverprotoddlpropagation
413955 EARLY INTROSPECT serverprotoconcurrentglobalddl
413955 EARLY INTROSPECT serverprotoddl
413955 EARLY INTROSPECT serverprotomigration
413955 EARLY INTROSPECT serverproto_413755
413955 EARLY INTROSPECT serverproto_413753
413955 EARLY INTROSPECT serverprotoddl_413751
413955 EARLY INTROSPECT serverprotoddl_413752
413955 EARLY INTROSPECT serverprotoddl_413745
413955 EARLY INTROSPECT serverprotoddlpropagation_413749
413955 EARLY INTROSPECT serverprotoddlpropagation_413748
413955 EARLY INTROSPECT serverprotoddlpropagation_413747
413955 EARLY INTROSPECT serverprotomigration_413750
```

(18 concurrent requests on the pool with a size of 10)

In detail, the pool had to steal a connection from other blocks, but the async task was never resumed for some reason. I'm still looking at this.

@msullivan @1st1 With databases transiting to branches, is it still necessary to maintain a connection pool that mostly provides fairness and service quality in the case when many databases/branches are competing for a limited number of connections?


Resolving patch:
diff --git a/edb/server/connpool/pool.py b/edb/server/connpool/pool.py
--- a/edb/server/connpool/pool.py
+++ b/edb/server/connpool/pool.py
@@ -778,6 +778,7 @@ def _tick(self) -> None:
             if min_demand > demand:
                 min_demand = demand
 
+        was_starving = self._is_starving
         self._is_starving = need_conns_at_least >= self._max_capacity
         if self._to_drop:
             for block in self._to_drop:
@@ -841,6 +842,33 @@ def _tick(self) -> None:
                     self._log_to_snapshot(
                         dbname=block.dbname, event='reset-quota')
 
+            if not was_starving and self._new_blocks_waitlist:
+                # Mode D assumes all connections are already in use or to be
+                # used, depending on their `release()` to schedule transfers.
+                # When just entering Mode D, there can be a special case when
+                # no further `release()` will be called because all acquired
+                # connections were returned to the pool before `_tick()` got a
+                # chance to set `self._is_starving`, while some other blocks
+                # are literally starving to death (blocked forever).
+                #
+                # This branch handles this particular case, by stealing
+                # connections from the idle blocks and try to free them into
+                # the starving blocks.
+
+                for block in list(self._blocks.values()):
+                    while self._should_free_conn(block):
+                        if (conn := block.try_steal()) is None:
+                            # no more from this block
+                            break
+
+                        elif not self._maybe_free_into_starving_blocks(
+                            block, conn
+                        ):
+                            # put back the last stolen connection if we
+                            # don't need to steal anymore
+                            self._release_unused(block, conn)
+                            return
+
         else:
             # Mode C: distribute the total connections by calibrated demand
             # setting the per-block quota, then trigger rebalance.
@@ -957,14 +985,11 @@ def _should_free_conn(self, from_block: Block[C]) -> bool:
 
         return True
 
-    def _maybe_free_conn(
+    def _maybe_free_into_starving_blocks(
         self,
         from_block: Block[C],
         conn: C,
     ) -> bool:
-        if not self._should_free_conn(from_block):
-            return False
-
         label, to_block = self._find_most_starving_block()
         if to_block is None or to_block is from_block:
             return False
@@ -1178,23 +1203,28 @@ def release(self, dbname: str, conn: C, *, discard: bool=False) -> None:
 
         self._maybe_schedule_tick()
 
-        if not self._maybe_free_conn(block, conn):
+        if not (
+            self._should_free_conn(block)
+            and self._maybe_free_into_starving_blocks(block, conn)
+        ):
             if discard:
                 # Concurrent `acquire()` may be waiting to reuse the released
                 # connection here - as we should discard this one, let's just
                 # schedule a new one in the same block.
                 self._schedule_discard(block, conn)
                 self._schedule_new_conn(block)
-                return
+            else:
+                self._release_unused(block, conn)
 
-            block.release(conn)
+    def _release_unused(self, block: Block[C], conn: C) -> None:
+        block.release(conn)
 
-            # Only request for GC if the connection is released unused
-            self._gc_requests += 1
-            if self._gc_requests == 1:
-                # Only schedule GC for the very first request - following
-                # requests will be grouped into the next GC
-                self._get_loop().call_later(self._gc_interval, self._run_gc)
+        # Only request for GC if the connection is released unused
+        self._gc_requests += 1
+        if self._gc_requests == 1:
+            # Only schedule GC for the very first request - following
+            # requests will be grouped into the next GC
+            self._get_loop().call_later(self._gc_interval, self._run_gc)
 
     async def prune_inactive_connections(self, dbname: str) -> None:
         try:


Test patch:
diff --git a/tests/test_server_pool.py b/tests/test_server_pool.py
--- a/tests/test_server_pool.py
+++ b/tests/test_server_pool.py
@@ -1570,6 +1570,39 @@ async def main():
 
         asyncio.run(main())
 
+    def test_connpool_eternal_starvation(self):
+        async def fake_connect(dbname):
+            # very fast connect
+            return FakeConnection(dbname)
+
+        async def test():
+            pool = connpool.Pool(
+                connect=fake_connect,
+                disconnect=self.make_fake_disconnect(),
+                max_capacity=5,
+            )
+            counter = 0
+            event = asyncio.Event()
+
+            async def job(dbname):
+                nonlocal counter
+
+                conn = await pool.acquire(dbname)
+                counter += 1
+                if counter >= 5:
+                    event.set()
+                await event.wait()
+                pool.release(dbname, conn)
+
+            async with asyncio.TaskGroup() as g:
+                for n in range(10):
+                    g.create_task(job(f"block_{n}"))
+
+        async def main():
+            await asyncio.wait_for(test(), timeout=3)
+
+        asyncio.run(main())
+
 
 HTML_TPL = R'''<!DOCTYPE html>
 <html>