Repository: edgedb/edgedb

Issue:
InternalServerError changing listen_addresses on bare metal install
I followed the instructions in the [bare metal deployment guide](https://www.edgedb.com/docs/guides/deployment/bare_metal) and discovered that I could not change the `listen_addresses` without restarting the server.

<!-- Please search existing issues to avoid creating duplicates. -->
- EdgeDB Version: EdgeDB 1.4+9d299d7 (repl 1.1.2+f832277)
- OS Version: ubuntu 22.04

Steps to Reproduce:
```bash
$ sudo edgedb --tls-security=insecure --admin -P 5656
EdgeDB 1.4+9d299d7 (repl 1.1.2+f832277)
Type \help for help, \quit to quit.
edgedb> alter role edgedb set password := 'super1secret';
OK: ALTER ROLE

edgedb> configure instance set listen_addresses := {'0.0.0.0'};
Error: InternalServerError: could not create any listen sockets
  Hint: This is most likely a bug in EdgeDB. Please consider opening an issue ticket at https://github.com/edgedb/edgedb/issues/new?template=bug_report.md

edgedb> \q
$ sudo systemctl restart edgedb-server-1.service
$ sudo edgedb --tls-security=insecure -P 5656 --password
Password for 'edgedb':
EdgeDB 1.4+9d299d7 (repl 1.1.2+f832277)
Type \help for help, \quit to quit.
edgedb> configure instance set listen_addresses := {'0.0.0.0'};
OK: CONFIGURE INSTANCE
edgedb> \q
```


Discussion:
This isn't specific to "bare metal", easily reproducible on a dev build too.
Still here in `EdgeDB 1.4+0a5653c (repl 1.2.2+e81d259)`

```
root@edgedb-test-1:~# sudo edgedb --port 5656 --tls-security insecure --password query  "CONFIGURE INSTANCE SET listen_addresses := {'0.0.0.0'};"
Password for 'edgedb':
edgedb error: InternalServerError: could not create any listen sockets
  Hint: This is most likely a bug in EdgeDB. Please consider opening an issue ticket at https://github.com/edgedb/edgedb/issues/new?template=bug_report.md
  Server traceback:
      Traceback (most recent call last):
        File "edb/server/protocol/binary.pyx", line 1845, in edb.server.protocol.binary.EdgeConnection.main
        File "edb/server/protocol/binary.pyx", line 1712, in execute
        File "edb/server/protocol/binary.pyx", line 1662, in _execute
        File "edb/server/protocol/binary.pyx", line 1619, in edb.server.protocol.binary.EdgeConnection._execute
        File "edb/server/protocol/binary.pyx", line 1560, in _execute_system_config
        File "edb/server/dbview/dbview.pyx", line 550, in apply_config_ops
        File "edb/server/dbview/dbview.pyx", line 693, in apply_system_config_op
        File "/usr/lib/x86_64-linux-gnu/edgedb-server-1/lib/python3.10/site-packages/edb/server/server.py", line 978, in _on_system_config_set
          await self._restart_servers_new_addr(value, self._listen_port)
        File "/usr/lib/x86_64-linux-gnu/edgedb-server-1/lib/python3.10/site-packages/edb/server/server.py", line 884, in _restart_servers_new_addr
          new_servers, *_ = await self._start_servers(
        File "/usr/lib/x86_64-linux-gnu/edgedb-server-1/lib/python3.10/site-packages/edb/server/server.py", line 1412, in _start_servers
          raise StartupError("could not create any listen sockets")
      edb.server.server.StartupError: could not create any listen sockets
```

Resolving patch:
diff --git a/edb/server/server.py b/edb/server/server.py
--- a/edb/server/server.py
+++ b/edb/server/server.py
@@ -896,8 +896,11 @@ def get_roles(self):
     async def _restart_servers_new_addr(self, nethosts, netport):
         if not netport:
             raise RuntimeError('cannot restart without network port specified')
-        nethosts = await _resolve_interfaces(nethosts)
+        nethosts, has_ipv4_wc, has_ipv6_wc = await _resolve_interfaces(
+            nethosts
+        )
         servers_to_stop = []
+        servers_to_stop_early = []
         servers = {}
         if self._listen_port == netport:
             hosts_to_start = [
@@ -906,7 +909,25 @@ async def _restart_servers_new_addr(self, nethosts, netport):
             for host, srv in self._servers.items():
                 if host == ADMIN_PLACEHOLDER or host in nethosts:
                     servers[host] = srv
+                elif host in ['::', '0.0.0.0']:
+                    servers_to_stop_early.append(srv)
                 else:
+                    if has_ipv4_wc:
+                        try:
+                            ipaddress.IPv4Address(host)
+                        except ValueError:
+                            pass
+                        else:
+                            servers_to_stop_early.append(srv)
+                            continue
+                    if has_ipv6_wc:
+                        try:
+                            ipaddress.IPv6Address(host)
+                        except ValueError:
+                            pass
+                        else:
+                            servers_to_stop_early.append(srv)
+                            continue
                     servers_to_stop.append(srv)
             admin = False
         else:
@@ -914,17 +935,29 @@ async def _restart_servers_new_addr(self, nethosts, netport):
             servers_to_stop = self._servers.values()
             admin = True
 
+        if servers_to_stop_early:
+            await self._stop_servers_with_logging(servers_to_stop_early)
+
         if hosts_to_start:
-            new_servers, *_ = await self._start_servers(
-                hosts_to_start,
-                netport,
-                admin=admin,
-            )
-            servers.update(new_servers)
+            try:
+                new_servers, *_ = await self._start_servers(
+                    hosts_to_start,
+                    netport,
+                    admin=admin,
+                )
+                servers.update(new_servers)
+            except StartupError:
+                raise errors.ConfigurationError(
+                    'Server updated its config but cannot serve on requested '
+                    'address/port, please see server log for more information.'
+                )
         self._servers = servers
         self._listen_hosts = nethosts
         self._listen_port = netport
 
+        await self._stop_servers_with_logging(servers_to_stop)
+
+    async def _stop_servers_with_logging(self, servers_to_stop):
         addrs = []
         unix_addr = None
         port = None
@@ -932,7 +965,7 @@ async def _restart_servers_new_addr(self, nethosts, netport):
             for s in srv.sockets:
                 addr = s.getsockname()
                 if isinstance(addr, tuple):
-                    addrs.append(addr)
+                    addrs.append(addr[:2])
                     if port is None:
                         port = addr[1]
                     elif port != addr[1]:
@@ -1634,7 +1667,7 @@ async def start(self):
             )
 
         self._servers, actual_port, listen_addrs = await self._start_servers(
-            await _resolve_interfaces(self._listen_hosts),
+            (await _resolve_interfaces(self._listen_hosts))[0],
             self._listen_port,
             sockets=self._listen_sockets,
         )
@@ -1851,7 +1884,7 @@ def serialize_config(cfg):
 
 def _cleanup_wildcard_addrs(
     hosts: Sequence[str]
-) -> tuple[list[str], list[str]]:
+) -> tuple[list[str], list[str], bool, bool]:
     """Filter out conflicting addresses in presence of INADDR_ANY wildcards.
 
     Attempting to bind to 0.0.0.0 (or ::) _and_ a non-wildcard address will
@@ -1894,28 +1927,36 @@ def _cleanup_wildcard_addrs(
         named_hosts.add(host)
 
     if not ipv4_hosts and not ipv6_hosts:
-        return (list(hosts), [])
+        return (list(hosts), [], False, False)
 
     if ipv4_wc not in ipv4_hosts and ipv6_wc not in ipv6_hosts:
-        return (list(hosts), [])
+        return (list(hosts), [], False, False)
 
     if ipv4_wc in ipv4_hosts and ipv6_wc in ipv6_hosts:
         return (
             ['0.0.0.0', '::'],
-            [str(a) for a in
-                ((named_hosts | ipv4_hosts | ipv6_hosts) - {ipv4_wc, ipv6_wc})]
+            [
+                str(a) for a in
+                ((named_hosts | ipv4_hosts | ipv6_hosts) - {ipv4_wc, ipv6_wc})
+            ],
+            True,
+            True,
         )
 
     if ipv4_wc in ipv4_hosts:
         return (
             [str(a) for a in ({ipv4_wc} | ipv6_hosts)],
-            [str(a) for a in ((named_hosts | ipv4_hosts) - {ipv4_wc})]
+            [str(a) for a in ((named_hosts | ipv4_hosts) - {ipv4_wc})],
+            True,
+            False,
         )
 
     if ipv6_wc in ipv6_hosts:
         return (
             [str(a) for a in ({ipv6_wc} | ipv4_hosts)],
-            [str(a) for a in ((named_hosts | ipv6_hosts) - {ipv6_wc})]
+            [str(a) for a in ((named_hosts | ipv6_hosts) - {ipv6_wc})],
+            False,
+            True,
         )
 
     raise AssertionError('unreachable')
@@ -1937,7 +1978,9 @@ async def _resolve_host(host: str) -> list[str] | Exception:
         return [addr[4][0] for addr in addrinfo]
 
 
-async def _resolve_interfaces(hosts: Sequence[str]) -> Sequence[str]:
+async def _resolve_interfaces(
+    hosts: Sequence[str]
+) -> Tuple[Sequence[str], bool, bool]:
 
     async with taskgroup.TaskGroup() as g:
         resolve_tasks = {
@@ -1954,7 +1997,9 @@ async def _resolve_interfaces(hosts: Sequence[str]) -> Sequence[str]:
         else:
             addrs.extend(result)
 
-    clean_addrs, rejected_addrs = _cleanup_wildcard_addrs(addrs)
+    (
+        clean_addrs, rejected_addrs, has_ipv4_wc, has_ipv6_wc
+    ) = _cleanup_wildcard_addrs(addrs)
 
     if rejected_addrs:
         logger.warning(
@@ -1963,4 +2008,4 @@ async def _resolve_interfaces(hosts: Sequence[str]) -> Sequence[str]:
             ", ".join(repr(h) for h in rejected_addrs)
         )
 
-    return clean_addrs
+    return clean_addrs, has_ipv4_wc, has_ipv6_wc


Test patch:
diff --git a/tests/test_server_config.py b/tests/test_server_config.py
--- a/tests/test_server_config.py
+++ b/tests/test_server_config.py
@@ -1184,9 +1184,6 @@ async def test_server_proto_configure_listen_addresses(self):
                 for i, con in enumerate((con1, con2, con3)):
                     self.assertEqual(await con.query_single(f"SELECT {i}"), i)
 
-                await con1.execute("""
-                    CONFIGURE INSTANCE SET listen_addresses := <str>{};
-                """)
                 await con1.execute("""
                     CONFIGURE INSTANCE SET listen_addresses := {
                         '0.0.0.0',
diff --git a/tests/test_server_unit.py b/tests/test_server_unit.py
--- a/tests/test_server_unit.py
+++ b/tests/test_server_unit.py
@@ -28,64 +28,83 @@ def test_server_unittest_fix_wildcard_addrs(self):
         CASES = [
             (
                 ['*'],
-                (['0.0.0.0', '::'], [])
+                (['0.0.0.0', '::'], []),
+                (True, True),
             ),
             (
                 ['0.0.0.0', '::0'],
-                (['0.0.0.0', '::'], [])
+                (['0.0.0.0', '::'], []),
+                (True, True),
             ),
             (
                 ['0.0.0.0', '127.0.0.1', '2001:db8::8a2e:370:7334', '::0'],
-                (['0.0.0.0', '::'], ['127.0.0.1', '2001:db8::8a2e:370:7334'])
+                (['0.0.0.0', '::'], ['127.0.0.1', '2001:db8::8a2e:370:7334']),
+                (True, True),
             ),
             (
                 ['0.0.0.0', 'example.com', '2001:db8::8a2e:370:7334', '::0'],
-                (['0.0.0.0', '::'], ['example.com', '2001:db8::8a2e:370:7334'])
+                (
+                    ['0.0.0.0', '::'],
+                    ['example.com', '2001:db8::8a2e:370:7334'],
+                ),
+                (True, True),
             ),
             (
                 ['127.0.0.1', 'example.com', '2001:db8::8a2e:370:7334', '::0'],
                 (
                     ['127.0.0.1', '::'],
                     ['example.com', '2001:db8::8a2e:370:7334']
-                )
+                ),
+                (False, True),
             ),
             (
                 ['example.com', '2001:db8::8a2e:370:7334', '::0'],
-                (['::'], ['example.com', '2001:db8::8a2e:370:7334'])
+                (['::'], ['example.com', '2001:db8::8a2e:370:7334']),
+                (False, True),
             ),
             (
                 ['example.com', 'sub.example.com'],
-                (['example.com', 'sub.example.com'], [])
+                (['example.com', 'sub.example.com'], []),
+                (False, False),
             ),
             (
                 ['example.com', '127.0.0.1'],
-                (['example.com', '127.0.0.1'], [])
+                (['example.com', '127.0.0.1'], []),
+                (False, False),
             ),
             (
                 ['example.com', '::1'],
-                (['::1', 'example.com'], [])
+                (['::1', 'example.com'], []),
+                (False, False),
             ),
             (
                 ['example.com', '::'],
-                (['::'], ['example.com'])
+                (['::'], ['example.com']),
+                (False, True),
             ),
             (
                 ['example.com', '::1', '127.0.0.1'],
-                (['example.com', '::1', '127.0.0.1'], [])
+                (['example.com', '::1', '127.0.0.1'], []),
+                (False, False),
             ),
             (
                 ['0.0.0.0', '2001:db8::8a2e:370:7334'],
-                (['0.0.0.0', '2001:db8::8a2e:370:7334'], [])
+                (['0.0.0.0', '2001:db8::8a2e:370:7334'], []),
+                (True, False),
             ),
             (
                 ['127.0.0.1', '2001:db8::8a2e:370:7334', '::'],
-                (['127.0.0.1', '::'], ['2001:db8::8a2e:370:7334'])
+                (['127.0.0.1', '::'], ['2001:db8::8a2e:370:7334']),
+                (False, True),
             ),
         ]
 
-        for hosts, expected in CASES:
-            new_hosts, rej_hosts = server._cleanup_wildcard_addrs(hosts)
+        for hosts, expected, expected_wildcard in CASES:
+            (
+                new_hosts, rej_hosts, *has_wildcards
+            ) = server._cleanup_wildcard_addrs(hosts)
             self.assertEqual(
                 (set(new_hosts), set(rej_hosts)),
                 (set(expected[0]), set(expected[1]))
             )
+            self.assertEqual(tuple(has_wildcards), expected_wildcard)