Repository: edgedb/edgedb

Issue:
expose Prometheus `/metrics` endpoint when running in remote-compiler mode
current behavior:

when running the server in remote compiler pool mode for the cloud, it does not accept any HTTP connections, either with or without SSL:

```
admin@nomad-free-tier-2-12-node0:~$ ps aux | grep compiler-pool-mode
nobody      5964  0.3  2.8 346408 235236 ?       Ssl  18:54   0:06 /local/edgedb-server/bin/python3 -I /local/edgedb-server/bin/edgedb-server.py --backend-dsn=stolon+consul+http://127.0.0.1:8500/nebula-e304100ed31911ed8d1bd78a7a5888c6 --port=24409 --instance-name=test202304041852 --compiler-pool-mode=remote --compiler-pool-addr=0.0.0.0:5660 --compiler-pool-size=2 --tls-cert-mode=require_file --tls-key-file=/secrets/tls-private-key.pem --tls-cert-file=/secrets/tls-certificate.pem --admin-ui=enabled --readiness-state-file=/local/readiness_state --jws-key-file=/secrets/edgedb-jws.pubkey.pem --default-auth-method=JWT --bind-address=0.0.0.0
admin       7131  0.0  0.0   7008  2124 pts/0    S+   19:29   0:00 grep --color=auto compiler-pool-mode
admin@nomad-free-tier-2-12-node0:~$ curl -v http://localhost:5660
*   Trying 127.0.0.1:5660...
* Connected to localhost (127.0.0.1) port 5660 (#0)
> GET / HTTP/1.1
> Host: localhost:5660
> User-Agent: curl/7.81.0
> Accept: */*
>
* Received HTTP/0.9 when not allowed
* Closing connection 0
curl: (1) Received HTTP/0.9 when not allowed
admin@nomad-free-tier-2-12-node0:~$ curl -v -k https://localhost:5660
*   Trying 127.0.0.1:5660...
* Connected to localhost (127.0.0.1) port 5660 (#0)
* ALPN, offering h2
* ALPN, offering http/1.1
* TLSv1.0 (OUT), TLS header, Certificate Status (22):
* TLSv1.3 (OUT), TLS handshake, Client hello (1):
* error:0A00010B:SSL routines::wrong version number
* Closing connection 0
curl: (35) error:0A00010B:SSL routines::wrong version number
```

desired behavior:

it should listen on HTTP and respond to the `/metrics` endpoint for monitoring purposes. 


Discussion:

Resolving patch:
diff --git a/edb/server/args.py b/edb/server/args.py
--- a/edb/server/args.py
+++ b/edb/server/args.py
@@ -928,6 +928,10 @@ def server_options(func):
         help="Directory to store UNIX domain socket file for IPC, a temporary "
              "directory will be used if not specified.",
     ),
+    click.option(
+        '--metrics-port', type=PortType(),
+        help=f'Port to listen on for metrics HTTP API.',
+    ),
 ]
 
 
diff --git a/edb/server/compiler_pool/server.py b/edb/server/compiler_pool/server.py
--- a/edb/server/compiler_pool/server.py
+++ b/edb/server/compiler_pool/server.py
@@ -32,11 +32,14 @@
 import typing
 
 import click
+import httptools
 import immutables
 
 from edb.common import debug
 from edb.common import markup
+from edb.common import taskgroup
 
+from .. import metrics
 from .. import args as srvargs
 from .. import defines
 from . import amsg
@@ -523,12 +526,62 @@ def reply(self, req_id, resp):
         )
 
 
+class MetricsProtocol(asyncio.Protocol):
+    def __init__(self):
+        self.transport = None
+        self.parser = httptools.HttpRequestParser(self)
+        self.url = None
+
+    def connection_made(self, transport):
+        self.transport = transport
+
+    def data_received(self, data):
+        try:
+            self.parser.feed_data(data)
+        except Exception as ex:
+            logger.exception(ex)
+
+    def on_url(self, url):
+        self.url = url
+
+    def on_message_complete(self):
+        match self.parser.get_method().upper(), self.url:
+            case b"GET", b"/ready":
+                self.respond("200 OK", "OK")
+
+            case b"GET", b"/metrics":
+                self.respond(
+                    "200 OK",
+                    metrics.registry.generate(),
+                    "Content-Type: text/plain; version=0.0.4; charset=utf-8",
+                )
+
+            case _:
+                self.respond("404 Not Found", "Not Found")
+
+    def respond(self, status, content, *extra_headers, encoding="utf-8"):
+        content = content.encode(encoding)
+        response = [
+            f"HTTP/{self.parser.get_http_version()} {status}",
+            f"Content-Length: {len(content)}",
+            *extra_headers,
+            "",
+            "",
+        ]
+
+        self.transport.write("\r\n".join(response).encode("ascii"))
+        self.transport.write(content)
+        if not self.parser.should_keep_alive():
+            self.transport.close()
+
+
 async def server_main(
     listen_addresses,
     listen_port,
     pool_size,
     client_schema_cache_size,
     runstate_dir,
+    metrics_port,
 ):
     if listen_port is None:
         listen_port = defines.EDGEDB_REMOTE_COMPILER_PORT
@@ -558,27 +611,26 @@ async def server_main(
         )
         await pool.start()
         try:
-            server = await loop.create_server(
-                lambda: CompilerServerProtocol(pool, loop),
-                listen_addresses,
-                listen_port,
-                start_serving=False,
-            )
-            if len(listen_addresses) == 1:
-                logger.info(
-                    "Listening on %s:%s", listen_addresses[0], listen_port
-                )
-            else:
-                logger.info(
-                    "Listening on [%s]:%s",
-                    ",".join(listen_addresses),
-                    listen_port,
+            async with taskgroup.TaskGroup() as tg:
+                tg.create_task(
+                    _run_server(
+                        loop,
+                        listen_addresses,
+                        listen_port,
+                        lambda: CompilerServerProtocol(pool, loop),
+                        "compile",
+                    )
                 )
-            try:
-                await server.serve_forever()
-            finally:
-                server.close()
-                await server.wait_closed()
+                if metrics_port:
+                    tg.create_task(
+                        _run_server(
+                            loop,
+                            listen_addresses,
+                            metrics_port,
+                            MetricsProtocol,
+                            "metrics",
+                        )
+                    )
         finally:
             await pool.stop()
     finally:
@@ -586,6 +638,36 @@ async def server_main(
             temp_runstate_dir.cleanup()
 
 
+async def _run_server(
+    loop, listen_addresses, listen_port, protocol, purpose
+):
+    server = await loop.create_server(
+        protocol,
+        listen_addresses,
+        listen_port,
+        start_serving=False,
+    )
+    if len(listen_addresses) == 1:
+        logger.info(
+            "Listening for %s on %s:%s",
+            purpose,
+            listen_addresses[0],
+            listen_port,
+        )
+    else:
+        logger.info(
+            "Listening for %s on [%s]:%s",
+            purpose,
+            ",".join(listen_addresses),
+            listen_port,
+        )
+    try:
+        await server.serve_forever()
+    finally:
+        server.close()
+        await server.wait_closed()
+
+
 @click.command()
 @srvargs.compiler_options
 def main(**kwargs):


Test patch: