Repository: TobikoData/sqlmesh
Pull request #4571 (2025-05-28 18:26:54)

Feat: Clickhouse HTTPS config parameters
This PR adds additional configuration parameters for configuring HTTPS certs and proxies in the Clickhouse connection

Note: we always create our own connection pool manager instead of letting `clickhouse_connect` construct one for us, so we must pass these parameters into both the primary connection and our pool manager instatiations.


Patch:
diff --git a/docs/integrations/engines/clickhouse.md b/docs/integrations/engines/clickhouse.md
index fb48ac86..14e931b0 100644
--- a/docs/integrations/engines/clickhouse.md
+++ b/docs/integrations/engines/clickhouse.md
@@ -421,4 +421,29 @@ If a model has many records in each partition, you may see additional performanc
     Choose a model's time partitioning granularity based on the characteristics of the data it will process, making sure the total number of partitions is 1000 or fewer.
 
 ## Local/Built-in Scheduler
+
 **Engine Adapter Type**: `clickhouse`
+
+| Option                    | Description                                                                                                                                                                                                                                                                     |  Type  | Required |
+| ------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | :----: | :------: |
+| `type`                    | Engine type name - must be `clickhouse`                                                                                                                                                                                                                                         | string |    Y     |
+| `host`                    | ClickHouse server hostname or IP address                                                                                                                                                                                                                                        | string |    Y     |
+| `username`                | ClickHouse user name                                                                                                                                                                                                                                                            | string |    Y     |
+| `password`                | ClickHouse user password                                                                                                                                                                                                                                                        | string |    N     |
+| `port`                    | The ClickHouse HTTP or HTTPS port (Default: `8123`)                                                                                                                                                                                                                             |  int   |    N     |
+| `cluster`                 | ClickHouse cluster name                                                                                                                                                                                                                                                         | string |    N     |
+| `connect_timeout`         | Connection timeout in seconds (Default: `10`)                                                                                                                                                                                                                                   |  int   |    N     |
+| `send_receive_timeout`    | Send/receive timeout in seconds (Default: `300`)                                                                                                                                                                                                                                |  int   |    N     |
+| `query_limit`             | Query result limit (Default: `0` - no limit)                                                                                                                                                                                                                                    |  int   |    N     |
+| `use_compression`         | Whether to use compression (Default: `True`)                                                                                                                                                                                                                                    |  bool  |    N     |
+| `compression_method`      | Compression method to use                                                                                                                                                                                                                                                       | string |    N     |
+| `http_proxy`              | HTTP proxy address (equivalent to setting the HTTP_PROXY environment variable)                                                                                                                                                                                                  | string |    N     |
+| `verify`                  | Verify server TLS/SSL certificate (Default: `True`)                                                                                                                                                                                                                             |  bool  |    N     |
+| `ca_cert`                 | Ignored if verify is `False`. If verify is `True`, the file path to Certificate Authority root to validate ClickHouse server certificate, in .pem format. Not necessary if the ClickHouse server certificate is a globally trusted root as verified by the operating system.    | string |    N     |
+| `client_cert`             | File path to a TLS Client certificate in .pem format (for mutual TLS authentication). The file should contain a full certificate chain, including any intermediate certificates.                                                                                                | string |    N     |
+| `client_cert_key`         | File path to the private key for the Client Certificate. Required if the private key is not included the Client Certificate key file.                                                                                                                                           | string |    N     |
+| `https_proxy`             | HTTPS proxy address (equivalent to setting the HTTPS_PROXY environment variable)                                                                                                                                                                                                | string |    N     |
+| `server_host_name`        | The ClickHouse server hostname as identified by the CN or SNI of its TLS certificate. Set this to avoid SSL errors when connecting through a proxy or tunnel with a different hostname.                                                                                         | string |    N     |
+| `tls_mode`                | Controls advanced TLS behavior. proxy and strict do not invoke ClickHouse mutual TLS connection, but do send client cert and key. mutual assumes ClickHouse mutual TLS auth with a client certificate.                                                                          | string |    N     |
+| `connection_settings`     | Additional [connection settings](https://clickhouse.com/docs/integrations/python#settings-argument)                                                                                                                                                                             |  dict  |    N     |
+| `connection_pool_options` | Additional [options](https://clickhouse.com/docs/integrations/python#customizing-the-http-connection-pool)                                                                                                                                         for the HTTP connection pool |  dict  |    N     |
\ No newline at end of file
diff --git a/sqlmesh/core/config/connection.py b/sqlmesh/core/config/connection.py
index 1ba66a0a..0c7055a5 100644
--- a/sqlmesh/core/config/connection.py
+++ b/sqlmesh/core/config/connection.py
@@ -1737,11 +1737,19 @@ class ClickhouseConnectionConfig(ConnectionConfig):
     cluster: t.Optional[str] = None
     connect_timeout: int = 10
     send_receive_timeout: int = 300
-    verify: bool = True
     query_limit: int = 0
     use_compression: bool = True
     compression_method: t.Optional[str] = None
     connection_settings: t.Optional[t.Dict[str, t.Any]] = None
+    http_proxy: t.Optional[str] = None
+    # HTTPS/TLS settings
+    verify: bool = True
+    ca_cert: t.Optional[str] = None
+    client_cert: t.Optional[str] = None
+    client_cert_key: t.Optional[str] = None
+    https_proxy: t.Optional[str] = None
+    server_host_name: t.Optional[str] = None
+    tls_mode: t.Optional[str] = None
 
     concurrent_tasks: int = 1
     register_comments: bool = True
@@ -1766,8 +1774,15 @@ class ClickhouseConnectionConfig(ConnectionConfig):
             "password",
             "connect_timeout",
             "send_receive_timeout",
-            "verify",
             "query_limit",
+            "http_proxy",
+            "verify",
+            "ca_cert",
+            "client_cert",
+            "client_cert_key",
+            "https_proxy",
+            "server_host_name",
+            "tls_mode",
         }
         return kwargs
 
@@ -1786,7 +1801,18 @@ class ClickhouseConnectionConfig(ConnectionConfig):
             maxsize=self.concurrent_tasks,
             # Block if there are no free connections
             block=True,
+            verify=self.verify,
+            ca_cert=self.ca_cert,
+            client_cert=self.client_cert,
+            client_cert_key=self.client_cert_key,
+            https_proxy=self.https_proxy,
         )
+        # this doesn't happen automatically because we always supply our own pool manager to the connection
+        # https://github.com/ClickHouse/clickhouse-connect/blob/3a7f4b04cad29c7c2536661b831fb744248e2ec0/clickhouse_connect/driver/httpclient.py#L109
+        if self.server_host_name:
+            pool_manager_options["server_hostname"] = self.server_host_name
+            if self.verify:
+                pool_manager_options["assert_hostname"] = self.server_host_name
         if self.connection_pool_options:
             pool_manager_options.update(self.connection_pool_options)
         pool_mgr = httputil.get_pool_manager(**pool_manager_options)


Test patch:
diff --git a/tests/core/test_connection_config.py b/tests/core/test_connection_config.py
index 63a32e8a..d106559b 100644
--- a/tests/core/test_connection_config.py
+++ b/tests/core/test_connection_config.py
@@ -902,6 +902,13 @@ def test_clickhouse(make_config):
         cluster="default",
         use_compression=True,
         connection_settings={"this_setting": "1"},
+        server_host_name="server_host_name",
+        verify=True,
+        ca_cert="ca_cert",
+        client_cert="client_cert",
+        client_cert_key="client_cert_key",
+        https_proxy="https://proxy",
+        connection_pool_options={"pool_option": "value"},
     )
     assert isinstance(config, ClickhouseConnectionConfig)
     assert config.cluster == "default"
@@ -912,6 +919,14 @@ def test_clickhouse(make_config):
     assert config.is_recommended_for_state_sync is False
     assert config.is_forbidden_for_state_sync
 
+    pool = config._connection_factory.keywords["pool_mgr"]
+    assert pool.connection_pool_kw["server_hostname"] == "server_host_name"
+    assert pool.connection_pool_kw["assert_hostname"] == "server_host_name"  # because verify=True
+    assert pool.connection_pool_kw["ca_certs"] == "ca_cert"
+    assert pool.connection_pool_kw["cert_file"] == "client_cert"
+    assert pool.connection_pool_kw["key_file"] == "client_cert_key"
+    assert pool.connection_pool_kw["pool_option"] == "value"
+
     config2 = make_config(
         type="clickhouse",
         host="localhost",