Repository: MaterializeInc/materialize
Pull request #31929: [sinks] Attempt to fetch (and optionally alter) topic config
Author: login: bkirwi
Created: 2025-03-17T19:07:26Z | Merged: 2025-03-20T22:07:05Z

This reverts commit 0fe9cb1ad43e15d0224d4ee7fc18ddd23df590ba with the logging issue fixed.

This also allows us to attempt reconfiguring the topic if the configuration doesn't match. 

### Motivation

https://github.com/MaterializeInc/database-issues/issues/9032

### Checklist

- [ ] This PR has adequate test coverage / QA involvement has been duly considered. ([trigger-ci for additional test/nightly runs](https://trigger-ci.dev.materialize.com/))
- [ ] This PR has an associated up-to-date [design doc](https://github.com/MaterializeInc/materialize/blob/main/doc/developer/design/README.md), is a design doc ([template](https://github.com/MaterializeInc/materialize/blob/main/doc/developer/design/00000000_template.md)), or is sufficiently small to not require a design.
  <!-- Reference the design in the description. -->
- [ ] If this PR evolves [an existing `$T ⇔ Proto$T` mapping](https://github.com/MaterializeInc/materialize/blob/main/doc/developer/command-and-response-binary-encoding.md) (possibly in a backwards-incompatible way), then it is tagged with a `T-proto` label.
- [ ] If this PR will require changes to cloud orchestration or tests, there is a companion cloud PR to account for those changes that is tagged with the release-blocker label ([example](https://github.com/MaterializeInc/cloud/pull/5021)).
  <!-- Ask in #team-cloud on Slack if you need help preparing the cloud PR. -->
- [ ] If this PR includes major [user-facing behavior changes](https://github.com/MaterializeInc/materialize/blob/main/doc/developer/guide-changes.md#what-changes-require-a-release-note), I have pinged the relevant PM to schedule a changelog post.



Diff:
diff --git a/Cargo.lock b/Cargo.lock
index 4880cd1252932..bf4ed11804723 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -5866,6 +5866,7 @@ dependencies = [
  "crossbeam",
  "fancy-regex",
  "futures",
+ "itertools 0.12.1",
  "mz-avro",
  "mz-build-tools",
  "mz-ccsr",
diff --git a/src/kafka-util/Cargo.toml b/src/kafka-util/Cargo.toml
index 21777581628b6..57a69b284f62a 100644
--- a/src/kafka-util/Cargo.toml
+++ b/src/kafka-util/Cargo.toml
@@ -21,6 +21,7 @@ clap = { version = "4.5.23", features = ["derive"] }
 crossbeam = "0.8.2"
 fancy-regex = "0.14.0"
 futures = "0.3.31"
+itertools = "0.12.1"
 mz-avro = { path = "../avro" }
 mz-ccsr = { path = "../ccsr" }
 mz-ore = { path = "../ore", features = ["cli", "network", "async"] }
diff --git a/src/kafka-util/src/admin.rs b/src/kafka-util/src/admin.rs
index e9a8de0f2f36f..533a3f8961f08 100644
--- a/src/kafka-util/src/admin.rs
+++ b/src/kafka-util/src/admin.rs
@@ -9,15 +9,176 @@
 
 //! Helpers for working with Kafka's admin API.
 
+use std::collections::BTreeMap;
 use std::iter;
 use std::time::Duration;
 
 use anyhow::{anyhow, bail};
+use itertools::Itertools;
 use mz_ore::collections::CollectionExt;
 use mz_ore::retry::Retry;
-use rdkafka::admin::{AdminClient, AdminOptions, NewTopic};
+use mz_ore::str::separated;
+use rdkafka::admin::{
+    AdminClient, AdminOptions, AlterConfig, ConfigEntry, ConfigResource, ConfigSource, NewTopic,
+    OwnedResourceSpecifier, ResourceSpecifier,
+};
 use rdkafka::client::ClientContext;
 use rdkafka::error::{KafkaError, RDKafkaErrorCode};
+use tracing::{info, warn};
+
+/// Get the current configuration for a particular topic.
+///
+/// Materialize may not have permission to list configs for the topic, so callers of this method
+/// should "fail open" if the configs are not available.
+pub async fn get_topic_config<'a, C>(
+    client: &'a AdminClient<C>,
+    admin_opts: &AdminOptions,
+    topic_name: &str,
+) -> anyhow::Result<Vec<ConfigEntry>>
+where
+    C: ClientContext,
+{
+    let ConfigResource { specifier, entries } = client
+        .describe_configs([&ResourceSpecifier::Topic(topic_name)], admin_opts)
+        .await?
+        .into_iter()
+        .exactly_one()??;
+
+    match specifier {
+        OwnedResourceSpecifier::Topic(name) if name.as_str() == topic_name => {}
+        unexpected => {
+            bail!("describe configs returned unexpected resource specifier: {unexpected:?}")
+        }
+    };
+
+    Ok(entries)
+}
+
+/// Alter the current configuration for a particular topic.
+///
+/// Materialize may not have permission to alter configs for the topic, so callers of this method
+/// should "fail open" if the configs are not available.
+pub async fn alter_topic_config<'a, C>(
+    client: &'a AdminClient<C>,
+    admin_opts: &AdminOptions,
+    topic_name: &str,
+    new_config: impl IntoIterator<Item = (&str, &str)>,
+) -> anyhow::Result<()>
+where
+    C: ClientContext,
+{
+    let mut alter_config = AlterConfig::new(ResourceSpecifier::Topic(topic_name));
+    for (key, val) in new_config {
+        alter_config = alter_config.set(key, val);
+    }
+    let result = client
+        .alter_configs([&alter_config], admin_opts)
+        .await?
+        .into_iter()
+        .exactly_one()?;
+
+    let (specifier, result) = match result {
+        Ok(specifier) => (specifier, Ok(())),
+        Err((specifier, err)) => (specifier, Err(KafkaError::AdminOp(err))),
+    };
+
+    match specifier {
+        OwnedResourceSpecifier::Topic(name) if name.as_str() == topic_name => {}
+        unexpected => {
+            bail!("alter configs returned unexpected resource specifier: {unexpected:?}")
+        }
+    };
+
+    Ok(result?)
+}
+
+/// When the topic we want to ensure already exists, what happens?
+#[derive(Debug, Copy, Clone)]
+pub enum EnsureTopicConfig {
+    /// Do nothing: assume whatever config is there is fine.
+    Skip,
+    /// Check and log the existing config, but don't try and change it.
+    Check,
+    /// If the remote config doesn't match, try and change it to match.
+    Alter,
+}
+
+/// Expect the current configuration for a particular topic to match a provided set of values.
+/// If there are other configs set, we will ignore them.
+///
+/// Materialize may not have permission to alter configs for the topic, so callers of this method
+/// should "fail open" if the configs are not available.
+pub async fn ensure_topic_config<'a, C>(
+    client: &'a AdminClient<C>,
+    admin_opts: &AdminOptions,
+    new_topic: &'a NewTopic<'a>,
+    expect: EnsureTopicConfig,
+) -> anyhow::Result<bool>
+where
+    C: ClientContext,
+{
+    let try_alter = match expect {
+        EnsureTopicConfig::Skip => return Ok(true),
+        EnsureTopicConfig::Check => false,
+        EnsureTopicConfig::Alter => true,
+    };
+
+    let mut expected_configs: BTreeMap<_, _> = new_topic.config.iter().copied().collect();
+
+    let actual_configs = get_topic_config(client, admin_opts, new_topic.name).await?;
+    info!(
+        topic = new_topic.name,
+        "got configuration for existing topic: [{}]",
+        separated(
+            ", ",
+            actual_configs.iter().map(|e| {
+                let kv = [&*e.name, e.value.as_ref().map_or("<none>", |v| &*v)];
+                separated(": ", kv)
+            })
+        )
+    );
+
+    let actual_config_values: BTreeMap<_, _> = actual_configs
+        .iter()
+        .filter_map(|e| e.value.as_ref().map(|v| (e.name.as_str(), v.as_str())))
+        .collect();
+    for (config, expected) in &expected_configs {
+        match actual_config_values.get(config) {
+            Some(actual) => {
+                if actual != expected {
+                    warn!(
+                        topic = new_topic.name,
+                        config, expected, actual, "unexpected value for config entry"
+                    )
+                }
+            }
+            None => {
+                warn!(
+                    topic = new_topic.name,
+                    config, expected, "missing expected value for config entry"
+                )
+            }
+        }
+    }
+
+    if try_alter {
+        // rdkafka does not support the newer incremental-alter APIs. Instead, we copy over values
+        // from the fetched dynamic config that we don't explicitly want to overwrite.
+        for entry in &actual_configs {
+            if entry.source != ConfigSource::DynamicTopic {
+                continue;
+            }
+            let Some(value) = entry.value.as_ref() else {
+                continue;
+            };
+            expected_configs.entry(&entry.name).or_insert(value);
+        }
+        alter_topic_config(client, admin_opts, new_topic.name, expected_configs).await?;
+        Ok(true)
+    } else {
+        Ok(false)
+    }
+}
 
 /// Creates a Kafka topic if it does not exist, and waits for it to be reported in the broker
 /// metadata.
@@ -40,6 +201,7 @@ pub async fn ensure_topic<'a, C>(
     client: &'a AdminClient<C>,
     admin_opts: &AdminOptions,
     new_topic: &'a NewTopic<'a>,
+    on_existing: EnsureTopicConfig,
 ) -> anyhow::Result<bool>
 where
     C: ClientContext,
@@ -60,6 +222,22 @@ where
 
     // We don't need to read in metadata / do any validation if the topic already exists.
     if already_exists {
+        match ensure_topic_config(client, admin_opts, new_topic, on_existing).await {
+            Ok(true) => {}
+            Ok(false) => {
+                info!(
+                    topic = new_topic.name,
+                    "did not sync topic config; configs may not match expected values"
+                );
+            }
+            Err(error) => {
+                warn!(
+                    topic=new_topic.name,
+                    "unable to enforce topic config; configs may not match expected values: {error:#}"
+                )
+            }
+        }
+
         return Ok(true);
     }
 
diff --git a/src/storage-client/src/sink.rs b/src/storage-client/src/sink.rs
index 0ebd0b7cb8d0a..2208e167bbd6f 100644
--- a/src/storage-client/src/sink.rs
+++ b/src/storage-client/src/sink.rs
@@ -12,6 +12,7 @@ use std::time::Duration;
 
 use anyhow::{anyhow, bail, Context};
 use mz_ccsr::GetSubjectConfigError;
+use mz_kafka_util::admin::EnsureTopicConfig;
 use mz_kafka_util::client::MzClientContext;
 use mz_ore::collections::CollectionExt;
 use mz_ore::future::{InTask, OreFutureExt};
@@ -159,6 +160,7 @@ pub async fn ensure_kafka_topic(
         replication_factor,
         topic_config,
     }: &KafkaTopicOptions,
+    ensure_topic_config: EnsureTopicConfig,
 ) -> Result<bool, anyhow::Error> {
     let client: AdminClient<_> = connection
         .connection
@@ -219,6 +221,7 @@ pub async fn ensure_kafka_topic(
         &client,
         &AdminOptions::new().request_timeout(Some(Duration::from_secs(5))),
         &kafka_topic,
+        ensure_topic_config,
     )
     .await
     .with_context(|| format!("Error creating topic {} for sink", topic))
diff --git a/src/storage-types/src/dyncfgs.rs b/src/storage-types/src/dyncfgs.rs
index 4ab9d12848887..a525b5d06641c 100644
--- a/src/storage-types/src/dyncfgs.rs
+++ b/src/storage-types/src/dyncfgs.rs
@@ -263,6 +263,15 @@ pub const SINK_PROGRESS_SEARCH: Config<bool> = Config::new(
     "If set, iteratively search the progress topic for a progress record with increasing lookback.",
 );
 
+/// Configure how to behave when trying to create an existing topic with specified configs.
+pub const SINK_ENSURE_TOPIC_CONFIG: Config<&'static str> = Config::new(
+    "storage_sink_ensure_topic_config",
+    "skip",
+    "If `skip`, don't check the config of existing topics; if `check`, fetch the config and \
+    warn if it does not match the expected configs; if `alter`, attempt to change the upstream to \
+    match the expected configs.",
+);
+
 /// Adds the full set of all storage `Config`s.
 pub fn all_dyncfgs(configs: ConfigSet) -> ConfigSet {
     configs
@@ -293,4 +302,5 @@ pub fn all_dyncfgs(configs: ConfigSet) -> ConfigSet {
         .add(&STORAGE_USE_CONTINUAL_FEEDBACK_UPSERT)
         .add(&STORAGE_SERVER_MAINTENANCE_INTERVAL)
         .add(&SINK_PROGRESS_SEARCH)
+        .add(&SINK_ENSURE_TOPIC_CONFIG)
 }
diff --git a/src/storage/src/sink/kafka.rs b/src/storage/src/sink/kafka.rs
index 81a645a3f29be..c3bff42f9692e 100644
--- a/src/storage/src/sink/kafka.rs
+++ b/src/storage/src/sink/kafka.rs
@@ -79,6 +79,11 @@ use std::sync::atomic::AtomicU64;
 use std::sync::{Arc, Weak};
 use std::time::Duration;
 
+use crate::healthcheck::{HealthStatusMessage, HealthStatusUpdate, StatusNamespace};
+use crate::metrics::sink::kafka::KafkaSinkMetrics;
+use crate::render::sinks::SinkRender;
+use crate::statistics::SinkStatistics;
+use crate::storage_state::StorageState;
 use anyhow::{anyhow, bail, Context};
 use differential_dataflow::{AsCollection, Collection, Hashable};
 use futures::StreamExt;
@@ -89,6 +94,7 @@ use mz_interchange::encode::Encode;
 use mz_interchange::envelopes::dbz_format;
 use mz_interchange::json::JsonEncoder;
 use mz_interchange::text_binary::{BinaryEncoder, TextEncoder};
+use mz_kafka_util::admin::EnsureTopicConfig;
 use mz_kafka_util::client::{
     GetPartitionsError, MzClientContext, TimeoutConfig, TunnelingClientContext,
     DEFAULT_FETCH_METADATA_TIMEOUT,
@@ -107,7 +113,7 @@ use mz_storage_client::sink::progress_key::ProgressKey;
 use mz_storage_types::configuration::StorageConfiguration;
 use mz_storage_types::controller::CollectionMetadata;
 use mz_storage_types::dyncfgs::{
-    KAFKA_BUFFERED_EVENT_RESIZE_THRESHOLD_ELEMENTS, SINK_PROGRESS_SEARCH,
+    KAFKA_BUFFERED_EVENT_RESIZE_THRESHOLD_ELEMENTS, SINK_ENSURE_TOPIC_CONFIG, SINK_PROGRESS_SEARCH,
 };
 use mz_storage_types::errors::{ContextCreationError, ContextCreationErrorExt, DataflowError};
 use mz_storage_types::sinks::{
@@ -134,12 +140,6 @@ use tokio::sync::watch;
 use tokio::time::{self, MissedTickBehavior};
 use tracing::{debug, error, info, warn};
 
-use crate::healthcheck::{HealthStatusMessage, HealthStatusUpdate, StatusNamespace};
-use crate::metrics::sink::kafka::KafkaSinkMetrics;
-use crate::render::sinks::SinkRender;
-use crate::statistics::SinkStatistics;
-use crate::storage_state::StorageState;
-
 impl<G: Scope<Timestamp = Timestamp>> SinkRender<G> for KafkaSinkConnection {
     fn get_key_indices(&self) -> Option<&[usize]> {
         self.key_desc_and_indices
@@ -399,6 +399,7 @@ impl TransactionalProducer {
                     storage_configuration,
                     &connection.topic,
                     &connection.topic_options,
+                    EnsureTopicConfig::Skip,
                 )
                 .await?;
                 Antichain::from_elem(Timestamp::minimum())
@@ -928,11 +929,25 @@ async fn determine_sink_progress(
     let ctx = Arc::clone(progress_client_read_committed.client().context());
 
     // Ensure the progress topic exists.
+    let ensure_topic_config =
+        match &*SINK_ENSURE_TOPIC_CONFIG.get(storage_configuration.config_set()) {
+            "skip" => EnsureTopicConfig::Skip,
+            "check" => EnsureTopicConfig::Check,
+            "alter" => EnsureTopicConfig::Alter,
+            _ => {
+                tracing::warn!(
+                    topic = progress_topic,
+                    "unexpected value for ensure-topic-config; skipping checks"
+                );
+                EnsureTopicConfig::Skip
+            }
+        };
     mz_storage_client::sink::ensure_kafka_topic(
         connection,
         storage_configuration,
         &progress_topic,
         progress_topic_options,
+        ensure_topic_config,
     )
     .await
     .add_context("error registering kafka progress topic for sink")?;
diff --git a/src/testdrive/src/action/kafka/create_topic.rs b/src/testdrive/src/action/kafka/create_topic.rs
index d26fbde53f34b..3c0095ab8931c 100644
--- a/src/testdrive/src/action/kafka/create_topic.rs
+++ b/src/testdrive/src/action/kafka/create_topic.rs
@@ -7,11 +7,11 @@
 // the Business Source License, use of this software will be governed
 // by the Apache License, Version 2.0.
 
-use anyhow::anyhow;
-use rdkafka::admin::{NewTopic, TopicReplication};
-
 use crate::action::{ControlFlow, State};
 use crate::parser::BuiltinCommand;
+use anyhow::anyhow;
+use mz_kafka_util::admin::EnsureTopicConfig;
+use rdkafka::admin::{NewTopic, TopicReplication};
 
 pub async fn run_create_topic(
     mut cmd: BuiltinCommand,
@@ -113,8 +113,13 @@ pub async fn run_create_topic(
         new_topic
     };
 
-    mz_kafka_util::admin::ensure_topic(&state.kafka_admin, &state.kafka_admin_opts, &new_topic)
-        .await?;
+    mz_kafka_util::admin::ensure_topic(
+        &state.kafka_admin,
+        &state.kafka_admin_opts,
+        &new_topic,
+        EnsureTopicConfig::Check,
+    )
+    .await?;
     state.kafka_topics.insert(topic_name, partitions);
     Ok(ControlFlow::Continue)
 }
diff --git a/test/test-util/src/kafka/kafka_client.rs b/test/test-util/src/kafka/kafka_client.rs
index 71f3f46f87444..88946247d4634 100644
--- a/test/test-util/src/kafka/kafka_client.rs
+++ b/test/test-util/src/kafka/kafka_client.rs
@@ -12,12 +12,12 @@
 use std::time::Duration;
 
 use anyhow::Context;
+use mz_kafka_util::admin::EnsureTopicConfig;
+use mz_kafka_util::client::{create_new_client_config_simple, MzClientContext};
 use rdkafka::admin::{AdminClient, AdminOptions, NewTopic, TopicReplication};
 use rdkafka::error::KafkaError;
 use rdkafka::producer::{DeliveryFuture, FutureProducer, FutureRecord};
 
-use mz_kafka_util::client::{create_new_client_config_simple, MzClientContext};
-
 pub struct KafkaClient {
     producer: FutureProducer<MzClientContext>,
     kafka_url: String,
@@ -66,7 +66,7 @@ impl KafkaClient {
             topic = topic.set(key, val);
         }
 
-        mz_kafka_util::admin::ensure_topic(&client, &admin_opts, &topic)
+        mz_kafka_util::admin::ensure_topic(&client, &admin_opts, &topic, EnsureTopicConfig::Check)
             .await
             .context(format!("creating Kafka topic: {}", topic_name))?;
 
diff --git a/test/testdrive/kafka-sink-topic-config.td b/test/testdrive/kafka-sink-topic-config.td
index 7202cca596bb5..db6cba014d0d4 100644
--- a/test/testdrive/kafka-sink-topic-config.td
+++ b/test/testdrive/kafka-sink-topic-config.td
@@ -79,3 +79,28 @@ SELECT '${arg.uses-redpanda}'::BOOL
 
 > SELECT status, error FROM mz_internal.mz_sink_statuses WHERE name = 'topic_config_unknown';
 stalled "kafka: Error creating topic testdrive-kafka-config-unknown-${testdrive.seed} for sink: Admin operation error: InvalidConfig (Broker: Configuration is invalid)"
+
+# Test whether MZ can alter the progress topic configuration when the relevant option is enabled
+
+$ postgres-execute connection=postgres://mz_system:materialize@${testdrive.materialize-internal-sql-addr}
+ALTER SYSTEM SET storage_sink_ensure_topic_config = 'alter'
+
+$ kafka-create-topic topic=kafka-progress partitions=1 compaction=false
+
+$ kafka-verify-topic topic=testdrive-kafka-progress-${testdrive.seed} topic-config={"cleanup.policy": "delete"} replication-factor=1 partition-count=1
+
+> CREATE CONNECTION kafka_conn_progress
+  TO KAFKA (BROKER '${testdrive.kafka-addr}', SECURITY PROTOCOL PLAINTEXT, PROGRESS TOPIC 'testdrive-kafka-progress-${testdrive.seed}');
+
+> CREATE SINK topic_config_check
+  IN CLUSTER ${arg.single-replica-cluster}
+  FROM v1
+  INTO KAFKA CONNECTION kafka_conn_progress (
+    TOPIC 'testdrive-kafka-config-check-${testdrive.seed}',
+    TOPIC PARTITION COUNT 3
+  )
+  KEY(f1)
+  FORMAT JSON
+  ENVELOPE UPSERT
+
+$ kafka-verify-topic topic=testdrive-kafka-progress-${testdrive.seed} topic-config={"cleanup.policy": "compact"} replication-factor=1 partition-count=1


Reviews:
user: login: petrosagg
body: Looks great!
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-03-20T15:32:55Z
user: login: bkirwi
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-20T21:12:01Z
user: login: bkirwi
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-20T21:13:17Z

Review comments:
diff_hunk: @@ -29,10 +30,10 @@ num_cpus = "1.16.0"
 prost = { version = "0.13.5", features = ["no-recursion-limit"] }
 rand = "0.8.5"
 rdkafka = { version = "0.29.0", features = ["cmake-build", "ssl-vendored", "libz-static", "zstd"] }
-serde = { version = "1.0.219", features = ["derive"] }
+serde = { version = "1.0.218", features = ["derive"] }
 serde_json = "1.0.125"
-tokio = { version = "1.44.1", features = ["macros", "rt", "sync"] }
-thiserror = "2.0.12"
+tokio = { version = "1.38.0", features = ["macros", "rt", "sync"] }
path: src/kafka-util/Cargo.toml
user: login: petrosagg
body: Are these downgrades intentional?
created_at: 2025-03-20T15:18:53Z
author_association: CONTRIBUTOR
diff_hunk: @@ -9,15 +9,175 @@
 
 //! Helpers for working with Kafka's admin API.
 
+use std::collections::BTreeMap;
 use std::iter;
 use std::time::Duration;
 
 use anyhow::{anyhow, bail};
+use itertools::Itertools;
 use mz_ore::collections::CollectionExt;
 use mz_ore::retry::Retry;
-use rdkafka::admin::{AdminClient, AdminOptions, NewTopic};
+use mz_ore::str::separated;
+use rdkafka::admin::{
+    AdminClient, AdminOptions, AlterConfig, ConfigEntry, ConfigResource, ConfigSource, NewTopic,
+    OwnedResourceSpecifier, ResourceSpecifier,
+};
 use rdkafka::client::ClientContext;
 use rdkafka::error::{KafkaError, RDKafkaErrorCode};
+use tracing::{info, warn};
+
+/// Get the current configuration for a particular topic.
+///
+/// Materialize may not have permission to list configs for the topic, so callers of this method
+/// should "fail open" if the configs are not available.
+pub async fn get_topic_config<'a, C>(
+    client: &'a AdminClient<C>,
+    admin_opts: &AdminOptions,
+    topic_name: &str,
+) -> anyhow::Result<Vec<ConfigEntry>>
+where
+    C: ClientContext,
+{
+    let ConfigResource { specifier, entries } = client
+        .describe_configs([&ResourceSpecifier::Topic(topic_name)], admin_opts)
+        .await?
+        .into_iter()
+        .exactly_one()??;
+
+    match specifier {
+        OwnedResourceSpecifier::Topic(name) if name.as_str() == topic_name => {}
+        unexpected => {
+            bail!("describe configs returned unexpected resource specifier: {unexpected:?}")
+        }
+    };
+
+    Ok(entries)
+}
+
+/// Alter the current configuration for a particular topic.
+///
+/// Materialize may not have permission to alter configs for the topic, so callers of this method
+/// should "fail open" if the configs are not available.
+pub async fn alter_topic_config<'a, C>(
+    client: &'a AdminClient<C>,
+    admin_opts: &AdminOptions,
+    topic_name: &str,
+    new_config: impl IntoIterator<Item = (&str, &str)>,
+) -> anyhow::Result<()>
+where
+    C: ClientContext,
+{
+    let mut alter_config = AlterConfig::new(ResourceSpecifier::Topic(topic_name));
+    for (key, val) in new_config {
+        alter_config = alter_config.set(key, val);
+    }
+    let result = client
+        .alter_configs([&alter_config], admin_opts)
+        .await?
+        .into_iter()
+        .exactly_one()?;
+
+    let (specifier, result) = match result {
+        Ok(specifier) => (specifier, Ok(())),
+        Err((specifier, err)) => (specifier, Err(KafkaError::AdminOp(err))),
+    };
+
+    match specifier {
+        OwnedResourceSpecifier::Topic(name) if name.as_str() == topic_name => {}
+        unexpected => {
+            bail!("alter configs returned unexpected resource specifier: {unexpected:?}")
+        }
+    };
+
+    Ok(result?)
+}
+
+/// When the topic we want to ensure already exists, what happens?
+#[derive(Debug, Copy, Clone)]
+pub enum EnsureTopicConfig {
+    /// Do nothing: assume whatever config is there is fine.
+    Skip,
+    /// Check and log the existing config, but don't try and change it.
+    Check,
+    /// If the remote config doesn't match, try and change it to match.
+    Alter,
+}
+
+/// Expect the current configuration for a particular topic to match a provided set of values.
+/// If there are other configs set, we will ignore them.
+///
+/// Materialize may not have permission to alter configs for the topic, so callers of this method
+/// should "fail open" if the configs are not available.
+pub async fn ensure_topic_config<'a, C>(
+    client: &'a AdminClient<C>,
+    admin_opts: &AdminOptions,
+    new_topic: &'a NewTopic<'a>,
+    expect: EnsureTopicConfig,
+) -> anyhow::Result<bool>
+where
+    C: ClientContext,
+{
+    let try_alter = match expect {
+        EnsureTopicConfig::Skip => return Ok(true),
+        EnsureTopicConfig::Check => false,
+        EnsureTopicConfig::Alter => true,
+    };
+
+    let mut expected_configs: BTreeMap<_, _> = new_topic.config.iter().copied().collect();
+
+    let actual_configs = get_topic_config(client, admin_opts, new_topic.name).await?;
+    info!(
+        topic = new_topic.name,
+        "got configuration for existing topic: [{}]",
+        separated(
+            ", ",
+            actual_configs.iter().map(|e| {
+                let kv = [&*e.name, e.value.as_ref().map_or("<none>", |v| &*v)];
+                separated(": ", kv)
+            })
+        )
+    );
+
+    for (config, expected) in &expected_configs {
+        match actual_configs.iter().find(|c| &c.name == *config) {
path: src/kafka-util/src/admin.rs
user: login: petrosagg
body: This loop quadratic to the number of configs, should we also dump the actual configs in a btreemap?
created_at: 2025-03-20T15:29:56Z
author_association: CONTRIBUTOR
diff_hunk: @@ -29,10 +30,10 @@ num_cpus = "1.16.0"
 prost = { version = "0.13.5", features = ["no-recursion-limit"] }
 rand = "0.8.5"
 rdkafka = { version = "0.29.0", features = ["cmake-build", "ssl-vendored", "libz-static", "zstd"] }
-serde = { version = "1.0.219", features = ["derive"] }
+serde = { version = "1.0.218", features = ["derive"] }
 serde_json = "1.0.125"
-tokio = { version = "1.44.1", features = ["macros", "rt", "sync"] }
-thiserror = "2.0.12"
+tokio = { version = "1.38.0", features = ["macros", "rt", "sync"] }
path: src/kafka-util/Cargo.toml
user: login: bkirwi
body: Not intentional - I'll back these out!
created_at: 2025-03-20T21:12:01Z
author_association: CONTRIBUTOR
diff_hunk: @@ -9,15 +9,175 @@
 
 //! Helpers for working with Kafka's admin API.
 
+use std::collections::BTreeMap;
 use std::iter;
 use std::time::Duration;
 
 use anyhow::{anyhow, bail};
+use itertools::Itertools;
 use mz_ore::collections::CollectionExt;
 use mz_ore::retry::Retry;
-use rdkafka::admin::{AdminClient, AdminOptions, NewTopic};
+use mz_ore::str::separated;
+use rdkafka::admin::{
+    AdminClient, AdminOptions, AlterConfig, ConfigEntry, ConfigResource, ConfigSource, NewTopic,
+    OwnedResourceSpecifier, ResourceSpecifier,
+};
 use rdkafka::client::ClientContext;
 use rdkafka::error::{KafkaError, RDKafkaErrorCode};
+use tracing::{info, warn};
+
+/// Get the current configuration for a particular topic.
+///
+/// Materialize may not have permission to list configs for the topic, so callers of this method
+/// should "fail open" if the configs are not available.
+pub async fn get_topic_config<'a, C>(
+    client: &'a AdminClient<C>,
+    admin_opts: &AdminOptions,
+    topic_name: &str,
+) -> anyhow::Result<Vec<ConfigEntry>>
+where
+    C: ClientContext,
+{
+    let ConfigResource { specifier, entries } = client
+        .describe_configs([&ResourceSpecifier::Topic(topic_name)], admin_opts)
+        .await?
+        .into_iter()
+        .exactly_one()??;
+
+    match specifier {
+        OwnedResourceSpecifier::Topic(name) if name.as_str() == topic_name => {}
+        unexpected => {
+            bail!("describe configs returned unexpected resource specifier: {unexpected:?}")
+        }
+    };
+
+    Ok(entries)
+}
+
+/// Alter the current configuration for a particular topic.
+///
+/// Materialize may not have permission to alter configs for the topic, so callers of this method
+/// should "fail open" if the configs are not available.
+pub async fn alter_topic_config<'a, C>(
+    client: &'a AdminClient<C>,
+    admin_opts: &AdminOptions,
+    topic_name: &str,
+    new_config: impl IntoIterator<Item = (&str, &str)>,
+) -> anyhow::Result<()>
+where
+    C: ClientContext,
+{
+    let mut alter_config = AlterConfig::new(ResourceSpecifier::Topic(topic_name));
+    for (key, val) in new_config {
+        alter_config = alter_config.set(key, val);
+    }
+    let result = client
+        .alter_configs([&alter_config], admin_opts)
+        .await?
+        .into_iter()
+        .exactly_one()?;
+
+    let (specifier, result) = match result {
+        Ok(specifier) => (specifier, Ok(())),
+        Err((specifier, err)) => (specifier, Err(KafkaError::AdminOp(err))),
+    };
+
+    match specifier {
+        OwnedResourceSpecifier::Topic(name) if name.as_str() == topic_name => {}
+        unexpected => {
+            bail!("alter configs returned unexpected resource specifier: {unexpected:?}")
+        }
+    };
+
+    Ok(result?)
+}
+
+/// When the topic we want to ensure already exists, what happens?
+#[derive(Debug, Copy, Clone)]
+pub enum EnsureTopicConfig {
+    /// Do nothing: assume whatever config is there is fine.
+    Skip,
+    /// Check and log the existing config, but don't try and change it.
+    Check,
+    /// If the remote config doesn't match, try and change it to match.
+    Alter,
+}
+
+/// Expect the current configuration for a particular topic to match a provided set of values.
+/// If there are other configs set, we will ignore them.
+///
+/// Materialize may not have permission to alter configs for the topic, so callers of this method
+/// should "fail open" if the configs are not available.
+pub async fn ensure_topic_config<'a, C>(
+    client: &'a AdminClient<C>,
+    admin_opts: &AdminOptions,
+    new_topic: &'a NewTopic<'a>,
+    expect: EnsureTopicConfig,
+) -> anyhow::Result<bool>
+where
+    C: ClientContext,
+{
+    let try_alter = match expect {
+        EnsureTopicConfig::Skip => return Ok(true),
+        EnsureTopicConfig::Check => false,
+        EnsureTopicConfig::Alter => true,
+    };
+
+    let mut expected_configs: BTreeMap<_, _> = new_topic.config.iter().copied().collect();
+
+    let actual_configs = get_topic_config(client, admin_opts, new_topic.name).await?;
+    info!(
+        topic = new_topic.name,
+        "got configuration for existing topic: [{}]",
+        separated(
+            ", ",
+            actual_configs.iter().map(|e| {
+                let kv = [&*e.name, e.value.as_ref().map_or("<none>", |v| &*v)];
+                separated(": ", kv)
+            })
+        )
+    );
+
+    for (config, expected) in &expected_configs {
+        match actual_configs.iter().find(|c| &c.name == *config) {
path: src/kafka-util/src/admin.rs
user: login: bkirwi
body: The thinking was that both of these are bounded by the set of valid configs, which is small... but honestly probably easier to just build the map than to comment the reasoning, so I'll do that!
created_at: 2025-03-20T21:13:17Z
author_association: CONTRIBUTOR

PR comments:
user: login: bkirwi
created_at: 2025-03-17T19:23:45Z
body: CI failure that confirms that the new lint catches the panicky `Itertools::with_format` method: https://buildkite.com/materialize/test/builds/100850#0195a584-a554-4a2b-a0db-016256a91bd1
author_association: CONTRIBUTOR
user: login: bkirwi
created_at: 2025-03-18T14:51:04Z
body: I added a flag, though it will only allow enabling this option for the progress topic for now. Let me know if that doesn't feel right!
author_association: CONTRIBUTOR