Repository: getsentry/snuba

Issue:
Make Snuba bootstrap less noisy
Currently, the `bootstrap` command prints out transitional errors to the console for debugging purposes. This causes confusion for self-hosted users when they are installing.

Instead of printing these at all times, we should only show these if the logging level is set to debug/verbose.


Discussion:

Resolving patch:
diff --git a/snuba/cli/bootstrap.py b/snuba/cli/bootstrap.py
--- a/snuba/cli/bootstrap.py
+++ b/snuba/cli/bootstrap.py
@@ -3,16 +3,20 @@
 
 import click
 
+from confluent_kafka import KafkaError, KafkaException
 from snuba.datasets.factory import ACTIVE_DATASET_NAMES, get_dataset
 from snuba.environment import setup_logging
 from snuba.migrations.connect import check_clickhouse_connections
 from snuba.migrations.runner import Runner
+from snuba.utils.logging import pylog_to_syslog_level
 from snuba.utils.streams.backends.kafka import get_default_kafka_configuration
 
 
 @click.command()
 @click.option(
-    "--bootstrap-server", multiple=True, help="Kafka bootstrap server to use.",
+    "--bootstrap-server",
+    multiple=True,
+    help="Kafka bootstrap server to use.",
 )
 @click.option("--kafka/--no-kafka", default=True)
 @click.option("--migrate/--no-migrate", default=True)
@@ -42,27 +46,40 @@ def bootstrap(
         logger.debug("Using Kafka with %r", bootstrap_server)
         from confluent_kafka.admin import AdminClient, NewTopic
 
+        override_params = {
+            # Same as above: override socket timeout as we expect Kafka
+            # to not getting ready for a while
+            "socket.timeout.ms": 1000,
+        }
+        if logger.getEffectiveLevel() != logging.DEBUG:
+            # Override rdkafka loglevel to be critical unless we are
+            # debugging as we expect failures when trying to connect
+            # (Kafka may not be up yet)
+            override_params["log_level"] = pylog_to_syslog_level(logging.CRITICAL)
+
         attempts = 0
         while True:
             try:
-                logger.debug("Attempting to connect to Kafka (attempt %d)", attempts)
+                logger.info("Attempting to connect to Kafka (attempt %d)...", attempts)
                 client = AdminClient(
                     get_default_kafka_configuration(
                         bootstrap_servers=bootstrap_server,
-                        override_params={"socket.timeout.ms": 1000},
+                        override_params=override_params,
                     )
                 )
                 client.list_topics(timeout=1)
                 break
-            except Exception as e:
-                logger.error(
-                    "Connection to Kafka failed (attempt %d)", attempts, exc_info=e
+            except KafkaException as err:
+                logger.debug(
+                    "Connection to Kafka failed (attempt %d)", attempts, exc_info=err
                 )
                 attempts += 1
                 if attempts == 60:
                     raise
                 time.sleep(1)
 
+        logger.info("Connected to Kafka on attempt %d", attempts)
+
         topics = {}
         for name in ACTIVE_DATASET_NAMES:
             dataset = get_dataset(name)
@@ -83,15 +100,16 @@ def bootstrap(
                             replication_factor=topic_spec.replication_factor,
                         )
 
-        logger.debug("Initiating topic creation")
+        logger.info("Creating Kafka topics...")
         for topic, future in client.create_topics(
             list(topics.values()), operation_timeout=1
         ).items():
             try:
                 future.result()
                 logger.info("Topic %s created", topic)
-            except Exception as e:
-                logger.error("Failed to create topic %s", topic, exc_info=e)
+            except KafkaException as err:
+                if err.args[0].code() != KafkaError.TOPIC_ALREADY_EXISTS:
+                    logger.error("Failed to create topic %s", topic, exc_info=err)
 
     if migrate:
         check_clickhouse_connections()
diff --git a/snuba/utils/logging.py b/snuba/utils/logging.py
new file mode 100644
--- /dev/null
+++ b/snuba/utils/logging.py
@@ -0,0 +1,15 @@
+from logging import NOTSET, DEBUG, INFO, WARNING, ERROR, CRITICAL
+
+
+PYTHON_TO_SYSLOG_MAP = {
+    NOTSET: 7,
+    DEBUG: 7,
+    INFO: 6,
+    WARNING: 4,
+    ERROR: 3,
+    CRITICAL: 2,
+}
+
+
+def pylog_to_syslog_level(level: int) -> int:
+    return PYTHON_TO_SYSLOG_MAP.get(level, 7)
diff --git a/snuba/utils/streams/backends/kafka.py b/snuba/utils/streams/backends/kafka.py
--- a/snuba/utils/streams/backends/kafka.py
+++ b/snuba/utils/streams/backends/kafka.py
@@ -40,6 +40,7 @@
 from snuba import settings
 from snuba.datasets.storages import StorageKey
 from snuba.utils.concurrent import execute
+from snuba.utils.logging import pylog_to_syslog_level
 from snuba.utils.retries import NoRetryPolicy, RetryPolicy
 from snuba.utils.streams.backends.abstract import (
     Consumer,
@@ -408,7 +409,9 @@ def poll(self, timeout: Optional[float] = None) -> Optional[Message[KafkaPayload
             Partition(Topic(message.topic()), message.partition()),
             message.offset(),
             KafkaPayload(
-                message.key(), message.value(), headers if headers is not None else [],
+                message.key(),
+                message.value(),
+                headers if headers is not None else [],
             ),
             datetime.utcfromtimestamp(message.timestamp()[1] / 1000.0),
         )
@@ -693,8 +696,14 @@ def get_default_kafka_configuration(
                 f"The `{configuration_key}` configuration key is not supported."
             )
 
+    broker_config["log_level"] = pylog_to_syslog_level(logger.getEffectiveLevel())
+
+    if settings.DEBUG:
+        broker_config["debug"] = "all"
+
     if override_params:
         broker_config.update(override_params)
+
     return broker_config
 
 
@@ -845,7 +854,9 @@ def __delivery_callback(
                 future.set_exception(error)
 
     def produce(
-        self, destination: Union[Topic, Partition], payload: KafkaPayload,
+        self,
+        destination: Union[Topic, Partition],
+        payload: KafkaPayload,
     ) -> Future[Message[KafkaPayload]]:
         if self.__shutdown_requested.is_set():
             raise RuntimeError("producer has been closed")


Test patch: