Repository: redpanda-data/redpanda
Pull request #25341: Move the bad timestamp metric
Author: login: travisdowns
Labels: name: area/redpanda
Created: 2025-03-12T20:08:09Z | Merged: 2025-03-13T05:51:59Z

This metric is in the wrong probe: it is in the generic "server" probe which applies to all `net::server` classes, but it is a kafka specific concept and only ever incremented in the kafka server. So we have an always zero version of this metric for internal rpc.

Move it to the right place and do some associated cleanup.

The surviving metric name is unchanged.

## Backports Required

<!-- Checking at least one of the checkboxes is REQUIRED if this PR is not a backport. -->

- [ ] none - not a bug fix
- [ ] none - this is a backport
- [ ] none - issue does not exist in previous branches
- [x] none - papercut/not impactful enough to backport
- [ ] v24.3.x
- [ ] v24.2.x
- [ ] v24.1.x

## Release Notes

### Bug Fixes

* The `vectorized_internal_rpc_produce_bad_create_time` metric has been removed. This metric was introduced in error and never had a value other than zero.



Diff:
diff --git a/src/v/kafka/server/handlers/produce.cc b/src/v/kafka/server/handlers/produce.cc
index 05d55e8ba1037..421c1a207eb1e 100644
--- a/src/v/kafka/server/handlers/produce.cc
+++ b/src/v/kafka/server/handlers/produce.cc
@@ -199,7 +199,7 @@ auto validate_batch_timestamps(
   const model::ntp& ntp,
   const model::record_batch_header& header,
   model::timestamp_type timestamp_type,
-  net::server_probe& probe) -> std::optional<model::timestamp> {
+  kafka::kafka_probe& probe) -> std::optional<model::timestamp> {
     // we compute in std::chrono::timepoints, we print in model::timestamps
     auto broker_time = model::timestamp::now();
     auto broker_timepoint = model::duration_since_epoch(broker_time);
@@ -302,10 +302,7 @@ partition_produce_stages produce_topic_partition(
     // validate the batch timestamps by checking skew against broker time
     if (
       auto new_timestamp = validate_batch_timestamps(
-        ntp,
-        batch->header(),
-        cfg_ctx.timestamp_type,
-        octx.rctx.server_probe())) {
+        ntp, batch->header(), cfg_ctx.timestamp_type, octx.rctx.probe())) {
         batch->set_max_timestamp(
           model::timestamp_type::append_time, new_timestamp.value());
     }
diff --git a/src/v/kafka/server/kafka_probe.h b/src/v/kafka/server/kafka_probe.h
index 85e5768ef19cf..c75914464cc6d 100644
--- a/src/v/kafka/server/kafka_probe.h
+++ b/src/v/kafka/server/kafka_probe.h
@@ -21,6 +21,8 @@
 #include <chrono>
 #include <cstdint>
 
+struct prod_consume_fixture;
+
 namespace kafka {
 class kafka_probe {
 public:
@@ -72,6 +74,17 @@ class kafka_probe {
           },
           {},
           {sm::shard_label});
+
+        _metrics.add_group(
+          "kafka_rpc",
+          {sm::make_counter(
+            "produce_bad_create_time",
+            [this] { return _produce_bad_create_time; },
+            sm::description(
+              "number of produce requests with timestamps too far "
+              "in the future or in the past"))},
+          {},
+          {sm::shard_label});
     }
 
     void setup_public_metrics() {
@@ -98,6 +111,12 @@ class kafka_probe {
           });
     }
 
+    // metric used to signal a produce request with a timestamp too far into the
+    // future or too far in the past see configuration
+    // log_message_timestamp_alert_after_ms and
+    // log_message_timestamp_alert_before_ms
+    void produce_bad_create_time() { _produce_bad_create_time++; }
+
     std::unique_ptr<hist_t::measurement> auto_produce_measurement() {
         return _produce_latency.auto_measure();
     }
@@ -109,6 +128,11 @@ class kafka_probe {
     void record_batch(uint64_t size) { _batch_size.record(size); }
 
 private:
+    // for testing
+    friend prod_consume_fixture;
+
+    uint32_t _produce_bad_create_time = 0;
+
     hist_t _produce_latency;
     hist_t _fetch_latency;
     // non partition or topic related as that is too expensive for histograms
diff --git a/src/v/kafka/server/request_context.h b/src/v/kafka/server/request_context.h
index 6fb983a6c8e6f..79b653b811d2d 100644
--- a/src/v/kafka/server/request_context.h
+++ b/src/v/kafka/server/request_context.h
@@ -118,12 +118,11 @@ class request_context {
 
     protocol::decoder& reader() { return _reader; }
 
+    // Return the kafka_probe associated with the request, suitable for
+    // kafka-specific metrics which do not need to be tracked per NTP.
     kafka_probe& probe() { return _conn->server().kafka_probe(); }
     sasl_probe& sasl_probe() { return _conn->server().sasl_probe(); }
 
-    // used to reach for server_probe::produce_bad_timestamp
-    net::server_probe& server_probe() { return _conn->server().probe(); }
-
     kafka::usage_manager& usage_mgr() const {
         return _conn->server().usage_mgr();
     }
diff --git a/src/v/kafka/server/tests/produce_consume_test.cc b/src/v/kafka/server/tests/produce_consume_test.cc
index 03c60e492528a..b5c1fe6c69ff7 100644
--- a/src/v/kafka/server/tests/produce_consume_test.cc
+++ b/src/v/kafka/server/tests/produce_consume_test.cc
@@ -165,6 +165,14 @@ struct prod_consume_fixture : public redpanda_thread_fixture {
         return fetch_next(consumers.front(), model::partition_id{0});
     }
 
+    kafka::kafka_probe& kafka_probe() {
+        return app._kafka_server.local().kafka_probe();
+    }
+
+    uint32_t produce_bad_ts_count() {
+        return kafka_probe()._produce_bad_create_time;
+    };
+
     std::vector<model::offset> fetch_offsets;
     std::vector<kafka::client::transport> consumers;
     std::vector<kafka::client::transport> producers;
@@ -858,51 +866,39 @@ FIXTURE_TEST(test_produce_bad_timestamps, prod_consume_fixture) {
     };
 
     BOOST_TEST_INFO("expect produce_bad_create_time to be 0");
-    auto bad_timestamps_metric
-      = app._kafka_server.local().probe().get_produce_bad_create_time();
+    auto bad_timestamps_metric = produce_bad_ts_count();
     BOOST_CHECK_EQUAL(0, bad_timestamps_metric);
 
     BOOST_TEST_INFO("messages with no skew do not trigger the probe");
     produce_messages(0s);
-    BOOST_CHECK_EQUAL(
-      0, app._kafka_server.local().probe().get_produce_bad_create_time());
+    BOOST_CHECK_EQUAL(0, produce_bad_ts_count());
 
     BOOST_TEST_INFO(
       "messages with a skew towards the future trigger the probe");
     config::shard_local_cfg().log_message_timestamp_alert_after_ms.set_value(
       std::chrono::duration_cast<std::chrono::milliseconds>(1h));
     produce_messages(2h);
-    BOOST_CHECK_LT(
-      bad_timestamps_metric,
-      app._kafka_server.local().probe().get_produce_bad_create_time());
+    BOOST_CHECK_LT(bad_timestamps_metric, produce_bad_ts_count());
 
-    bad_timestamps_metric
-      = app._kafka_server.local().probe().get_produce_bad_create_time();
+    bad_timestamps_metric = produce_bad_ts_count();
 
     BOOST_TEST_INFO("messages with a skew towards the past trigger the probe");
     config::shard_local_cfg().log_message_timestamp_alert_before_ms.set_value(
       std::optional{std::chrono::duration_cast<std::chrono::milliseconds>(1h)});
     produce_messages(-2h);
-    BOOST_CHECK_LT(
-      bad_timestamps_metric,
-      app._kafka_server.local().probe().get_produce_bad_create_time());
+    BOOST_CHECK_LT(bad_timestamps_metric, produce_bad_ts_count());
 
-    bad_timestamps_metric
-      = app._kafka_server.local().probe().get_produce_bad_create_time();
+    bad_timestamps_metric = produce_bad_ts_count();
 
     BOOST_TEST_INFO("messages within the bounds to not trigger the probe");
     produce_messages(-30min);
     produce_messages(30min);
-    BOOST_CHECK_EQUAL(
-      bad_timestamps_metric,
-      app._kafka_server.local().probe().get_produce_bad_create_time());
+    BOOST_CHECK_EQUAL(bad_timestamps_metric, produce_bad_ts_count());
 
     BOOST_TEST_INFO("disabling the alert for the past allows messages in the "
                     "past without triggering the probe");
     config::shard_local_cfg().log_message_timestamp_alert_before_ms.set_value(
       std::optional<std::chrono::milliseconds>{});
     produce_messages(-365 * 24h);
-    BOOST_CHECK_EQUAL(
-      bad_timestamps_metric,
-      app._kafka_server.local().probe().get_produce_bad_create_time());
+    BOOST_CHECK_EQUAL(bad_timestamps_metric, produce_bad_ts_count());
 }
diff --git a/src/v/net/probes.cc b/src/v/net/probes.cc
index 8f9496c9573de..4e10cc5e36e78 100644
--- a/src/v/net/probes.cc
+++ b/src/v/net/probes.cc
@@ -116,11 +116,6 @@ void server_probe::setup_metrics(
           sm::description(ssx::sformat(
             "{}: Number of connections are blocked by connection rate",
             proto))),
-        sm::make_counter(
-          "produce_bad_create_time",
-          [this] { return _produce_bad_create_time; },
-          sm::description("number of produce requests with timestamps too far "
-                          "in the future or in the past")),
       },
       {},
       {sm::shard_label});
@@ -186,9 +181,7 @@ std::ostream& operator<<(std::ostream& o, const server_probe& p) {
       << "corrupted headers: " << p._corrupted_headers << ", "
       << "method not found errors: " << p._method_not_found_errors << ", "
       << "requests blocked by memory: " << p._requests_blocked_memory << ", "
-      << "connections wait rate: " << p._connections_wait_rate << ", "
-      << "produce bad create time: " << p._produce_bad_create_time << ", "
-      << "}";
+      << "connections wait rate: " << p._connections_wait_rate << "}";
     return o;
 }
 
diff --git a/src/v/net/server_probe.h b/src/v/net/server_probe.h
index aa3ff582bb406..c6c49d29ff940 100644
--- a/src/v/net/server_probe.h
+++ b/src/v/net/server_probe.h
@@ -64,16 +64,6 @@ class server_probe {
 
     void waiting_for_conection_rate() { ++_connections_wait_rate; }
 
-    // metric used to signal a produce request with a timestamp too far into the
-    // future or too far in the past see configuration
-    // log_message_timestamp_alert_after_ms and
-    // log_message_timestamp_alert_before_ms
-    void produce_bad_create_time() { _produce_bad_create_time++; }
-    // for testing
-    auto get_produce_bad_create_time() const {
-        return _produce_bad_create_time;
-    }
-
     void
     setup_metrics(metrics::internal_metric_groups& mgs, std::string_view proto);
 
@@ -98,7 +88,6 @@ class server_probe {
     uint32_t _method_not_found_errors = 0;
     uint32_t _requests_blocked_memory = 0;
     uint32_t _connections_wait_rate = 0;
-    uint32_t _produce_bad_create_time = 0;
     friend std::ostream& operator<<(std::ostream& o, const server_probe& p);
 };
 


Reviews:
user: login: StephanDollberg
state: DISMISSED
author_association: MEMBER
submitted_at: 2025-03-12T21:57:12Z
user: login: oleiman
body: lgtm
state: DISMISSED
author_association: MEMBER
submitted_at: 2025-03-12T22:13:26Z
user: login: travisdowns
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-13T01:53:22Z
user: login: oleiman
state: APPROVED
author_association: MEMBER
submitted_at: 2025-03-13T01:59:20Z

Review comments:
diff_hunk: @@ -34,6 +34,11 @@
 #include <boost/container_hash/extensions.hpp>
 #include <fmt/ostream.h>
 
+#include <chrono>
+#include <cstdint>
+#include <memory>
+#include <ranges>
+
path: src/v/kafka/server/handlers/produce.cc
user: login: oleiman
body: nitpick: commit says we're removing unused header includes, but the diff appears to be adding four new ones?
created_at: 2025-03-12T22:02:59Z
author_association: MEMBER
diff_hunk: @@ -34,6 +34,11 @@
 #include <boost/container_hash/extensions.hpp>
 #include <fmt/ostream.h>
 
+#include <chrono>
+#include <cstdint>
+#include <memory>
+#include <ranges>
+
path: src/v/kafka/server/handlers/produce.cc
user: login: travisdowns
body: Not really a nitpick, a good observation! 

This is an older change and Michal submitted a change which removed the same headers so this change is pointless after I rebased it (don't ask my why it adds those headers). I will delete it.
created_at: 2025-03-13T01:53:22Z
author_association: MEMBER

PR comments:
user: login: travisdowns
created_at: 2025-03-12T20:08:17Z
body: This shares its first commit with https://github.com/redpanda-data/redpanda/pull/25340, I'll rebase whichever commit goes in second.
author_association: MEMBER
user: login: travisdowns
created_at: 2025-03-12T20:15:26Z
body: cc @StephanDollberg I'm removing a metric so we can fell less guilty about adding others
author_association: MEMBER
user: login: vbotbuildovich
created_at: 2025-03-13T00:32:50Z
body: <!--TEST RESULTS-->
<!--9d78c5f5-616d-45c6-a5ec-099cab7910a1-->
<!--TEST RESULTS-->
<!--e0183a8b-34a9-42a0-8cad-61be62eca869-->
<!--title-->
## CI test results
<!--/title-->
<!--body-->
<details>
<summary>test results on <a href="https://buildkite.com/redpanda/redpanda/builds/63002">build#63002</a></summary>


| test_id | test_kind | job_url | test_status | passed |
|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:------------|:------------------------------------------------------------------------------------------|:--------------|:---------|
| rptest.tests.compaction_recovery_test.CompactionRecoveryUpgradeTest.test_index_recovery_after_upgrade | ducktape | https://buildkite.com/redpanda/redpanda/builds/63002#01958c5d-c217-4587-a16d-6057574c7e4c | FLAKY | 1/2 |
| rptest.tests.datalake.rest_catalog_connection_test.RestCatalogConnectionTest.test_redpanda_connection_to_rest_catalog.cloud_storage_type=CloudStorageType.S3 | ducktape | https://buildkite.com/redpanda/redpanda/builds/63002#01958c5d-c21a-4562-9c3b-193b2fc11b47 | FLAKY | 1/2 |
| rptest.tests.random_node_operations_test.RandomNodeOperationsTest.test_node_operations.enable_failures=False.mixed_versions=False.with_tiered_storage=True.with_iceberg=True.with_chunked_compaction=False.cloud_storage_type=CloudStorageType.S3 | ducktape | https://buildkite.com/redpanda/redpanda/builds/63002#01958c5d-c217-4587-a16d-6057574c7e4c | FLAKY | 1/2 |
| rptest.tests.scaling_up_test.ScalingUpTest.test_scaling_up_with_recovered_topic | ducktape | https://buildkite.com/redpanda/redpanda/builds/63002#01958c5d-c21a-4562-9c3b-193b2fc11b47 | FLAKY | 1/4 |
</details>
<details>
<summary>test results on <a href="https://buildkite.com/redpanda/redpanda/builds/63019">build#63019</a></summary>


| test_id                                                                                                                                                      | test_kind   | job_url                                                                                   | test_status   | passed   |
|:-------------------------------------------------------------------------------------------------------------------------------------------------------------|:------------|:------------------------------------------------------------------------------------------|:--------------|:---------|
| rptest.tests.datalake.rest_catalog_connection_test.RestCatalogConnectionTest.test_redpanda_connection_to_rest_catalog.cloud_storage_type=CloudStorageType.S3 | ducktape    | https://buildkite.com/redpanda/redpanda/builds/63019#01958d82-7f56-450b-b50a-ea9c3d2cd45c | FLAKY         | 1/2      |
</details>
<!--/body-->
author_association: COLLABORATOR