Repository: redpanda-data/redpanda
Pull request #25974: `storage`: set `base_offset` for `index_state` during compaction
Author: login: WillemKauf
Labels: name: area/build
name: area/redpanda
Created: 2025-04-29T19:44:45Z | Merged: 2025-05-01T19:39:32Z

This value would be incorrectly persisted as `0` during and after compaction.

This means batches indexed in the `index_state` during compaction would have an incorrect `relative_offset` added, and furthermore, the value of `base_offset` would forever be `0` on disk.

Fix this by adding `base_offset` as a required parameter in `index_state::make_empty_index()` and setting the `index_base_offset` when constructing the `copy_data_segment_reducer` for use with self compaction and sliding window compaction.

## Backports Required

- [ ] none - not a bug fix
- [ ] none - this is a backport
- [ ] none - issue does not exist in previous branches
- [ ] none - papercut/not impactful enough to backport
- [X] v25.1.x
- [X] v24.3.x
- [X] v24.2.x
- [X] v24.1.x

## Release Notes

### Bug Fixes

* Fixes a bug that lead to the `base_offset` in `segment_index`es being persisted as 0 during and after compaction.


Diff:
diff --git a/src/v/storage/compaction_reducers.cc b/src/v/storage/compaction_reducers.cc
index c047800c05c2c..7b6141db853bb 100644
--- a/src/v/storage/compaction_reducers.cc
+++ b/src/v/storage/compaction_reducers.cc
@@ -318,7 +318,7 @@ ss::future<ss::stop_iteration> copy_data_segment_reducer::filter_and_append(
     // caller who has more context
     if (_idx.maybe_index(
           _acc,
-          32_KiB,
+          segment_index::default_data_buffer_step,
           start_pos,
           batch.base_offset(),
           batch.last_offset(),
diff --git a/src/v/storage/compaction_reducers.h b/src/v/storage/compaction_reducers.h
index 8c91792ba138b..7c32820af5ad0 100644
--- a/src/v/storage/compaction_reducers.h
+++ b/src/v/storage/compaction_reducers.h
@@ -162,6 +162,7 @@ class copy_data_segment_reducer : public compaction_reducer {
       segment_appender* a,
       bool internal_topic,
       offset_delta_time apply_offset,
+      model::offset index_base_offset,
       model::offset segment_last_offset,
       bool compaction_placeholder_enabled,
       compacted_index_writer* cidx = nullptr,
@@ -172,7 +173,7 @@ class copy_data_segment_reducer : public compaction_reducer {
       , _compaction_placeholder_enabled(compaction_placeholder_enabled)
       , _appender(a)
       , _compacted_idx(cidx)
-      , _idx(index_state::make_empty_index(apply_offset))
+      , _idx(index_state::make_empty_index(index_base_offset, apply_offset))
       , _internal_topic(internal_topic)
       , _inject_failure(inject_failure)
       , _as(as) {}
diff --git a/src/v/storage/index_state.cc b/src/v/storage/index_state.cc
index ffd0c127e2762..9f93d53bfb66a 100644
--- a/src/v/storage/index_state.cc
+++ b/src/v/storage/index_state.cc
@@ -208,8 +208,10 @@ offset_time_index::offset_time_index(
 
 uint32_t offset_time_index::raw_value() const { return _val; }
 
-index_state index_state::make_empty_index(offset_delta_time with_offset) {
+index_state index_state::make_empty_index(
+  model::offset base_offset, offset_delta_time with_offset) {
     index_state idx{};
+    idx.base_offset = base_offset;
     idx.with_offset = with_offset;
 
     return idx;
diff --git a/src/v/storage/index_state.h b/src/v/storage/index_state.h
index 5c4fb56940b4c..0cea765d2d65d 100644
--- a/src/v/storage/index_state.h
+++ b/src/v/storage/index_state.h
@@ -138,7 +138,8 @@ struct index_state
     static constexpr auto clean_compact_timestamp_version = 8;
     static constexpr auto may_have_tombstone_records_version = 9;
 
-    static index_state make_empty_index(offset_delta_time with_offset);
+    static index_state
+    make_empty_index(model::offset base_offset, offset_delta_time with_offset);
 
     struct entry {
         model::offset offset;
diff --git a/src/v/storage/segment_deduplication_utils.cc b/src/v/storage/segment_deduplication_utils.cc
index 4d879212af333..d68c872925b48 100644
--- a/src/v/storage/segment_deduplication_utils.cc
+++ b/src/v/storage/segment_deduplication_utils.cc
@@ -224,6 +224,7 @@ ss::future<index_state> deduplicate_segment(
       &appender,
       seg->path().is_internal_topic(),
       should_offset_delta_times,
+      seg->index().base_offset(),
       segment_last_offset,
       compaction_placeholder_enabled,
       &cmp_idx_writer,
diff --git a/src/v/storage/segment_index.cc b/src/v/storage/segment_index.cc
index 5c3a1f61bf76d..78e176dc2eb18 100644
--- a/src/v/storage/segment_index.cc
+++ b/src/v/storage/segment_index.cc
@@ -42,9 +42,8 @@ segment_index::segment_index(
   , _step(step)
   , _feature_table(std::ref(feature_table))
   , _state(index_state::make_empty_index(
-      storage::internal::should_apply_delta_time_offset(_feature_table)))
+      base, storage::internal::should_apply_delta_time_offset(_feature_table)))
   , _sanitizer_config(std::move(sanitizer_config)) {
-    _state.base_offset = base;
     _state.broker_timestamp = broker_timestamp;
     _state.clean_compact_timestamp = clean_compact_timestamp;
     _state.may_have_tombstone_records = may_have_tombstone_records;
@@ -60,10 +59,8 @@ segment_index::segment_index(
   , _step(step)
   , _feature_table(std::ref(feature_table))
   , _state(index_state::make_empty_index(
-      storage::internal::should_apply_delta_time_offset(_feature_table)))
-  , _mock_file(mock_file) {
-    _state.base_offset = base;
-}
+      base, storage::internal::should_apply_delta_time_offset(_feature_table)))
+  , _mock_file(mock_file) {}
 
 ss::future<ss::file> segment_index::open() {
     if (_mock_file) {
@@ -86,9 +83,8 @@ void segment_index::reset() {
     auto may_have_tombstone_records = _state.may_have_tombstone_records;
 
     _state = index_state::make_empty_index(
-      storage::internal::should_apply_delta_time_offset(_feature_table));
+      base, storage::internal::should_apply_delta_time_offset(_feature_table));
 
-    _state.base_offset = base;
     _state.clean_compact_timestamp = clean_compact_timestamp;
     _state.may_have_tombstone_records = may_have_tombstone_records;
 
diff --git a/src/v/storage/segment_index.h b/src/v/storage/segment_index.h
index ee855bd10451b..d80cfdbc70271 100644
--- a/src/v/storage/segment_index.h
+++ b/src/v/storage/segment_index.h
@@ -253,6 +253,8 @@ class segment_index {
     ss::future<size_t> disk_usage();
     void clear_cached_disk_usage() { _disk_usage_size.reset(); }
 
+    const index_state& get_index_state() const { return _state; }
+
 private:
     ss::future<bool> materialize_index_from_file(ss::file);
     ss::future<> flush_to_file(ss::file);
diff --git a/src/v/storage/segment_utils.cc b/src/v/storage/segment_utils.cc
index 14ba30bcdb1ca..cef26e0985f50 100644
--- a/src/v/storage/segment_utils.cc
+++ b/src/v/storage/segment_utils.cc
@@ -349,8 +349,9 @@ ss::future<storage::index_state> do_copy_segment_data(
   storage_resources& resources,
   offset_delta_time apply_offset,
   ss::sharded<features::feature_table>& feature_table) {
-    // preserve broker_timestamp and clean_compact_timestamp from the segment's
-    // index
+    // preserve base_offset, broker_timestamp, and clean_compact_timestamp from
+    // the segment's index
+    auto old_base_offset = seg->index().base_offset();
     auto old_broker_timestamp = seg->index().broker_timestamp();
     auto old_clean_compact_timestamp = seg->index().clean_compact_timestamp();
 
@@ -428,6 +429,7 @@ ss::future<storage::index_state> do_copy_segment_data(
       appender.get(),
       seg->path().is_internal_topic(),
       apply_offset,
+      old_base_offset,
       segment_last_offset,
       compaction_placeholder_enabled,
       /*cidx=*/nullptr,
diff --git a/src/v/storage/tests/BUILD b/src/v/storage/tests/BUILD
index 75896470e2a86..b4a42fbe222d8 100644
--- a/src/v/storage/tests/BUILD
+++ b/src/v/storage/tests/BUILD
@@ -838,6 +838,7 @@ redpanda_cc_gtest(
         "//src/v/base",
         "//src/v/cluster",
         "//src/v/cluster:features",
+        "//src/v/container:fragmented_vector",
         "//src/v/features",
         "//src/v/kafka/server/tests:kafka_test_utils",
         "//src/v/model",
diff --git a/src/v/storage/tests/compaction_e2e_test.cc b/src/v/storage/tests/compaction_e2e_test.cc
index d8b0cd1fe3253..6d882ab46764a 100644
--- a/src/v/storage/tests/compaction_e2e_test.cc
+++ b/src/v/storage/tests/compaction_e2e_test.cc
@@ -10,6 +10,7 @@
 #include "base/vlog.h"
 #include "cluster/feature_manager.h"
 #include "cluster/feature_update_action.h"
+#include "container/fragmented_vector.h"
 #include "features/feature_state.h"
 #include "features/feature_table.h"
 #include "gtest/gtest.h"
@@ -348,6 +349,10 @@ TEST_P(CompactionFixtureParamTest, TestDedupeOnePass) {
                                       model::offset(0))
                                     .get();
     ASSERT_EQ(consumed_kvs, consumed_kvs_restarted);
+
+    for (const auto& seg : log->segments()) {
+        ASSERT_EQ(seg->offsets().get_base_offset(), seg->index().base_offset());
+    }
 }
 
 INSTANTIATE_TEST_SUITE_P(
@@ -389,6 +394,10 @@ TEST_F(CompactionFixtureTest, TestDedupeMultiPass) {
     ASSERT_EQ(segments_compacted_2, segments_compacted_3);
 
     ASSERT_NO_FATAL_FAILURE(check_records(cardinality, num_segments - 1).get());
+
+    for (const auto& seg : disk_log.segments()) {
+        ASSERT_EQ(seg->offsets().get_base_offset(), seg->index().base_offset());
+    }
 }
 
 TEST_F(CompactionFixtureTest, TestChunkedCompaction) {
@@ -478,6 +487,10 @@ TEST_F(CompactionFixtureTest, TestChunkedCompaction) {
     num_chunked_compaction_runs
       = disk_log.get_probe().get_chunked_compaction_runs();
     ASSERT_EQ(num_chunked_compaction_runs, 1);
+
+    for (const auto& seg : disk_log.segments()) {
+        ASSERT_EQ(seg->offsets().get_base_offset(), seg->index().base_offset());
+    }
 }
 
 TEST_F(CompactionFixtureTest, TestDedupeMultiPassAddedSegment) {
@@ -564,6 +577,10 @@ TEST_F(CompactionFixtureTest, TestDedupeMultiPassAddedSegment) {
       disk_log.get_last_compaction_window_start_offset().has_value());
 
     ASSERT_NO_FATAL_FAILURE(check_records(cardinality, num_segments - 1).get());
+
+    for (const auto& seg : disk_log.segments()) {
+        ASSERT_EQ(seg->offsets().get_base_offset(), seg->index().base_offset());
+    }
 }
 
 class CompactionFixtureBatchSizeParamTest
@@ -1451,9 +1468,79 @@ TEST_P(
         auto num_expected_data_batches = placeholder_batch_enabled ? 0 : 1;
         check_num_data_batches(seg_0_batches, num_expected_data_batches);
     }
+
+    ASSERT_EQ(
+      segs[0]->offsets().get_base_offset(), segs[0]->index().base_offset());
 }
 
 INSTANTIATE_TEST_SUITE_P(
   PlaceholderBatchEnabled,
   CompactionFixturePlaceHolderBatchTest,
   ::testing::Bool());
+
+TEST_F(CompactionFixtureTest, TestSegmentIndexReconstructed) {
+    constexpr auto num_segments = 5;
+    constexpr auto cardinality
+      = 1000000; // Large enough to ensure no duplicates- segment index relative
+                 // offsets _should_ be the same before and after compaction.
+    size_t batches_per_segment = 100;
+    size_t records_per_batch = 10;
+    generate_data(
+      num_segments, cardinality, batches_per_segment, records_per_batch)
+      .get();
+
+    auto& disk_log = dynamic_cast<storage::disk_log_impl&>(*log);
+    auto& segs = disk_log.segments();
+
+    for (const auto& seg : segs) {
+        if (!seg->has_appender()) {
+            auto pre_compact_relative_offset_index
+              = seg->index()
+                  .get_index_state()
+                  .index.copy_relative_offset_index();
+            // Self compact the segment
+            do_segment_self_compact(seg, model::offset::max()).get();
+            auto post_compact_relative_offset_index
+              = seg->index()
+                  .get_index_state()
+                  .index.copy_relative_offset_index();
+
+            ASSERT_EQ(
+              pre_compact_relative_offset_index,
+              post_compact_relative_offset_index);
+            ASSERT_EQ(
+              seg->offsets().get_base_offset(), seg->index().base_offset());
+        }
+    }
+
+    std::vector<chunked_vector<uint32_t>>
+      pre_sliding_window_index_relative_offsets;
+    for (const auto& seg : segs) {
+        if (!seg->has_appender()) {
+            pre_sliding_window_index_relative_offsets.push_back(
+              seg->index()
+                .get_index_state()
+                .index.copy_relative_offset_index());
+        }
+    }
+
+    bool did_compact = do_sliding_window_compact(model::offset::max()).get();
+    ASSERT_TRUE(did_compact);
+
+    std::vector<chunked_vector<uint32_t>>
+      post_sliding_window_index_relative_offsets;
+    for (const auto& seg : segs) {
+        if (!seg->has_appender()) {
+            post_sliding_window_index_relative_offsets.push_back(
+              seg->index()
+                .get_index_state()
+                .index.copy_relative_offset_index());
+            ASSERT_EQ(
+              seg->offsets().get_base_offset(), seg->index().base_offset());
+        }
+    }
+
+    ASSERT_EQ(
+      pre_sliding_window_index_relative_offsets,
+      post_sliding_window_index_relative_offsets);
+}
diff --git a/src/v/storage/tests/index_state_test.cc b/src/v/storage/tests/index_state_test.cc
index f4a3fe91b6404..0f0875dc6c64b 100644
--- a/src/v/storage/tests/index_state_test.cc
+++ b/src/v/storage/tests/index_state_test.cc
@@ -19,9 +19,9 @@
 
 static storage::index_state make_random_index_state(
   storage::offset_delta_time apply_offset = storage::offset_delta_time::yes) {
-    auto st = storage::index_state::make_empty_index(apply_offset);
+    auto base_offset = model::offset(random_generators::get_int<int64_t>());
+    auto st = storage::index_state::make_empty_index(base_offset, apply_offset);
     st.bitflags = random_generators::get_int<uint32_t>();
-    st.base_offset = model::offset(random_generators::get_int<int64_t>());
     st.max_offset = model::offset(random_generators::get_int<int64_t>());
     st.base_timestamp = model::timestamp(random_generators::get_int<int64_t>());
     st.max_timestamp = model::timestamp(random_generators::get_int<int64_t>());
@@ -265,10 +265,10 @@ BOOST_AUTO_TEST_CASE(binary_compatibility_test) {
     // format. In order to do this the index_state has to be refactored and the
     // columnar part has to be extracted. But the serialized form shouldn't
     // change.
+    auto base_offset = model::offset(6321451485771820344);
     auto expected_state = storage::index_state::make_empty_index(
-      storage::offset_delta_time::yes);
+      base_offset, storage::offset_delta_time::yes);
     // NOLINTBEGIN(cppcoreguidelines-avoid-magic-numbers)
-    expected_state.base_offset = model::offset(6321451485771820344);
     expected_state.base_timestamp = model::timestamp(5331697842032508203);
     expected_state.max_offset = model::offset(6925876299231340900);
     expected_state.max_timestamp = model::timestamp(659192121601013627);


Reviews:
user: login: andrwng
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-29T19:52:37Z
user: login: WillemKauf
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-29T20:13:07Z
user: login: andrwng
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-30T16:49:12Z
user: login: WillemKauf
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-30T19:46:25Z
user: login: WillemKauf
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-30T19:49:16Z
user: login: andrwng
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-01T06:48:08Z
user: login: WillemKauf
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-01T15:21:29Z
user: login: WillemKauf
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-01T15:23:15Z
user: login: andrwng
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-05-01T19:38:29Z

Review comments:
diff_hunk: @@ -247,8 +247,9 @@ ss::future<index_state> deduplicate_segment(
           stats);
     }
 
-    // restore broker timestamp and clean compact timestamp
+    // restore base offset, broker timestamp and clean compact timestamp
path: src/v/storage/segment_deduplication_utils.cc
user: login: andrwng
body: >but could heavily reduce the efficacy of seeks using the `segment_index`.

Could you elaborate on this a bit, and maybe give an example? Do you mean there is a case where the index will return an entry that is too low? A dedicated unit test would be helpful to illustrate, if it's not too much trouble
created_at: 2025-04-29T19:52:36Z
author_association: CONTRIBUTOR
diff_hunk: @@ -247,8 +247,9 @@ ss::future<index_state> deduplicate_segment(
           stats);
     }
 
-    // restore broker timestamp and clean compact timestamp
+    // restore base offset, broker timestamp and clean compact timestamp
path: src/v/storage/segment_deduplication_utils.cc
user: login: WillemKauf
body: > Could you elaborate on this a bit, and maybe give an example? 

Fair request, I'll have to come back to this. The only inspection I've really done is eye-balling the code to reason that having a `base_offset` of `0` for all segments is probably really bad for [`find_nearest()`](https://github.com/redpanda-data/redpanda/blob/44d7b62a54c1132dd16d135a40d085d5e75f3062/src/v/storage/index_state.cc#L673-L689) performance.

I'm shocked that in general, this base offset being 0 doesn't affect correctness elsewhere in the code.

> A dedicated unit test would be helpful to illustrate, if it's not too much trouble

Also a reasonable request that I will revisit soon.
created_at: 2025-04-29T20:13:07Z
author_association: CONTRIBUTOR
diff_hunk: @@ -247,8 +247,9 @@ ss::future<index_state> deduplicate_segment(
           stats);
     }
 
-    // restore broker timestamp and clean compact timestamp
+    // restore base offset, broker timestamp and clean compact timestamp
     auto& new_idx = res.new_idx;
+    new_idx.base_offset = seg->index().base_offset();
path: src/v/storage/segment_deduplication_utils.cc
user: login: andrwng
body: Don't we need to set this before running the reducer? Won't this result in incorrect offset deltas in the index?
created_at: 2025-04-30T16:49:11Z
author_association: CONTRIBUTOR
diff_hunk: @@ -247,8 +247,9 @@ ss::future<index_state> deduplicate_segment(
           stats);
     }
 
-    // restore broker timestamp and clean compact timestamp
+    // restore base offset, broker timestamp and clean compact timestamp
     auto& new_idx = res.new_idx;
+    new_idx.base_offset = seg->index().base_offset();
path: src/v/storage/segment_deduplication_utils.cc
user: login: WillemKauf
body: Oops, of course. Fixed.
created_at: 2025-04-30T19:46:25Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1476,3 +1477,70 @@ INSTANTIATE_TEST_SUITE_P(
   PlaceholderBatchEnabled,
   CompactionFixturePlaceHolderBatchTest,
   ::testing::Bool());
+
+TEST_F(CompactionFixtureTest, TestSegmentIndexReconstructed) {
path: src/v/storage/tests/compaction_e2e_test.cc
user: login: WillemKauf
body: This probably isn't a very good test for comparing the `relative_offset_index` vectors due to the step size of `32_KiB`.

We don't really have a good way to override this in tests...
created_at: 2025-04-30T19:49:16Z
author_association: CONTRIBUTOR
diff_hunk: @@ -247,8 +247,9 @@ ss::future<index_state> deduplicate_segment(
           stats);
     }
 
-    // restore broker timestamp and clean compact timestamp
+    // restore base offset, broker timestamp and clean compact timestamp
path: src/v/storage/segment_deduplication_utils.cc
user: login: andrwng
body: Just making sure I understand: the description in this PR makes it seem like the problem is that we're seeking somewhere we shouldn't be. But ignoring overflow, that doesn't seem to be the case, in which case seek efficacy seems unaffected -- our deltas are offset by the base_offset, but that seems fine because the base offset is also offset by the same amount. Is that your understanding as well?
created_at: 2025-05-01T06:48:08Z
author_association: CONTRIBUTOR
diff_hunk: @@ -247,8 +247,9 @@ ss::future<index_state> deduplicate_segment(
           stats);
     }
 
-    // restore broker timestamp and clean compact timestamp
+    // restore base offset, broker timestamp and clean compact timestamp
path: src/v/storage/segment_deduplication_utils.cc
user: login: WillemKauf
body: You're correct, I hadn't clocked this originally but if the base offset is set to `0` during compaction then future seeks should also be unaffected, since the base offset is consistently wrong.

I will rewrite the PR cover letter, I don't think any commits needs changing though.
created_at: 2025-05-01T15:21:29Z
author_association: CONTRIBUTOR
diff_hunk: @@ -247,8 +247,9 @@ ss::future<index_state> deduplicate_segment(
           stats);
     }
 
-    // restore broker timestamp and clean compact timestamp
+    // restore base offset, broker timestamp and clean compact timestamp
path: src/v/storage/segment_deduplication_utils.cc
user: login: WillemKauf
body: And yes, to your point, the overflow is the only observable issue we saw fall out of the `base_offset` being 0, luckily.
created_at: 2025-05-01T15:23:14Z
author_association: CONTRIBUTOR

PR comments:
user: login: secpanda
created_at: 2025-04-29T19:45:04Z
body: ![](https://res.cloudinary.com/snyk/image/upload/r-d/scm-platform/snyk-pull-requests/pr-banner-default.svg)

### :tada: **Snyk checks have passed. No issues have been found so far.**

:white_check_mark: **security/snyk** check is complete. No issues have been found. [(View Details)](https://app.snyk.io/org/devex-c3w/pr-checks/8b042b28-4049-4473-b215-c3f56aaeb291?refs=530b6c5bef98cfd8dc276a7885cbebd2a7394601&source=prChecksComment)

:white_check_mark: **license/snyk** check is complete. No issues have been found. [(View Details)](https://app.snyk.io/org/devex-c3w/pr-checks/8b042b28-4049-4473-b215-c3f56aaeb291/license?refs=530b6c5bef98cfd8dc276a7885cbebd2a7394601&source=prChecksComment)



author_association: NONE
user: login: secpanda
created_at: 2025-04-29T19:45:05Z
body: ![](https://res.cloudinary.com/snyk/image/upload/r-d/scm-platform/snyk-pull-requests/pr-banner-default.svg)

### :tada: **Snyk checks have passed. No issues have been found so far.**

:white_check_mark: **security/snyk** check is complete. No issues have been found. [(View Details)](https://app.snyk.io/org/devprod/pr-checks/b2e01ede-d6c3-4245-86a2-a423c5de8ec6?refs=530b6c5bef98cfd8dc276a7885cbebd2a7394601&source=prChecksComment)

:white_check_mark: **license/snyk** check is complete. No issues have been found. [(View Details)](https://app.snyk.io/org/devprod/pr-checks/b2e01ede-d6c3-4245-86a2-a423c5de8ec6/license?refs=530b6c5bef98cfd8dc276a7885cbebd2a7394601&source=prChecksComment)



author_association: NONE
user: login: vbotbuildovich
created_at: 2025-04-29T23:21:25Z
body: <!--FAILURES SLASH COMMAND https://buildkite.com/redpanda/redpanda/builds/65258-->
<!--807e6736-eb71-4a74-9b5b-27476bf82deb-->
<!--994c07df-0a94-42af-bf9c-b8183d55d05f-->
<!--75c73e56-8ae2-48a9-97db-e76050f587fc-->
<!--title-->
## Retry command for [Build#65258](https://buildkite.com/redpanda/redpanda/builds/65258)
please wait until all jobs are finished before running the slash command
<!--/title-->
<!--body-->
```


/ci-repeat 1
tests/rptest/tests/data_migrations_api_test.py::DataMigrationsApiTest.test_conflicting_names
tests/rptest/tests/scaling_up_test.py::ScalingUpTest.test_scaling_up_with_recovered_topic
tests/rptest/tests/simple_e2e_test.py::SimpleEndToEndTest.test_consumer_interruption
```
<!--/body-->
author_association: COLLABORATOR
user: login: vbotbuildovich
created_at: 2025-04-30T00:18:28Z
body: <!--TEST RESULTS-->
<!--543c81ca-c0ae-42b8-ab94-4f35f03f3d42-->
<!--TEST RESULTS-->
<!--9cc40a4a-6890-4127-a318-256b8f70fb4f-->
<!--TEST RESULTS-->
<!--afb62a20-8c8b-49a4-a726-47cadd3a84f6-->
<!--TEST RESULTS-->
<!--a6ce5972-24fc-4a6b-b428-6269d5fca96d-->
<!--title-->
## CI test results
<!--/title-->
<!--body-->
<details>
<summary>test results on <a href="https://buildkite.com/redpanda/redpanda/builds/65258">build#65258</a></summary>


| test_id | test_kind | job_url | test_status | passed |
|:-----------------------------------------------------------------------------------------------------|:------------|:------------------------------------------------------------------------------------------|:--------------|:---------|
| rptest.tests.data_migrations_api_test.DataMigrationsApiTest.test_conflicting_names | ducktape | https://buildkite.com/redpanda/redpanda/builds/65258#0196836d-3a63-421f-92c8-b8ac0bb44fc4 | FLAKY | 6/21 |
| rptest.tests.scaling_up_test.ScalingUpTest.test_scaling_up_with_recovered_topic | ducktape | https://buildkite.com/redpanda/redpanda/builds/65258#01968361-4318-47a8-ad01-af36d9f99437 | FLAKY | 14/21 |
| rptest.tests.simple_e2e_test.SimpleEndToEndTest.test_consumer_interruption | ducktape | https://buildkite.com/redpanda/redpanda/builds/65258#01968361-431b-40c7-8f10-ea1bea8d0e7f | FLAKY | 9/21 |
| rptest.tests.upgrade_test.UpgradeBackToBackTest.test_upgrade_with_all_workloads.single_upgrade=False | ducktape | https://buildkite.com/redpanda/redpanda/builds/65258#01968361-431b-40c7-8f10-ea1bea8d0e7f | FLAKY | 15/21 |
</details>
<details>
<summary>test results on <a href="https://buildkite.com/redpanda/redpanda/builds/65295">build#65295</a></summary>


| test_id | test_kind | job_url | test_status | passed |
|:-----------------------------------------------------------------------------------|:------------|:------------------------------------------------------------------------------------------|:--------------|:---------|
| rptest.tests.data_migrations_api_test.DataMigrationsApiTest.test_conflicting_names | ducktape | https://buildkite.com/redpanda/redpanda/builds/65295#019686ee-c114-46a6-9c4e-2f55b7e0d6e2 | FLAKY | 5/21 |
| rptest.tests.simple_e2e_test.SimpleEndToEndTest.test_consumer_interruption | ducktape | https://buildkite.com/redpanda/redpanda/builds/65295#019686ee-c11c-46b5-a928-9f467052f2c7 | FLAKY | 7/21 |
</details>
<details>
<summary>test results on <a href="https://buildkite.com/redpanda/redpanda/builds/65302">build#65302</a></summary>


| test_id | test_kind | job_url | test_status | passed |
|:-----------------------------------------------------------------------------------|:------------|:------------------------------------------------------------------------------------------|:--------------|:---------|
| rptest.tests.data_migrations_api_test.DataMigrationsApiTest.test_conflicting_names | ducktape | https://buildkite.com/redpanda/redpanda/builds/65302#01968717-24a2-4767-8a9f-30cada82b163 | FLAKY | 6/21 |
</details>
<details>
<summary>test results on <a href="https://buildkite.com/redpanda/redpanda/builds/65347">build#65347</a></summary>


| test_id                                                                                                                                                                            | test_kind   | job_url                                                                                   | test_status   | passed   |
|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:------------|:------------------------------------------------------------------------------------------|:--------------|:---------|
| rptest.tests.data_migrations_api_test.DataMigrationsApiTest.test_migrated_topic_data_integrity.transfer_leadership=True.params=.cancellation.dir.in.stage.preparing.use_alias.True | ducktape    | https://buildkite.com/redpanda/redpanda/builds/65347#0196890c-3dbd-412c-b169-7c7221289cd6 | FLAKY         | 20/21    |
| rptest.tests.maintenance_test.MaintenanceCycleTest.test_leader_distribution.use_rpk=False                                                                                          | ducktape    | https://buildkite.com/redpanda/redpanda/builds/65347#0196890c-3dbe-48b8-b9da-04b0ee6224c4 | FLAKY         | 20/21    |
| rptest.tests.partition_balancer_test.PartitionBalancerTest.test_unavailable_nodes                                                                                                  | ducktape    | https://buildkite.com/redpanda/redpanda/builds/65347#01968911-800c-4377-8d72-4d2bdf01544d | FLAKY         | 14/21    |
| rptest.tests.partition_reassignments_test.PartitionReassignmentsTest.test_add_partitions_with_inprogress_reassignments                                                             | ducktape    | https://buildkite.com/redpanda/redpanda/builds/65347#0196890c-3dbd-412c-b169-7c7221289cd6 | FLAKY         | 20/21    |
| rptest.tests.tiered_storage_pause_test.TestTieredStoragePause.test_safe_pause_resume.allow_gaps_topic_level=None.allow_gaps_cluster_level=True                                     | ducktape    | https://buildkite.com/redpanda/redpanda/builds/65347#01968911-800d-4aca-9838-4285d5668ce4 | FLAKY         | 20/21    |
| rptest.transactions.consumer_offsets_test.VerifyConsumerOffsets.test_consumer_group_offsets                                                                                        | ducktape    | https://buildkite.com/redpanda/redpanda/builds/65347#0196890c-3dbe-48b8-b9da-04b0ee6224c4 | FLAKY         | 18/21    |
</details>
<!--/body-->
author_association: COLLABORATOR
user: login: WillemKauf
created_at: 2025-04-30T13:40:20Z
body: /ci-repeat 1
skip-redpanda-build
skip-units
tests/rptest/tests/data_migrations_api_test.py::DataMigrationsApiTest.test_conflicting_names
tests/rptest/tests/scaling_up_test.py::ScalingUpTest.test_scaling_up_with_recovered_topic
tests/rptest/tests/simple_e2e_test.py::SimpleEndToEndTest.test_consumer_interruption
author_association: CONTRIBUTOR
user: login: vbotbuildovich
created_at: 2025-04-30T13:58:51Z
body: <!--FAILURES SLASH COMMAND https://buildkite.com/redpanda/redpanda/builds/65295-->
<!--0071c2d0-e29b-4d86-a325-a1f3c3071b80-->
<!--89d74cee-684c-4b08-92f9-a120ef5867ed-->
<!--title-->
## Retry command for [Build#65295](https://buildkite.com/redpanda/redpanda/builds/65295)
please wait until all jobs are finished before running the slash command
<!--/title-->
<!--body-->
```

/ci-repeat 1
tests/rptest/tests/simple_e2e_test.py::SimpleEndToEndTest.test_consumer_interruption
tests/rptest/tests/data_migrations_api_test.py::DataMigrationsApiTest.test_conflicting_names
```
<!--/body-->
author_association: COLLABORATOR
user: login: WillemKauf
created_at: 2025-04-30T14:24:26Z
body: /ci-repeat 1
skip-redpanda-build
skip-units
tests/rptest/tests/simple_e2e_test.py::SimpleEndToEndTest.test_consumer_interruption
tests/rptest/tests/data_migrations_api_test.py::DataMigrationsApiTest.test_conflicting_names
author_association: CONTRIBUTOR
user: login: vbotbuildovich
created_at: 2025-04-30T14:51:26Z
body: <!--FAILURES SLASH COMMAND https://buildkite.com/redpanda/redpanda/builds/65302-->
<!--7ed49b61-398e-4284-b640-6bcb1371c93f-->
<!--title-->
## Retry command for [Build#65302](https://buildkite.com/redpanda/redpanda/builds/65302)
please wait until all jobs are finished before running the slash command
<!--/title-->
<!--body-->
```
/ci-repeat 1
tests/rptest/tests/data_migrations_api_test.py::DataMigrationsApiTest.test_conflicting_names
```
<!--/body-->
author_association: COLLABORATOR
user: login: WillemKauf
created_at: 2025-04-30T15:57:04Z
body: /ci-repeat 1
skip-redpanda-build
skip-units
tests/rptest/tests/data_migrations_api_test.py::DataMigrationsApiTest.test_conflicting_names
author_association: CONTRIBUTOR
user: login: secpanda
created_at: 2025-04-30T22:20:03Z
body: ![](https://res.cloudinary.com/snyk/image/upload/r-d/scm-platform/snyk-pull-requests/pr-banner-default.svg)

### :tada: **Snyk checks have passed. No issues have been found so far.**

:white_check_mark: **security/snyk** check is complete. No issues have been found. [(View Details)](https://app.snyk.io/org/cloud-1t5/pr-checks/856b0611-c3ac-49b5-a134-c29b3bf280df?refs=530b6c5bef98cfd8dc276a7885cbebd2a7394601&source=prChecksComment)

:white_check_mark: **license/snyk** check is complete. No issues have been found. [(View Details)](https://app.snyk.io/org/cloud-1t5/pr-checks/856b0611-c3ac-49b5-a134-c29b3bf280df/license?refs=530b6c5bef98cfd8dc276a7885cbebd2a7394601&source=prChecksComment)



author_association: NONE
user: login: vbotbuildovich
created_at: 2025-05-01T01:40:08Z
body: <!--FAILURES SLASH COMMAND https://buildkite.com/redpanda/redpanda/builds/65347-->
<!--73da4cc6-c509-4ad6-b385-a6faf8591be6-->
<!--d2c498b8-4e79-4868-a49f-164e36b0b7db-->
<!--005779a8-0642-4b55-9142-f70e0a27f572-->
<!--ee004792-1d0a-40f0-9c55-17af24b525f7-->
<!--title-->
## Retry command for [Build#65347](https://buildkite.com/redpanda/redpanda/builds/65347)
please wait until all jobs are finished before running the slash command
<!--/title-->
<!--body-->
```


/ci-repeat 1
tests/rptest/tests/maintenance_test.py::MaintenanceCycleTest.test_leader_distribution@{"use_rpk":false}
tests/rptest/transactions/consumer_offsets_test.py::VerifyConsumerOffsets.test_consumer_group_offsets
tests/rptest/tests/partition_reassignments_test.py::PartitionReassignmentsTest.test_add_partitions_with_inprogress_reassignments
tests/rptest/tests/data_migrations_api_test.py::DataMigrationsApiTest.test_migrated_topic_data_integrity@{"params":{"cancellation":{"dir":"in","stage":"preparing"},"use_alias":true},"transfer_leadership":true}
tests/rptest/tests/tiered_storage_pause_test.py::TestTieredStoragePause.test_safe_pause_resume@{"allow_gaps_cluster_level":true,"allow_gaps_topic_level":null}
tests/rptest/tests/partition_balancer_test.py::PartitionBalancerTest.test_unavailable_nodes
```
<!--/body-->
author_association: COLLABORATOR
user: login: WillemKauf
created_at: 2025-05-01T15:46:27Z
body: /ci-repeat 1
skip-redpanda-build
skip-units
tests/rptest/tests/maintenance_test.py::MaintenanceCycleTest.test_leader_distribution@{"use_rpk":false}
tests/rptest/transactions/consumer_offsets_test.py::VerifyConsumerOffsets.test_consumer_group_offsets
tests/rptest/tests/partition_reassignments_test.py::PartitionReassignmentsTest.test_add_partitions_with_inprogress_reassignments
tests/rptest/tests/data_migrations_api_test.py::DataMigrationsApiTest.test_migrated_topic_data_integrity@{"params":{"cancellation":{"dir":"in","stage":"preparing"},"use_alias":true},"transfer_leadership":true}
tests/rptest/tests/tiered_storage_pause_test.py::TestTieredStoragePause.test_safe_pause_resume@{"allow_gaps_cluster_level":true,"allow_gaps_topic_level":null}
tests/rptest/tests/partition_balancer_test.py::PartitionBalancerTest.test_unavailable_nodes
author_association: CONTRIBUTOR
user: login: vbotbuildovich
created_at: 2025-05-01T19:39:51Z
body: /backport v25.1.x
author_association: COLLABORATOR
user: login: vbotbuildovich
created_at: 2025-05-01T19:39:52Z
body: /backport v24.3.x
author_association: COLLABORATOR
user: login: vbotbuildovich
created_at: 2025-05-01T19:39:53Z
body: /backport v24.2.x
author_association: COLLABORATOR
user: login: vbotbuildovich
created_at: 2025-05-01T19:39:54Z
body: /backport v24.1.x
author_association: COLLABORATOR
user: login: vbotbuildovich
created_at: 2025-05-01T19:40:33Z
body: Oops! Something went wrong.

[Workflow run logs](https://github.com/redpanda-data/redpanda/actions/runs/14782194338).
author_association: COLLABORATOR
user: login: vbotbuildovich
created_at: 2025-05-01T19:40:58Z
body: Failed to create a backport PR to v24.2.x branch. I tried:

```
git remote add upstream https://github.com/redpanda-data/redpanda.git
git fetch --all
git checkout -b backport-pr-25974-v24.2.x-556 remotes/upstream/v24.2.x
git cherry-pick -x dffa97e16c 1a23958cda 2114ad0c2e 530b6c5bef
```

[Workflow run logs](https://github.com/redpanda-data/redpanda/actions/runs/14782193945).
author_association: COLLABORATOR
user: login: vbotbuildovich
created_at: 2025-05-01T19:41:02Z
body: Failed to create a backport PR to v24.3.x branch. I tried:

```
git remote add upstream https://github.com/redpanda-data/redpanda.git
git fetch --all
git checkout -b backport-pr-25974-v24.3.x-172 remotes/upstream/v24.3.x
git cherry-pick -x dffa97e16c 1a23958cda 2114ad0c2e 530b6c5bef
```

[Workflow run logs](https://github.com/redpanda-data/redpanda/actions/runs/14782193829).
author_association: COLLABORATOR