Repository: ceph/ceph
Pull request #62837: crimson: fix several bugs causing stuck backfills
Author: login: athanatos
Labels: name: crimson
Created: 2025-04-16T02:07:45Z | Merged: 2025-04-22T15:02:59Z

  - https://tracker.ceph.com/issues/70935
  - https://tracker.ceph.com/issues/70936
  - https://tracker.ceph.com/issues/70939

<details>
<summary>Show available Jenkins commands</summary>

- `jenkins test classic perf` [Jenkins Job](https://jenkins.ceph.com/view/all/job/ceph-perf-classic/) | [Jenkins Job Definition](https://github.com/ceph/ceph-build/blob/main/ceph-perf-pull-requests/config/definitions/ceph-perf-pull-requests.yml)
- `jenkins test crimson perf` [Jenkins Job](https://jenkins.ceph.com/view/all/job/ceph-perf-crimson/) | [Jenkins Job Definition](https://github.com/ceph/ceph-build/blob/main/ceph-perf-pull-requests/config/definitions/ceph-perf-pull-requests.yml)
- `jenkins test signed` [Jenkins Job](https://jenkins.ceph.com/job/ceph-pr-commits/) | [Jenkins Job Definition](https://github.com/ceph/ceph-build/blob/main/ceph-pr-commits/config/definitions/ceph-pr-commits.yml)
- `jenkins test make check` [Jenkins Job](https://jenkins.ceph.com/job/ceph-pull-requests/) | [Jenkins Job Definition](https://github.com/ceph/ceph-build/blob/main/ceph-pull-requests/config/definitions/ceph-pull-requests.yml)
- `jenkins test make check arm64` [Jenkins Job](https://jenkins.ceph.com/job/ceph-pull-requests-arm64/) | [Jenkins Job Definition](https://github.com/ceph/ceph-build/blob/main/ceph-pull-requests-arm64/config/definitions/ceph-pull-requests-arm64.yml)
- `jenkins test submodules` [Jenkins Job](https://jenkins.ceph.com/view/all/job/ceph-pr-submodules/) | [Jenkins Job Definition](https://github.com/ceph/ceph-build/blob/main/ceph-pr-submodules/config/definitions/ceph-pr-commits.yml)
- `jenkins test dashboard` [Jenkins Job](https://jenkins.ceph.com/view/all/job/ceph-dashboard-pull-requests/) | [Jenkins Job Definition](https://github.com/ceph/ceph-build/blob/main/ceph-dashboard-pull-requests/config/definitions/ceph-dashboard-pull-requests.yml)
- `jenkins test dashboard cephadm` [Jenkins Job](https://jenkins.ceph.com/view/all/job/ceph-dashboard-cephadm-e2e/) | [Jenkins Job Definition](https://github.com/ceph/ceph-build/blob/main/ceph-dashboard-cephadm-e2e/config/definitions/ceph-dashboard-cephadm-e2e.yml)
- `jenkins test api` [Jenkins Job](https://jenkins.ceph.com/view/all/job/ceph-api/) | [Jenkins Job Definition](https://github.com/ceph/ceph-build/blob/main/ceph-pr-api/config/definitions/ceph-pr-api.yml)
- `jenkins test docs` [ReadTheDocs](https://readthedocs.org/projects/ceph/) | [Github Workflow Definition](https://github.com/ceph/ceph/blob/main/.readthedocs.yml)
- `jenkins test ceph-volume all` [Jenkins Jobs](https://jenkins.ceph.com/view/ceph-volume%20PR/) | [Jenkins Jobs Definition](https://github.com/ceph/ceph-build/blob/main/ceph-volume-cephadm-prs/config/definitions/ceph-volume-pr.yml)
- `jenkins test windows` [Jenkins Job](https://jenkins.ceph.com/job/ceph-windows-pull-requests/) | [Jenkins Job Definition](https://github.com/ceph/ceph-build/blob/main/ceph-windows-pull-requests/config/definitions/ceph-windows-pull-requests.yml)
- `jenkins test rook e2e` [Jenkins Job](https://jenkins.ceph.com/view/all/job/ceph-orchestrator-rook-e2e/) | [Jenkins Job Definition](https://github.com/ceph/ceph-build/blob/main/ceph-rook-e2e/config/definitions/ceph-orchestrator-rook-e2e.yml)
</details>



Diff:
diff --git a/src/crimson/osd/osd_operation.h b/src/crimson/osd/osd_operation.h
index 6e75f2f826f25..167ceac463f3a 100644
--- a/src/crimson/osd/osd_operation.h
+++ b/src/crimson/osd/osd_operation.h
@@ -298,34 +298,6 @@ class OperationThrottler : public BlockerT<OperationThrottler>,
   friend BlockerT<OperationThrottler>;
   static constexpr const char* type_name = "OperationThrottler";
 
-  template <typename OperationT, typename F>
-  auto with_throttle(
-    OperationT* op,
-    crimson::osd::scheduler::params_t params,
-    F &&f) {
-    if (!max_in_progress) return f();
-    return acquire_throttle(params)
-      .then(std::forward<F>(f))
-      .then([this](auto x) {
-	release_throttle();
-	return x;
-      });
-  }
-
-  template <typename OperationT, typename F>
-  seastar::future<> with_throttle_while(
-    OperationT* op,
-    crimson::osd::scheduler::params_t params,
-    F &&f) {
-    return with_throttle(op, params, f).then([this, params, op, f](bool cont) {
-      return cont
-	? seastar::yield().then([params, op, f, this] {
-	  return with_throttle_while(op, params, f); })
-	: seastar::now();
-    });
-  }
-
-
 public:
   OperationThrottler(ConfigProxy &conf);
 
@@ -338,24 +310,28 @@ class OperationThrottler : public BlockerT<OperationThrottler>,
     return !max_in_progress || in_progress < max_in_progress;
   }
 
-  template <typename F>
-  auto with_throttle(
-    crimson::osd::scheduler::params_t params,
-    F &&f) {
-    if (!max_in_progress) return f();
-    return acquire_throttle(params)
-      .then(std::forward<F>(f))
-      .finally([this] {
-	release_throttle();
-      });
-  }
+  class ThrottleReleaser {
+    OperationThrottler *parent = nullptr;
+  public:
+    ThrottleReleaser(OperationThrottler *parent) : parent(parent) {}
+    ThrottleReleaser(const ThrottleReleaser &) = delete;
+    ThrottleReleaser(ThrottleReleaser &&rhs) noexcept {
+      std::swap(parent, rhs.parent);
+    }
 
-  template <class OpT, class... Args>
-  seastar::future<> with_throttle_while(
-    BlockingEvent::Trigger<OpT>&& trigger,
-    Args&&... args) {
-    return trigger.maybe_record_blocking(
-      with_throttle_while(std::forward<Args>(args)...), *this);
+    ~ThrottleReleaser() {
+      if (parent) {
+	parent->release_throttle();
+      }
+    }
+  };
+
+  auto get_throttle(crimson::osd::scheduler::params_t params) {
+    return acquire_throttle(
+      params
+    ).then([this] {
+      return ThrottleReleaser{this};
+    });
   }
 
 private:
diff --git a/src/crimson/osd/pg.h b/src/crimson/osd/pg.h
index 0a20f2c4d6241..110c5e79dad61 100644
--- a/src/crimson/osd/pg.h
+++ b/src/crimson/osd/pg.h
@@ -733,6 +733,9 @@ class PG : public boost::intrusive_ref_counter<
   ShardServices& get_shard_services() final {
     return shard_services;
   }
+  DoutPrefixProvider& get_dpp() final {
+    return *this;
+  }
   seastar::future<> stop();
 private:
   class C_PG_FinishRecovery : public Context {
diff --git a/src/crimson/osd/pg_recovery.cc b/src/crimson/osd/pg_recovery.cc
index 2b28f7e4f0f4b..d99ed7dfd7084 100644
--- a/src/crimson/osd/pg_recovery.cc
+++ b/src/crimson/osd/pg_recovery.cc
@@ -5,6 +5,8 @@
 #include <fmt/ostream.h>
 #include <fmt/ranges.h>
 
+#include "crimson/common/coroutine.h"
+#include "crimson/common/log.h"
 #include "crimson/common/type_helpers.h"
 #include "crimson/osd/backfill_facades.h"
 #include "crimson/osd/osd_operations/background_recovery.h"
@@ -16,6 +18,8 @@
 #include "osd/osd_types.h"
 #include "osd/PeeringState.h"
 
+SET_SUBSYS(osd);
+
 namespace {
   seastar::logger& logger() {
     return crimson::get_logger(ceph_subsys_osd);
@@ -522,21 +526,19 @@ void PGRecovery::request_primary_scan(
 
 PGRecovery::interruptible_future<>
 PGRecovery::recover_object_with_throttle(
-  const hobject_t &soid,
+  hobject_t soid,
   eversion_t need)
 {
-  crimson::osd::scheduler::params_t params =
-    {1, 0, crimson::osd::scheduler::scheduler_class_t::background_best_effort};
-  auto &ss = pg->get_shard_services();
-  logger().debug("{} {}", soid, need);
-  return ss.with_throttle(
-    std::move(params),
-    [this, soid, need] {
-    logger().debug("got throttle: {} {}", soid, need);
-    auto backend = pg->get_recovery_backend();
-    assert(backend);
-    return backend->recover_object(soid, need);
-  });
+  LOG_PREFIX(PGRecovery::recover_object_with_throttle);
+  DEBUGDPP("{} {}", pg->get_dpp(), soid, need);
+  auto releaser = co_await interruptor::make_interruptible(
+    pg->get_shard_services().get_throttle(
+      crimson::osd::scheduler::params_t{
+	1, 0, crimson::osd::scheduler::scheduler_class_t::background_best_effort
+      }));
+  DEBUGDPP("got throttle: {} {}", pg->get_dpp(), soid, need);
+  co_await pg->get_recovery_backend()->recover_object(soid, need);
+  co_return;
 }
 
 void PGRecovery::enqueue_push(
diff --git a/src/crimson/osd/pg_recovery.h b/src/crimson/osd/pg_recovery.h
index 37fae278fa09f..a44118cac5a00 100644
--- a/src/crimson/osd/pg_recovery.h
+++ b/src/crimson/osd/pg_recovery.h
@@ -25,6 +25,9 @@ class PGBackend;
 
 class PGRecovery : public crimson::osd::BackfillState::BackfillListener {
 public:
+  using interruptor =
+    ::crimson::interruptible::interruptor<
+      ::crimson::osd::IOInterruptCondition>;
   template <typename T = void>
   using interruptible_future = RecoveryBackend::interruptible_future<T>;
   PGRecovery(PGRecoveryListener* pg) : pg(pg) {}
@@ -100,7 +103,7 @@ class PGRecovery : public crimson::osd::BackfillState::BackfillListener {
   friend class crimson::osd::UrgentRecovery;
 
   interruptible_future<> recover_object_with_throttle(
-    const hobject_t &soid,
+    hobject_t soid,
     eversion_t need);
 
   interruptible_future<> recover_object(
diff --git a/src/crimson/osd/pg_recovery_listener.h b/src/crimson/osd/pg_recovery_listener.h
index 6c88b170b4fb9..347a3b2194e31 100644
--- a/src/crimson/osd/pg_recovery_listener.h
+++ b/src/crimson/osd/pg_recovery_listener.h
@@ -20,6 +20,7 @@ class PGRecovery;
 class PGRecoveryListener {
 public:
   virtual crimson::osd::ShardServices& get_shard_services() = 0;
+  virtual DoutPrefixProvider& get_dpp() = 0;
   virtual PGRecovery* get_recovery_handler() = 0;
   virtual epoch_t get_osdmap_epoch() const = 0;
   virtual bool is_primary() const = 0;
diff --git a/src/crimson/osd/recovery_backend.cc b/src/crimson/osd/recovery_backend.cc
index b0dbcfba9839f..f30d34195aa25 100644
--- a/src/crimson/osd/recovery_backend.cc
+++ b/src/crimson/osd/recovery_backend.cc
@@ -235,12 +235,25 @@ RecoveryBackend::scan_for_backfill(
   co_await interruptor::parallel_for_each(objects, seastar::coroutine::lambda([FNAME, this, version_map]
     (const hobject_t& object) -> interruptible_future<> {
     DEBUGDPP("querying obj:{}", pg, object);
-    auto obc_manager = pg.obc_loader.get_obc_manager(object);
-    co_await pg.obc_loader.load_and_lock(
+    auto obc_manager = pg.obc_loader.get_obc_manager(
+      object, /* resolve_clone = */ false);
+
+    auto found = co_await pg.obc_loader.load_and_lock(
       obc_manager, RWState::RWREAD
-    ).handle_error_interruptible(
+    ).si_then([] {
+      return true;
+    }).handle_error_interruptible(
+      crimson::ct_error::enoent::handle([](auto) {
+	return false;
+      }),
       crimson::ct_error::assert_all("unexpected error")
     );
+    if (!found) {
+      // if the object does not exist here, it must have been removed
+      // between the collection_list_partial and here.  This can happen
+      // for the first item in the range, which is usually last_backfill.
+      co_return;
+    }
 
     if (obc_manager.get_obc()->obs.exists) {
       auto version = obc_manager.get_obc()->obs.oi.version;
diff --git a/src/crimson/osd/shard_services.h b/src/crimson/osd/shard_services.h
index 38daf776b9925..61880909bcddd 100644
--- a/src/crimson/osd/shard_services.h
+++ b/src/crimson/osd/shard_services.h
@@ -593,7 +593,7 @@ class ShardServices : public OSDMapService {
   }
 
   FORWARD_TO_OSD_SINGLETON(get_pool_info)
-  FORWARD(with_throttle, with_throttle, local_state.throttler)
+  FORWARD(get_throttle, get_throttle, local_state.throttler)
 
   FORWARD_TO_OSD_SINGLETON(build_incremental_map_msg)
   FORWARD_TO_OSD_SINGLETON(send_incremental_map)


Reviews:
user: login: Matan-B
body: lgtm! One question regarding the resolving bug - probably not a blocker.
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-22T07:22:47Z
user: login: Matan-B
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-22T07:25:16Z
user: login: athanatos
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-22T14:47:09Z
user: login: Matan-B
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-04-22T14:48:41Z
user: login: athanatos
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-22T15:02:00Z

Review comments:
diff_hunk: @@ -235,8 +235,8 @@ RecoveryBackend::scan_for_backfill(
   co_await interruptor::parallel_for_each(objects, seastar::coroutine::lambda([FNAME, this, version_map]
     (const hobject_t& object) -> interruptible_future<> {
     DEBUGDPP("querying obj:{}", pg, object);
-    auto obc_manager = pg.obc_loader.get_obc_manager(object);
-    co_await pg.obc_loader.load_and_lock(
+    auto obc_manager = pg.obc_loader.get_obc_manager(
+      object, /* resolve_clone = */ false);
path: src/crimson/osd/recovery_backend.cc
user: login: Matan-B
body: > The clone_id of the clone is not necessarily a valid snapshot of the object.
> Reproduces on any clone whose id doesn't happen to be a live snapshot.

Since this by no means a client originated request this, getting the _exact_ hobject_t makes sense to me.
However, my only hesitation here is with us actually emplacing the (possibly) to-be trimmed objects into the backfill interval.
Meaning shouldn't we rely on resolve_oid here to be used as a removed snap management check?
In a scenario in which a snapshot is removed, a belonging clone won't be resolved, why should it be part of the backfill interval in this case?

I think that the described case is possible since the object here is obtained _directly_ from disk - without any metadata on the logical state of this object. The resolve check might actually help us gain information on this object (i.e to-be trimmed).

Assuming the above is somewhat correct, without resolving the clone object, we possibly backfill a to-be trimmed clone or if trimming was already kicked - this clone object will be logically stale(leaked in way)?
created_at: 2025-04-22T07:10:08Z
author_association: CONTRIBUTOR
diff_hunk: @@ -237,10 +237,23 @@ RecoveryBackend::scan_for_backfill(
     DEBUGDPP("querying obj:{}", pg, object);
     auto obc_manager = pg.obc_loader.get_obc_manager(
       object, /* resolve_clone = */ false);
+
+    auto found = co_await pg.obc_loader.load_and_lock(
       obc_manager, RWState::RWREAD
-    ).handle_error_interruptible(
+    ).si_then([] {
+      return true;
+    }).handle_error_interruptible(
+      crimson::ct_error::enoent::handle([](auto) {
+	return false;
+      }),
       crimson::ct_error::assert_all("unexpected error")
     );
+    if (!found) {
+      // if the object does not exist here, it must have been removed
+      // between the collection_list_partial and here.  This can happen
+      // for the first item in the range, which is usually last_backfill.
+      co_return;
path: src/crimson/osd/recovery_backend.cc
user: login: Matan-B
body: nit: Should we add a log line here to more easily spot cases where `scan_for_backfill` returns earlier than expected? Could help with future investigations 
created_at: 2025-04-22T07:18:35Z
author_association: CONTRIBUTOR
diff_hunk: @@ -525,22 +526,19 @@ void PGRecovery::request_primary_scan(
 
 PGRecovery::interruptible_future<>
 PGRecovery::recover_object_with_throttle(
-  const hobject_t &soid,
+  hobject_t soid,
   eversion_t need)
 {
   LOG_PREFIX(PGRecovery::recover_object_with_throttle);
-  crimson::osd::scheduler::params_t params =
-    {1, 0, crimson::osd::scheduler::scheduler_class_t::background_best_effort};
-  auto &ss = pg->get_shard_services();
   DEBUGDPP("{} {}", pg->get_dpp(), soid, need);
-  return ss.with_throttle(
-    std::move(params),
-    [FNAME, this, soid, need] {
-    DEBUGDPP("got throttle: {} {}", pg->get_dpp(), soid, need);
-    auto backend = pg->get_recovery_backend();
-    assert(backend);
-    return backend->recover_object(soid, need);
-  });
+  auto releaser = co_await interruptor::make_interruptible(
+    pg->get_shard_services().get_throttle(
+      crimson::osd::scheduler::params_t{
+	1, 0, crimson::osd::scheduler::scheduler_class_t::background_best_effort
+      }));
+  DEBUGDPP("got throttle: {} {}", pg->get_dpp(), soid, need);
+  co_await pg->get_recovery_backend()->recover_object(soid, need);
+  co_return;
path: src/crimson/osd/pg_recovery.cc
user: login: Matan-B
body: nit: `DEBUGDPP("releasing throttle: {} {}", pg->get_dpp(), soid, need);`
created_at: 2025-04-22T07:21:15Z
author_association: CONTRIBUTOR
diff_hunk: @@ -522,21 +526,19 @@ void PGRecovery::request_primary_scan(
 
 PGRecovery::interruptible_future<>
 PGRecovery::recover_object_with_throttle(
-  const hobject_t &soid,
+  hobject_t soid,
   eversion_t need)
 {
-  crimson::osd::scheduler::params_t params =
-    {1, 0, crimson::osd::scheduler::scheduler_class_t::background_best_effort};
-  auto &ss = pg->get_shard_services();
-  logger().debug("{} {}", soid, need);
-  return ss.with_throttle(
-    std::move(params),
-    [this, soid, need] {
-    logger().debug("got throttle: {} {}", soid, need);
-    auto backend = pg->get_recovery_backend();
-    assert(backend);
-    return backend->recover_object(soid, need);
-  });
+  LOG_PREFIX(PGRecovery::recover_object_with_throttle);
+  DEBUGDPP("{} {}", pg->get_dpp(), soid, need);
+  auto releaser = co_await interruptor::make_interruptible(
path: src/crimson/osd/pg_recovery.cc
user: login: Matan-B
body: :+1: 
created_at: 2025-04-22T07:25:16Z
author_association: CONTRIBUTOR
diff_hunk: @@ -237,10 +237,23 @@ RecoveryBackend::scan_for_backfill(
     DEBUGDPP("querying obj:{}", pg, object);
     auto obc_manager = pg.obc_loader.get_obc_manager(
       object, /* resolve_clone = */ false);
+
+    auto found = co_await pg.obc_loader.load_and_lock(
       obc_manager, RWState::RWREAD
-    ).handle_error_interruptible(
+    ).si_then([] {
+      return true;
+    }).handle_error_interruptible(
+      crimson::ct_error::enoent::handle([](auto) {
+	return false;
+      }),
       crimson::ct_error::assert_all("unexpected error")
     );
+    if (!found) {
+      // if the object does not exist here, it must have been removed
+      // between the collection_list_partial and here.  This can happen
+      // for the first item in the range, which is usually last_backfill.
+      co_return;
path: src/crimson/osd/recovery_backend.cc
user: login: athanatos
body: Doing that now in a follow up for https://tracker.ceph.com/issues/71003
created_at: 2025-04-22T14:47:09Z
author_association: CONTRIBUTOR
diff_hunk: @@ -235,8 +235,8 @@ RecoveryBackend::scan_for_backfill(
   co_await interruptor::parallel_for_each(objects, seastar::coroutine::lambda([FNAME, this, version_map]
     (const hobject_t& object) -> interruptible_future<> {
     DEBUGDPP("querying obj:{}", pg, object);
-    auto obc_manager = pg.obc_loader.get_obc_manager(object);
-    co_await pg.obc_loader.load_and_lock(
+    auto obc_manager = pg.obc_loader.get_obc_manager(
+      object, /* resolve_clone = */ false);
path: src/crimson/osd/recovery_backend.cc
user: login: athanatos
body: resolve_clone is really only for client reads -- the client supplied snapshot needs to be a current, actual snapshot of the object.  However, the cloneid on the object may be for a removed clone or may not necessarily ever have been a valid snapshot on the object.
created_at: 2025-04-22T15:02:00Z
author_association: CONTRIBUTOR

PR comments:
user: login: athanatos
created_at: 2025-04-21T16:08:49Z
body: https://pulpito.ceph.com/sjust-2025-04-17_01:11:15-crimson-rados-wip-sjust-crimson-testing-2025-04-16-1744840247-distro-default-gibba/ -- 50/51 pass

Failure https://tracker.ceph.com/issues/71002 -- seems to be in scan_for_backfill, but doesn't seem to be related to these patches.

Tested
author_association: CONTRIBUTOR
user: login: Matan-B
created_at: 2025-04-22T06:52:45Z
body: jenkins test make check arm64
author_association: CONTRIBUTOR
user: login: Matan-B
created_at: 2025-04-22T07:35:24Z
body: @athanatos, please see https://github.com/ceph/ceph/pull/62619#issuecomment-2820407929
Do you see it as a blocker?
author_association: CONTRIBUTOR