Repository: daos-stack/daos
Pull request #15994: DAOS-17142 rebuild: exit rebuild_tgt_status_check_ult when RPT stale
Author: login: liuxuezhao
Created: 2025-02-28T10:27:15Z | Merged: 2025-03-17T12:05:17Z

rebuild_tgt_status_check_ult() ULT only can exit when rebuild abort or globally done. When rebuild done, the leader notifies each target server by IV LAZY SYNC, the notification lost due to some network issue and lead to dangling rebuild_tgt_status_check_ult.

### Steps for the author:

* [ ] Commit message follows the [guidelines](https://daosio.atlassian.net/wiki/spaces/DC/pages/11133911069/Commit+Comments).
* [ ] Appropriate [Features or Test-tag](https://daosio.atlassian.net/wiki/spaces/DC/pages/10984259629/Test+Tags) pragmas were used.
* [ ] At least two positive code reviews including at least one code owner from each category referenced in the PR.
* [ ] Testing is complete. If necessary, forced-landing label added and a reason added in a comment.

#### After all prior steps are complete:
* [ ] Gatekeeper requested (daos-gatekeeper added as a reviewer).



Diff:
diff --git a/src/container/container_iv.c b/src/container/container_iv.c
index c51ebe183a3..7b0b79f1a17 100644
--- a/src/container/container_iv.c
+++ b/src/container/container_iv.c
@@ -473,14 +473,14 @@ cont_iv_ent_fetch(struct ds_iv_entry *entry, struct ds_iv_key *key,
 				rc = cont_iv_snap_ent_create(entry, key);
 				if (rc == 0)
 					goto again;
-				D_DEBUG(DB_MD, "create cont snap iv entry failed "
-					""DF_RC"\n", DP_RC(rc));
+				DL_ERROR(rc, "cont " DF_UUID " create IV_CONT_SNAP iv entry failed",
+					 DP_UUID(civ_key->cont_uuid));
 			} else if (class_id == IV_CONT_PROP) {
 				rc = cont_iv_prop_ent_create(entry, key);
 				if (rc == 0)
 					goto again;
-				D_ERROR("create cont prop iv entry failed "
-					""DF_RC"\n", DP_RC(rc));
+				DL_ERROR(rc, "cont " DF_UUID " create IV_CONT_PROP iv entry failed",
+					 DP_UUID(civ_key->cont_uuid));
 			} else if (class_id == IV_CONT_CAPA) {
 				struct container_hdl	chdl = { 0 };
 				int			rc1;
diff --git a/src/container/srv_container.c b/src/container/srv_container.c
index d15c597b8e2..a6e14b2f20e 100644
--- a/src/container/srv_container.c
+++ b/src/container/srv_container.c
@@ -5934,8 +5934,10 @@ ds_cont_get_prop(uuid_t pool_uuid, uuid_t cont_uuid, daos_prop_t **prop_out)
 
 	D_ASSERT(dss_get_module_info()->dmi_xs_id == 0);
 	rc = cont_svc_lookup_leader(pool_uuid, 0, &svc, NULL);
-	if (rc != 0)
+	if (rc != 0) {
+		DL_ERROR(rc, "pool " DF_UUID " cont_svc_lookup_leader failed", DP_UUID(pool_uuid));
 		return rc;
+	}
 
 	rc = rdb_tx_begin(svc->cs_rsvc->s_db, svc->cs_rsvc->s_term, &tx);
 	if (rc != 0)
@@ -5943,8 +5945,10 @@ ds_cont_get_prop(uuid_t pool_uuid, uuid_t cont_uuid, daos_prop_t **prop_out)
 
 	ABT_rwlock_rdlock(svc->cs_lock);
 	rc = cont_lookup(&tx, svc, cont_uuid, &cont);
-	if (rc != 0)
+	if (rc != 0) {
+		DL_ERROR(rc, DF_CONT " cont_lookup failed", DP_CONT(pool_uuid, cont_uuid));
 		D_GOTO(out_lock, rc);
+	}
 
 	rc = cont_prop_read(&tx, cont, DAOS_CO_QUERY_PROP_ALL, &prop, true);
 	cont_put(cont);
diff --git a/src/container/srv_epoch.c b/src/container/srv_epoch.c
index ae1e7be322e..f327f481bc4 100644
--- a/src/container/srv_epoch.c
+++ b/src/container/srv_epoch.c
@@ -1,5 +1,6 @@
 /**
  * (C) Copyright 2016-2023 Intel Corporation.
+ * (C) Copyright 2025 Hewlett Packard Enterprise Development LP
  *
  * SPDX-License-Identifier: BSD-2-Clause-Patent
  */
@@ -682,8 +683,10 @@ ds_cont_get_snapshots(uuid_t pool_uuid, uuid_t cont_uuid,
 
 	D_ASSERT(dss_get_module_info()->dmi_xs_id == 0);
 	rc = cont_svc_lookup_leader(pool_uuid, 0, &svc, NULL);
-	if (rc != 0)
+	if (rc != 0) {
+		DL_ERROR(rc, "pool " DF_UUID " cont_svc_lookup_leader failed", DP_UUID(pool_uuid));
 		return rc;
+	}
 
 	rc = rdb_tx_begin(svc->cs_rsvc->s_db, svc->cs_rsvc->s_term, &tx);
 	if (rc != 0)
@@ -691,8 +694,10 @@ ds_cont_get_snapshots(uuid_t pool_uuid, uuid_t cont_uuid,
 
 	ABT_rwlock_rdlock(svc->cs_lock);
 	rc = cont_lookup(&tx, svc, cont_uuid, &cont);
-	if (rc != 0)
+	if (rc != 0) {
+		DL_ERROR(rc, DF_CONT " cont_lookup failed", DP_CONT(pool_uuid, cont_uuid));
 		D_GOTO(out_lock, rc);
+	}
 
 	rc = read_snap_list(&tx, cont, snapshots, snap_count);
 	cont_put(cont);
diff --git a/src/rebuild/srv.c b/src/rebuild/srv.c
index af9e8b52f8c..6d12224277b 100644
--- a/src/rebuild/srv.c
+++ b/src/rebuild/srv.c
@@ -47,6 +47,44 @@ rebuild_pool_map_put(struct pool_map *map)
 	pool_map_decref(map);
 }
 
+/**
+ * Check whether the RPT is stale (new rebuild started).
+ * Only used in rebuild_tgt_status_check_ult(), the ULT only can exit when rebuild abort
+ * or globally done. When rebuild done, the leader notifies each target server by IV LAZY SYNC
+ * (see rebuild_leader_status_notify), so possibly the rebuild globally done but the async IV
+ * notification lost due to some network issue and lead to dangling rebuild_tgt_status_check_ult().
+ */
+bool
+rpt_stale(struct rebuild_tgt_pool_tracker *rpt)
+{
+	struct rebuild_tls      *tls = rebuild_tls_get();
+	struct rebuild_pool_tls *pool_tls;
+	bool                     found = false;
+
+	D_ASSERT(tls != NULL);
+	/* Only 1 thread will access the list, no need lock */
+	d_list_for_each_entry(pool_tls, &tls->rebuild_pool_list, rebuild_pool_list) {
+		if (uuid_compare(pool_tls->rebuild_pool_uuid, rpt->rt_pool_uuid) != 0)
+			continue;
+
+		if (rpt->rt_rebuild_ver == pool_tls->rebuild_pool_ver &&
+		    rpt->rt_rebuild_gen == pool_tls->rebuild_pool_gen)
+			found = true;
+
+		if ((rpt->rt_rebuild_ver < pool_tls->rebuild_pool_ver) ||
+		    (rpt->rt_rebuild_ver == pool_tls->rebuild_pool_ver &&
+		     rpt->rt_rebuild_gen < pool_tls->rebuild_pool_gen)) {
+			D_ERROR(DF_RB ": found new rebuild ver %d, gen %d\n", DP_RB_RPT(rpt),
+				pool_tls->rebuild_pool_ver, pool_tls->rebuild_pool_gen);
+			return true;
+		}
+	}
+
+	if (!found)
+		D_ERROR(DF_RB ": rebuild_tls not found\n", DP_RB_RPT(rpt));
+	return !found;
+}
+
 struct rebuild_pool_tls *
 rebuild_pool_tls_lookup(uuid_t pool_uuid, unsigned int ver, uint32_t gen)
 {
@@ -2601,6 +2639,10 @@ rebuild_tgt_status_check_ult(void *arg)
 			break;
 
 		sched_req_sleep(rpt->rt_ult, RBLD_CHECK_INTV);
+		if (iv.riv_pull_done && rpt_stale(rpt)) {
+			D_ERROR(DF_RB " is stale, exit the ULT.\n", DP_RB_RPT(rpt));
+			break;
+		}
 	}
 
 	sched_req_put(rpt->rt_ult);


Reviews:
user: login: Nasf-Fan
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-07T04:42:01Z
user: login: liuxuezhao
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-07T07:46:42Z
user: login: Nasf-Fan
state: DISMISSED
author_association: CONTRIBUTOR
submitted_at: 2025-03-07T14:25:42Z
user: login: kccain
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-07T20:04:31Z
user: login: kccain
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-03-07T20:05:50Z
user: login: kccain
body: Probably need to run this with Features: rebuild actually.
state: CHANGES_REQUESTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-07T20:12:53Z
user: login: liuxuezhao
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-10T02:26:34Z
user: login: kccain
state: DISMISSED
author_association: CONTRIBUTOR
submitted_at: 2025-03-10T14:27:19Z
user: login: liuxuezhao
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-10T15:06:53Z
user: login: daltonbohning
body: The lint check is required and the errors are introduced by this PR so IMO that should be fixed now.
state: DISMISSED
author_association: CONTRIBUTOR
submitted_at: 2025-03-10T16:42:30Z
user: login: kccain
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-03-12T12:24:10Z
user: login: Nasf-Fan
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-03-17T10:55:45Z

Review comments:
diff_hunk: @@ -47,6 +47,44 @@ rebuild_pool_map_put(struct pool_map *map)
 	pool_map_decref(map);
 }
 
+/**
+ * Check whether the RPT is stale (new rebuild started).
+ * Only used in rebuild_tgt_status_check_ult(), the ULT only can exit when rebuild abort
+ * or globally done. When rebuild done, the leader notifies each target server by IV LAZY SYNC
+ * (see rebuild_leader_status_notify), so possibly the rebuild globally done but the async IV
+ * notification lost due to some network issue and lead to dangling rebuild_tgt_status_check_ult().
+ */
+bool
+rpt_stale(struct rebuild_tgt_pool_tracker *rpt)
+{
+	struct rebuild_tls      *tls = rebuild_tls_get();
+	struct rebuild_pool_tls *pool_tls;
+	bool                     found = false;
+
+	D_ASSERT(tls != NULL);
+	/* Only 1 thread will access the list, no need lock */
+	d_list_for_each_entry(pool_tls, &tls->rebuild_pool_list, rebuild_pool_list) {
+		if (uuid_compare(pool_tls->rebuild_pool_uuid, rpt->rt_pool_uuid) != 0)
+			continue;
+
+		if (rpt->rt_rebuild_ver == pool_tls->rebuild_pool_ver &&
+		    rpt->rt_rebuild_gen == pool_tls->rebuild_pool_gen)
+			found = true;
path: src/rebuild/srv.c
user: login: Nasf-Fan
body: Will we merge multiple rebuild instances for the same pool in the list?
created_at: 2025-03-07T04:42:01Z
author_association: CONTRIBUTOR
diff_hunk: @@ -47,6 +47,44 @@ rebuild_pool_map_put(struct pool_map *map)
 	pool_map_decref(map);
 }
 
+/**
+ * Check whether the RPT is stale (new rebuild started).
+ * Only used in rebuild_tgt_status_check_ult(), the ULT only can exit when rebuild abort
+ * or globally done. When rebuild done, the leader notifies each target server by IV LAZY SYNC
+ * (see rebuild_leader_status_notify), so possibly the rebuild globally done but the async IV
+ * notification lost due to some network issue and lead to dangling rebuild_tgt_status_check_ult().
+ */
+bool
+rpt_stale(struct rebuild_tgt_pool_tracker *rpt)
+{
+	struct rebuild_tls      *tls = rebuild_tls_get();
+	struct rebuild_pool_tls *pool_tls;
+	bool                     found = false;
+
+	D_ASSERT(tls != NULL);
+	/* Only 1 thread will access the list, no need lock */
+	d_list_for_each_entry(pool_tls, &tls->rebuild_pool_list, rebuild_pool_list) {
+		if (uuid_compare(pool_tls->rebuild_pool_uuid, rpt->rt_pool_uuid) != 0)
+			continue;
+
+		if (rpt->rt_rebuild_ver == pool_tls->rebuild_pool_ver &&
+		    rpt->rt_rebuild_gen == pool_tls->rebuild_pool_gen)
+			found = true;
path: src/rebuild/srv.c
user: login: liuxuezhao
body: Rebuild leader may try to merge to one rebuild, but in tgt server it will not merge the rpt for different versions/gens.
In a test we found orphan rebuild_tgt_status_check_ult left after rebuild done, reason not very clear, not very sure if it is the root cause of the original case TBH. This PR is to add a kind of exiting method by a safe way so should be no losing at least.
created_at: 2025-03-07T07:46:41Z
author_association: CONTRIBUTOR
diff_hunk: @@ -473,14 +473,16 @@ cont_iv_ent_fetch(struct ds_iv_entry *entry, struct ds_iv_key *key,
 				rc = cont_iv_snap_ent_create(entry, key);
 				if (rc == 0)
 					goto again;
-				D_DEBUG(DB_MD, "create cont snap iv entry failed "
-					""DF_RC"\n", DP_RC(rc));
+				D_DEBUG(DB_MD,
+					"cont " DF_UUID " create IV_CONT_SNAP iv entry failed\n",
+					DP_UUID(civ_key->cont_uuid));
 			} else if (class_id == IV_CONT_PROP) {
 				rc = cont_iv_prop_ent_create(entry, key);
 				if (rc == 0)
 					goto again;
-				D_ERROR("create cont prop iv entry failed "
-					""DF_RC"\n", DP_RC(rc));
+				DL_ERROR(rc,
path: src/container/container_iv.c
user: login: kccain
body: do we know why this is ERR, but the above IV_CONT_SNAP is DBG?
created_at: 2025-03-07T18:39:38Z
author_association: CONTRIBUTOR
diff_hunk: @@ -47,6 +47,44 @@ rebuild_pool_map_put(struct pool_map *map)
 	pool_map_decref(map);
 }
 
+/**
+ * Check whether the RPT is stale (new rebuild started).
+ * Only used in rebuild_tgt_status_check_ult(), the ULT only can exit when rebuild abort
+ * or globally done. When rebuild done, the leader notifies each target server by IV LAZY SYNC
+ * (see rebuild_leader_status_notify), so possibly the rebuild globally done but the async IV
+ * notification lost due to some network issue and lead to dangling rebuild_tgt_status_check_ult().
+ */
+bool
+rpt_stale(struct rebuild_tgt_pool_tracker *rpt)
path: src/rebuild/srv.c
user: login: kccain
body: With this new logic, the tgt ULT for a global done rebuild can be cleaned up the next time a new rebuild with a higher pool map version or generation number is started on the tgt engine? So most likely for a rebuild that is really globally done, this cleanup would happen with the Reclaim request, whenever it may be able to be received by this tgt engine (which may be having problems receiving RPCs from other engines)?

If we have a different sequence (outside scope of this change I guess?): rebuild (aborted) map version 285 gen 1, then a "Reclaim fail" version 268 gen 2, then Rebuild version 302 gen3, we expect that other existing logic in the tgt engine will recognize that the first rebuild is aborted, so that will get reaped? Because this logic here will not be able to detect that rebuild version 285 gen 1 is stale when "reclaim fail" version 268 gen 2 is in progress. It can be cleaned up I think when rebuild version 302 gen 3 arrives. Do I have that right?
created_at: 2025-03-07T20:03:23Z
author_association: CONTRIBUTOR
diff_hunk: @@ -47,6 +47,44 @@ rebuild_pool_map_put(struct pool_map *map)
 	pool_map_decref(map);
 }
 
+/**
+ * Check whether the RPT is stale (new rebuild started).
+ * Only used in rebuild_tgt_status_check_ult(), the ULT only can exit when rebuild abort
+ * or globally done. When rebuild done, the leader notifies each target server by IV LAZY SYNC
+ * (see rebuild_leader_status_notify), so possibly the rebuild globally done but the async IV
+ * notification lost due to some network issue and lead to dangling rebuild_tgt_status_check_ult().
+ */
+bool
+rpt_stale(struct rebuild_tgt_pool_tracker *rpt)
path: src/rebuild/srv.c
user: login: liuxuezhao
body: > With this new logic, the tgt ULT for a global done rebuild can be cleaned up the next time a new rebuild with a higher pool map version or generation number is started on the tgt engine? 

right

> 
> If we have a different sequence (outside scope of this change I guess?): rebuild (aborted) map version 285 gen 1, then a "Reclaim fail" version 268 gen 2, then Rebuild version 302 gen3, we expect that other existing logic in the tgt engine will recognize that the first rebuild is aborted, so that will get reaped? Because this logic here will not be able to detect that rebuild version 285 gen 1 is stale when "reclaim fail" version 268 gen 2 is in progress. It can be cleaned up I think when rebuild version 302 gen 3 arrives. Do I have that right?

if the rebuild ver 285 aborted, it will schedule a RECLAIM_FAIL rebuild task at ver 285.
If something wrong happened that with a "Reclaim fail" version 268 (I don't know how could it happen although), yes it will be cleanup when higher version rebuild started.


created_at: 2025-03-10T02:26:34Z
author_association: CONTRIBUTOR
diff_hunk: @@ -473,14 +473,16 @@ cont_iv_ent_fetch(struct ds_iv_entry *entry, struct ds_iv_key *key,
 				rc = cont_iv_snap_ent_create(entry, key);
 				if (rc == 0)
 					goto again;
-				D_DEBUG(DB_MD, "create cont snap iv entry failed "
-					""DF_RC"\n", DP_RC(rc));
+				D_DEBUG(DB_MD,
+					"cont " DF_UUID " create IV_CONT_SNAP iv entry failed\n",
+					DP_UUID(civ_key->cont_uuid));
 			} else if (class_id == IV_CONT_PROP) {
 				rc = cont_iv_prop_ent_create(entry, key);
 				if (rc == 0)
 					goto again;
-				D_ERROR("create cont prop iv entry failed "
-					""DF_RC"\n", DP_RC(rc));
+				DL_ERROR(rc,
path: src/container/container_iv.c
user: login: liuxuezhao
body: I did not try to change the log level, just to add the cont uuid to the log.
Probably can change L476 to D_ERROR, the extra "\n" can be removed, I'll change it in a following PR.
created_at: 2025-03-10T15:06:53Z
author_association: CONTRIBUTOR

PR comments:
user: login: github-actions[bot]
created_at: 2025-02-28T10:59:20Z
body: Ticket title is 'clean up rebuild_tgt_status_check() ULTs for globally-done rebuilds whose leader to target engine IV updates failed'
Status is 'In Progress'
https://daosio.atlassian.net/browse/DAOS-17142
<!-- thollander/actions-comment-pull-request "jira_query_message" -->
author_association: NONE
user: login: liuxuezhao
created_at: 2025-03-10T02:29:00Z
body: > Probably need to run this with Features: rebuild actually.

the code change in this PR did not change core rebuild logic, it only check rpt_stale() when v.riv_pull_done which should be quite safe. The default testing already with lots of rebuild test cases, so I think unnecessary to add Features: rebuild to save CI resource.
author_association: CONTRIBUTOR
user: login: kccain
created_at: 2025-03-10T13:12:10Z
body: > > Probably need to run this with Features: rebuild actually.
> 
> the code change in this PR did not change core rebuild logic, it only check rpt_stale() when v.riv_pull_done which should be quite safe. The default testing already with lots of rebuild test cases, so I think unnecessary to add Features: rebuild to save CI resource.

OK that seems reasonable then.

The other comments in the patch were from GHA complaining about DL_ERROR() that has an unnecessary "\n". Will those be fixed in this PR?
author_association: CONTRIBUTOR
user: login: liuxuezhao
created_at: 2025-03-10T15:09:07Z
body: > The other comments in the patch were from GHA complaining about DL_ERROR() that has an unnecessary "\n". Will those be fixed in this PR?

I'll fix the log format, may in a following PR.
author_association: CONTRIBUTOR
user: login: liuxuezhao
created_at: 2025-03-11T09:16:26Z
body: > The lint check is required and the errors are introduced by this PR so IMO that should be fixed now.

OK, fair enough, I'll fix it then.
author_association: CONTRIBUTOR