Repository: aws/aws-ofi-nccl
Pull request #870: Add accessors to device and domain for OFI objects
Author: login: rauteric
Created: 2025-05-02T22:35:58Z | Merged: 2025-05-23T01:59:17Z

- device->get_ofi_info(): get fi_info * associated with this device
- domain->get_ofi_domain(): get fid_domain * associated with this plugin
  domain
- domain->get_ofi_cq(): get fid_cq * associated with this plugin domain

For RDMA protocol, these accessors return objects associated with the
"leader NIC" of the multi-rail group.

This will be useful for the upcoming connection manager, which can take
a `nccl_net_ofi_domain_t` object as a parameter, instead of individual
Libfabric objects.

By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.



Diff:
diff --git a/include/nccl_ofi.h b/include/nccl_ofi.h
index e880fc59e2..7c6c7fbfd5 100644
--- a/include/nccl_ofi.h
+++ b/include/nccl_ofi.h
@@ -306,6 +306,13 @@ struct nccl_net_ofi_device {
 	 */
 	nccl_net_ofi_domain_t *(*get_domain)(nccl_net_ofi_device_t *dev);
 
+	/**
+	 * Retrieve an fi_info object associated with this device. There may be
+	 * more than one info per device, depending on the transport; in that
+	 * case, this will be the info object associated with the "leader NIC"
+	 */
+	struct fi_info *(*get_ofi_info)(nccl_net_ofi_device_t *dev);
+
 	int (*get_ep)(nccl_net_ofi_device_t *base_dev,
 		      nccl_net_ofi_ep_t **ep);
 
@@ -395,6 +402,21 @@ struct nccl_net_ofi_domain {
 
 	pthread_mutex_t domain_lock;
 
+	/**
+	 * Retrieve an fid_domain object associated with this domain. There may
+	 * be more than one fid_domain per domain, depending on the transport;
+	 * in that case, this will be the domain object associated with the
+	 * "leader NIC"
+	 */
+	struct fid_domain *(*get_ofi_domain)(nccl_net_ofi_domain_t *domain);
+
+	/**
+	 * Retrieve an fid_cq object associated with this domain. There may be
+	 * more than one cq per domain, depending on the transport; in that
+	 * case, this will be the info object associated with the "leader NIC"
+	 */
+	struct fid_cq *(*get_ofi_cq)(nccl_net_ofi_domain_t *domain);
+
 /* Private */
 	/* pure virtual function called when resources associated with
 	 * the ep should be destroyed.  Device lock will be held when
diff --git a/src/nccl_ofi_rdma.cpp b/src/nccl_ofi_rdma.cpp
index 99770c5b08..67a7a43ceb 100644
--- a/src/nccl_ofi_rdma.cpp
+++ b/src/nccl_ofi_rdma.cpp
@@ -7462,6 +7462,20 @@ nccl_net_ofi_rdma_domain_free(nccl_net_ofi_domain_t *base_domain)
 }
 
 
+static inline struct fid_domain *rdma_domain_get_ofi_domain(nccl_net_ofi_domain_t *base_domain) {
+	auto domain = reinterpret_cast<nccl_net_ofi_rdma_domain_t *>(base_domain);
+	auto domain_rail_0 = rdma_domain_get_rail(domain, 0);
+	return domain_rail_0->domain;
+};
+
+
+static inline struct fid_cq *rdma_domain_get_ofi_cq(nccl_net_ofi_domain_t *base_domain) {
+	auto domain = reinterpret_cast<nccl_net_ofi_rdma_domain_t *>(base_domain);
+	auto domain_rail_0 = rdma_domain_get_rail(domain, 0);
+	return domain_rail_0->cq;
+};
+
+
 static nccl_net_ofi_domain_t *nccl_net_ofi_rdma_device_create_domain(nccl_net_ofi_device_t *base_dev)
 {
 	int ret = 0;
@@ -7488,6 +7502,8 @@ static nccl_net_ofi_domain_t *nccl_net_ofi_rdma_device_create_domain(nccl_net_of
 
 	domain->base.free = nccl_net_ofi_rdma_domain_free;
 	domain->base.create_endpoint = nccl_net_ofi_rdma_domain_create_endpoint;
+	domain->base.get_ofi_domain = rdma_domain_get_ofi_domain;
+	domain->base.get_ofi_cq = rdma_domain_get_ofi_cq;
 
 	domain->num_rails = device->num_rails;
 
@@ -7734,6 +7750,13 @@ nccl_net_ofi_rdma_device_release(nccl_net_ofi_device_t *base_device)
 }
 
 
+static inline struct fi_info *rdma_device_get_ofi_info(nccl_net_ofi_device_t *dev) {
+	auto device = reinterpret_cast<nccl_net_ofi_rdma_device_t *>(dev);
+	auto device_rail_0 = rdma_device_get_rail(device, 0);
+	return device_rail_0->info;
+};
+
+
 /**
  * Create an rdma device object
  */
@@ -7760,6 +7783,7 @@ static nccl_net_ofi_rdma_device_t *nccl_net_ofi_rdma_device_create(
 	device->base.get_mr_key = get_mr_key;
 	device->base.release = nccl_net_ofi_rdma_device_release;
 	device->base.create_domain = nccl_net_ofi_rdma_device_create_domain;
+	device->base.get_ofi_info = rdma_device_get_ofi_info;
 
 	/* at this point, we can safely call the destructor to clean
 	 * up */
diff --git a/src/nccl_ofi_sendrecv.cpp b/src/nccl_ofi_sendrecv.cpp
index 483db36e70..850cda8714 100644
--- a/src/nccl_ofi_sendrecv.cpp
+++ b/src/nccl_ofi_sendrecv.cpp
@@ -2447,6 +2447,18 @@ static int nccl_net_ofi_sendrecv_domain_free(nccl_net_ofi_domain_t *base_domain)
 }
 
 
+static inline struct fid_domain *sendrecv_domain_get_ofi_domain(nccl_net_ofi_domain_t *base_domain)
+{
+	return reinterpret_cast<nccl_net_ofi_sendrecv_domain_t *>(base_domain)->domain;
+}
+
+
+static inline struct fid_cq *sendrecv_domain_get_ofi_cq(nccl_net_ofi_domain_t *base_domain)
+{
+	return reinterpret_cast<nccl_net_ofi_sendrecv_domain_t *>(base_domain)->cq;
+}
+
+
 static nccl_net_ofi_domain_t *nccl_net_ofi_sendrecv_device_create_domain(nccl_net_ofi_device_t *base_device)
 {
 	int ret;
@@ -2461,6 +2473,8 @@ static nccl_net_ofi_domain_t *nccl_net_ofi_sendrecv_device_create_domain(nccl_ne
 
 	domain->base.free = nccl_net_ofi_sendrecv_domain_free;
 	domain->base.create_endpoint = nccl_net_ofi_sendrecv_domain_create_endpoint;
+	domain->base.get_ofi_domain = sendrecv_domain_get_ofi_domain;
+	domain->base.get_ofi_cq = sendrecv_domain_get_ofi_cq;
 
 	ret = nccl_net_ofi_domain_init(base_device, &domain->base);
 	if (ret != 0) {
@@ -2575,6 +2589,13 @@ nccl_net_ofi_sendrecv_device_release(nccl_net_ofi_device_t *base_device)
 	return 0;
 }
 
+
+static inline struct fi_info *sendrecv_device_get_ofi_info(nccl_net_ofi_device_t *device)
+{
+	return reinterpret_cast<nccl_net_ofi_sendrecv_device_t *>(device)->info;
+}
+
+
 /**
  * Create a sendrecv device object
  */
@@ -2603,6 +2624,7 @@ nccl_net_ofi_sendrecv_device_create(nccl_net_ofi_plugin_t *plugin,
 	device->base.release = nccl_net_ofi_sendrecv_device_release;
 	device->base.get_mr_key = NULL;
 	device->base.create_domain = nccl_net_ofi_sendrecv_device_create_domain;
+	device->base.get_ofi_info = sendrecv_device_get_ofi_info;
 
 	/* at this point, we can safely call the destructor to clean
 	 * up */


Reviews:
user: login: avivbenchorin
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-06T20:57:09Z
user: login: rauteric
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-06T21:22:39Z
user: login: rauteric
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-06T21:25:46Z
user: login: rauteric
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-06T21:28:36Z
user: login: rauteric
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-07T21:14:16Z
user: login: rauteric
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-07T21:34:20Z
user: login: rauteric
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-07T21:34:26Z
user: login: rauteric
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-07T21:40:49Z
user: login: avivbenchorin
state: DISMISSED
author_association: CONTRIBUTOR
submitted_at: 2025-05-07T22:40:49Z
user: login: mozarhua
state: DISMISSED
author_association: CONTRIBUTOR
submitted_at: 2025-05-09T17:39:41Z
user: login: bwbarrett
state: DISMISSED
author_association: CONTRIBUTOR
submitted_at: 2025-05-09T21:38:52Z
user: login: rauteric
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-21T00:34:50Z
user: login: avivbenchorin
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-21T19:20:10Z
user: login: rauteric
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-21T19:29:21Z
user: login: avivbenchorin
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-05-21T20:27:52Z
user: login: bwbarrett
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-05-22T21:07:26Z

Review comments:
diff_hunk: @@ -389,6 +396,21 @@ struct nccl_net_ofi_domain {
 
 	pthread_mutex_t domain_lock;
 
+	/**
+	 * Retrieve an fid_domain object associated with this domain. There may
+	 * be more than one fid_domain per domain, depending on the transport;
+	 * in that case, this will be the domain object associated with the
+	 * "leader NIC"
+	 */
+	fid_domain *(*get_ofi_domain)(nccl_net_ofi_domain_t *domain);
+
+	/**
+	 * Retrieve an fid_cq object associated with this domain. There may be
+	 * more than one cq per domain, depending on the transport; in that
+	 * case, this will be the info object associated with the "leader NIC"
+	 */
+	fid_cq *(*get_cq)(nccl_net_ofi_domain_t *domain);
path: include/nccl_ofi.h
user: login: avivbenchorin
body: Should this be `struct fid_domain *` and `struct fid_cq *`?
created_at: 2025-05-06T20:50:14Z
author_association: CONTRIBUTOR
diff_hunk: @@ -7422,6 +7422,16 @@ static nccl_net_ofi_domain_t *nccl_net_ofi_rdma_device_create_domain(nccl_net_of
 
 	domain->base.free = nccl_net_ofi_rdma_domain_free;
 	domain->base.create_endpoint = nccl_net_ofi_rdma_domain_create_endpoint;
+	domain->base.get_ofi_domain = [](nccl_net_ofi_domain_t *base_domain) {
path: src/nccl_ofi_rdma.cpp
user: login: avivbenchorin
body: Is there a specific reason you chose to use lambda functions compared to defining separate functions for the getters?
created_at: 2025-05-06T20:51:07Z
author_association: CONTRIBUTOR
diff_hunk: @@ -7422,6 +7422,16 @@ static nccl_net_ofi_domain_t *nccl_net_ofi_rdma_device_create_domain(nccl_net_of
 
 	domain->base.free = nccl_net_ofi_rdma_domain_free;
 	domain->base.create_endpoint = nccl_net_ofi_rdma_domain_create_endpoint;
+	domain->base.get_ofi_domain = [](nccl_net_ofi_domain_t *base_domain) {
+		auto _domain = reinterpret_cast<nccl_net_ofi_rdma_domain_t *>(base_domain);
path: src/nccl_ofi_rdma.cpp
user: login: avivbenchorin
body: I personally am not a fan of the leading underscore in the naming here (since to me it signifies being a private member, which doesn't make sense in this context). Any reason we couldn't just name it `domain` (or `device` instead of `_device` in the `device->base.get_info` function)?
created_at: 2025-05-06T20:56:58Z
author_association: CONTRIBUTOR
diff_hunk: @@ -7422,6 +7422,16 @@ static nccl_net_ofi_domain_t *nccl_net_ofi_rdma_device_create_domain(nccl_net_of
 
 	domain->base.free = nccl_net_ofi_rdma_domain_free;
 	domain->base.create_endpoint = nccl_net_ofi_rdma_domain_create_endpoint;
+	domain->base.get_ofi_domain = [](nccl_net_ofi_domain_t *base_domain) {
+		auto _domain = reinterpret_cast<nccl_net_ofi_rdma_domain_t *>(base_domain);
path: src/nccl_ofi_rdma.cpp
user: login: rauteric
body: Yeah, I hate the leading underscore as well. But without the leading underscore, the compiler warns about a local variable shadowing an outer variable. I thought the leading underscore was better than trying to come up with unique names for the parameters (maybe something like domain_inner).

I had to use this style in the CM code as well.
created_at: 2025-05-06T21:22:39Z
author_association: CONTRIBUTOR
diff_hunk: @@ -389,6 +396,21 @@ struct nccl_net_ofi_domain {
 
 	pthread_mutex_t domain_lock;
 
+	/**
+	 * Retrieve an fid_domain object associated with this domain. There may
+	 * be more than one fid_domain per domain, depending on the transport;
+	 * in that case, this will be the domain object associated with the
+	 * "leader NIC"
+	 */
+	fid_domain *(*get_ofi_domain)(nccl_net_ofi_domain_t *domain);
+
+	/**
+	 * Retrieve an fid_cq object associated with this domain. There may be
+	 * more than one cq per domain, depending on the transport; in that
+	 * case, this will be the info object associated with the "leader NIC"
+	 */
+	fid_cq *(*get_cq)(nccl_net_ofi_domain_t *domain);
path: include/nccl_ofi.h
user: login: rauteric
body: > Should this be `struct fid_domain *` and `struct fid_cq *`?

Maybe. C++ doesn't enforce needing to put `struct` in front of everything, but it could be argued that we should to be consistent with the rest of the code. (Or, we gradually migrate the rest of the code toward not using the `struct` prefix.)
created_at: 2025-05-06T21:25:46Z
author_association: CONTRIBUTOR
diff_hunk: @@ -7422,6 +7422,16 @@ static nccl_net_ofi_domain_t *nccl_net_ofi_rdma_device_create_domain(nccl_net_of
 
 	domain->base.free = nccl_net_ofi_rdma_domain_free;
 	domain->base.create_endpoint = nccl_net_ofi_rdma_domain_create_endpoint;
+	domain->base.get_ofi_domain = [](nccl_net_ofi_domain_t *base_domain) {
path: src/nccl_ofi_rdma.cpp
user: login: rauteric
body: > Is there a specific reason you chose to use lambda functions compared to defining separate functions for the getters?

No strong opinion, but I think a lambda is more readable here than defining a separate one-line function. `nccl_ofi_rdma.cpp` is long enough as is 🙃 
created_at: 2025-05-06T21:28:35Z
author_association: CONTRIBUTOR
diff_hunk: @@ -7422,6 +7422,16 @@ static nccl_net_ofi_domain_t *nccl_net_ofi_rdma_device_create_domain(nccl_net_of
 
 	domain->base.free = nccl_net_ofi_rdma_domain_free;
 	domain->base.create_endpoint = nccl_net_ofi_rdma_domain_create_endpoint;
+	domain->base.get_ofi_domain = [](nccl_net_ofi_domain_t *base_domain) {
path: src/nccl_ofi_rdma.cpp
user: login: rauteric
body: Talked to Aviv, eventually (after C++ refactoring) this is going to be a class method, so it makes sense to use a real function here and not a lambda.
created_at: 2025-05-07T21:14:16Z
author_association: CONTRIBUTOR
diff_hunk: @@ -7422,6 +7422,16 @@ static nccl_net_ofi_domain_t *nccl_net_ofi_rdma_device_create_domain(nccl_net_of
 
 	domain->base.free = nccl_net_ofi_rdma_domain_free;
 	domain->base.create_endpoint = nccl_net_ofi_rdma_domain_create_endpoint;
+	domain->base.get_ofi_domain = [](nccl_net_ofi_domain_t *base_domain) {
+		auto _domain = reinterpret_cast<nccl_net_ofi_rdma_domain_t *>(base_domain);
path: src/nccl_ofi_rdma.cpp
user: login: rauteric
body: No longer relevant with the split to separate function.
created_at: 2025-05-07T21:34:20Z
author_association: CONTRIBUTOR
diff_hunk: @@ -7422,6 +7422,16 @@ static nccl_net_ofi_domain_t *nccl_net_ofi_rdma_device_create_domain(nccl_net_of
 
 	domain->base.free = nccl_net_ofi_rdma_domain_free;
 	domain->base.create_endpoint = nccl_net_ofi_rdma_domain_create_endpoint;
+	domain->base.get_ofi_domain = [](nccl_net_ofi_domain_t *base_domain) {
path: src/nccl_ofi_rdma.cpp
user: login: rauteric
body: Done
created_at: 2025-05-07T21:34:26Z
author_association: CONTRIBUTOR
diff_hunk: @@ -389,6 +396,21 @@ struct nccl_net_ofi_domain {
 
 	pthread_mutex_t domain_lock;
 
+	/**
+	 * Retrieve an fid_domain object associated with this domain. There may
+	 * be more than one fid_domain per domain, depending on the transport;
+	 * in that case, this will be the domain object associated with the
+	 * "leader NIC"
+	 */
+	fid_domain *(*get_ofi_domain)(nccl_net_ofi_domain_t *domain);
+
+	/**
+	 * Retrieve an fid_cq object associated with this domain. There may be
+	 * more than one cq per domain, depending on the transport; in that
+	 * case, this will be the info object associated with the "leader NIC"
+	 */
+	fid_cq *(*get_cq)(nccl_net_ofi_domain_t *domain);
path: include/nccl_ofi.h
user: login: rauteric
body: Put `struct` prefix in front of Libfabric objects.
created_at: 2025-05-07T21:40:49Z
author_association: CONTRIBUTOR
diff_hunk: @@ -300,6 +300,13 @@ struct nccl_net_ofi_device {
 	 */
 	nccl_net_ofi_domain_t *(*get_domain)(nccl_net_ofi_device_t *dev);
 
+	/**
+	 * Retrieve an fi_info object associated with this device. There may be
+	 * more than one info per device, depending on the transport; in that
+	 * case, this will be the info object associated with the "leader NIC"
+	 */
+	struct fi_info *(*get_info)(nccl_net_ofi_device_t *dev);
path: include/nccl_ofi.h
user: login: bwbarrett
body: Similar to domain, this is going to end up confusing (get_info / get_properties are way too close).  Let's be consistent and just call this get_ofi_info().  Similar with get_cq below -- just be consistent in naming.
created_at: 2025-05-09T21:37:11Z
author_association: CONTRIBUTOR
diff_hunk: @@ -300,6 +300,13 @@ struct nccl_net_ofi_device {
 	 */
 	nccl_net_ofi_domain_t *(*get_domain)(nccl_net_ofi_device_t *dev);
 
+	/**
+	 * Retrieve an fi_info object associated with this device. There may be
+	 * more than one info per device, depending on the transport; in that
+	 * case, this will be the info object associated with the "leader NIC"
+	 */
+	struct fi_info *(*get_info)(nccl_net_ofi_device_t *dev);
path: include/nccl_ofi.h
user: login: rauteric
body: Done
created_at: 2025-05-21T00:34:50Z
author_association: CONTRIBUTOR
diff_hunk: @@ -7734,6 +7750,13 @@ nccl_net_ofi_rdma_device_release(nccl_net_ofi_device_t *base_device)
 }
 
 
+static inline struct fi_info *device_get_info(nccl_net_ofi_device_t *dev) {
path: src/nccl_ofi_rdma.cpp
user: login: avivbenchorin
body: Would you also want to rename `device_get_info`/`domain_get_cq`/`sendrecv_domain_get_cq`/`sendrecv_device_get_info` to match that change to the base function pointer to specify `ofi_info` and `ofi_cq`? (you already made the change for `domain_get_ofi_domain` and `sendrecv_domain_get_ofi_domain`)
created_at: 2025-05-21T19:19:54Z
author_association: CONTRIBUTOR
diff_hunk: @@ -7734,6 +7750,13 @@ nccl_net_ofi_rdma_device_release(nccl_net_ofi_device_t *base_device)
 }
 
 
+static inline struct fi_info *device_get_info(nccl_net_ofi_device_t *dev) {
path: src/nccl_ofi_rdma.cpp
user: login: rauteric
body: Hopefully cleaned that all up now.
created_at: 2025-05-21T19:29:21Z
author_association: CONTRIBUTOR

PR comments:
user: login: rauteric
created_at: 2025-05-02T23:24:54Z
body: Note: the first commit in this PR (refactor completion queue to domain) is from https://github.com/aws/aws-ofi-nccl/pull/869.
author_association: CONTRIBUTOR
user: login: rauteric
created_at: 2025-05-07T21:33:11Z
body: Force-push to [d963716](https://github.com/aws/aws-ofi-nccl/commit/d9637169f5d0b19ba9c53eb5dc753d1f2727ca4b) is a rebase on master.
author_association: CONTRIBUTOR
user: login: rauteric
created_at: 2025-05-07T21:34:02Z
body: Force-push to [3e22f46](https://github.com/aws/aws-ofi-nccl/commit/3e22f46b94f5fef7d22963e973dcfd0cfc75ff46) addresses feedback.
author_association: CONTRIBUTOR