Repository: linux-rdma/rdma-core
Pull request #1580: RDMA FLUSH and ATOMIC WRITE with ODP
Author: login: ddmatsu
Created: 2025-03-14T07:56:15Z | Merged: 2025-04-27T10:54:52Z

RDMA FLUSH[1] and ATOMIC WRITE[2] were added to rxe, but they cannot run
in the ODP mode as of now. This series is for the userspace enablement.

The former three patches correspond to an uapi change in the kernel.
(**v1/merged**) https://lore.kernel.org/linux-rdma/20250418051345.1022339-1-matsuda-daisuke@fujitsu.com/T/#t

The latter four add definitions of ODP capabilities and rdma-core testcases for the new features.
For kernel-side enablement in rxe, please see:
(**Updated v3/merged**) https://lore.kernel.org/linux-rdma/20250324075649.3313968-1-matsuda-daisuke@fujitsu.com/T/#t

[1] [for-next PATCH 00/10] RDMA/rxe: Add RDMA FLUSH operation
https://lore.kernel.org/lkml/20221206130201.30986-1-lizhijian@fujitsu.com/

[2] [PATCH v7 0/8] RDMA/rxe: Add atomic write operation
https://lore.kernel.org/linux-rdma/1669905432-14-1-git-send-email-yangx.jy@fujitsu.com/


Diff:
diff --git a/kernel-headers/rdma/ib_user_verbs.h b/kernel-headers/rdma/ib_user_verbs.h
index e16650f0c..3b7bd9981 100644
--- a/kernel-headers/rdma/ib_user_verbs.h
+++ b/kernel-headers/rdma/ib_user_verbs.h
@@ -233,6 +233,22 @@ struct ib_uverbs_ex_query_device {
 	__u32 reserved;
 };
 
+enum ib_uverbs_odp_general_cap_bits {
+	IB_UVERBS_ODP_SUPPORT          = 1 << 0,
+	IB_UVERBS_ODP_SUPPORT_IMPLICIT = 1 << 1,
+};
+
+enum ib_uverbs_odp_transport_cap_bits {
+	IB_UVERBS_ODP_SUPPORT_SEND     = 1 << 0,
+	IB_UVERBS_ODP_SUPPORT_RECV     = 1 << 1,
+	IB_UVERBS_ODP_SUPPORT_WRITE    = 1 << 2,
+	IB_UVERBS_ODP_SUPPORT_READ     = 1 << 3,
+	IB_UVERBS_ODP_SUPPORT_ATOMIC   = 1 << 4,
+	IB_UVERBS_ODP_SUPPORT_SRQ_RECV = 1 << 5,
+	IB_UVERBS_ODP_SUPPORT_FLUSH    = 1 << 6,
+	IB_UVERBS_ODP_SUPPORT_ATOMIC_WRITE     = 1 << 7,
+};
+
 struct ib_uverbs_odp_caps {
 	__aligned_u64 general_caps;
 	struct {
diff --git a/libibverbs/examples/devinfo.c b/libibverbs/examples/devinfo.c
index 0b375c2ff..c245b1f28 100644
--- a/libibverbs/examples/devinfo.c
+++ b/libibverbs/examples/devinfo.c
@@ -330,7 +330,9 @@ static void print_odp_trans_caps(uint32_t trans)
 					    IBV_ODP_SUPPORT_WRITE |
 					    IBV_ODP_SUPPORT_READ |
 					    IBV_ODP_SUPPORT_ATOMIC |
-					    IBV_ODP_SUPPORT_SRQ_RECV);
+					    IBV_ODP_SUPPORT_SRQ_RECV |
+					    IBV_ODP_SUPPORT_FLUSH |
+					    IBV_ODP_SUPPORT_ATOMIC_WRITE);
 
 	if (!trans) {
 		printf("\t\t\t\t\tNO SUPPORT\n");
@@ -347,6 +349,10 @@ static void print_odp_trans_caps(uint32_t trans)
 			printf("\t\t\t\t\tSUPPORT_ATOMIC\n");
 		if (trans & IBV_ODP_SUPPORT_SRQ_RECV)
 			printf("\t\t\t\t\tSUPPORT_SRQ\n");
+		if (trans & IBV_ODP_SUPPORT_FLUSH)
+			printf("\t\t\t\t\tSUPPORT_FLUSH\n");
+		if (trans & IBV_ODP_SUPPORT_ATOMIC_WRITE)
+			printf("\t\t\t\t\tSUPPORT_ATOMIC_WRITE\n");
 		if (trans & unknown_transport_caps)
 			printf("\t\t\t\t\tUnknown flags: 0x%" PRIX32 "\n",
 			       trans & unknown_transport_caps);
diff --git a/libibverbs/man/ibv_query_device_ex.3 b/libibverbs/man/ibv_query_device_ex.3
index 6db8944d6..c77e8b4f8 100644
--- a/libibverbs/man/ibv_query_device_ex.3
+++ b/libibverbs/man/ibv_query_device_ex.3
@@ -68,6 +68,8 @@ enum ibv_odp_transport_cap_bits {
         IBV_ODP_SUPPORT_READ     = 1 << 3, /* RDMA-Read operations support on-demand paging */
         IBV_ODP_SUPPORT_ATOMIC   = 1 << 4, /* RDMA-Atomic operations support on-demand paging */
         IBV_ODP_SUPPORT_SRQ_RECV = 1 << 5, /* SRQ receive operations support on-demand paging */
+        IBV_ODP_SUPPORT_FLUSH    = 1 << 6, /* RDMA-Flush operations support on-demand paging */
+        IBV_ODP_SUPPORT_ATOMIC_WRITE     = 1 << 7, /* RDMA-Atomic-write operations support on-demand paing */
 };
 
 struct ibv_tso_caps {
diff --git a/libibverbs/verbs.h b/libibverbs/verbs.h
index 30f426d7e..ae5995706 100644
--- a/libibverbs/verbs.h
+++ b/libibverbs/verbs.h
@@ -228,6 +228,11 @@ struct ibv_query_device_ex_input {
 	uint32_t		comp_mask;
 };
 
+enum ibv_odp_general_caps {
+	IBV_ODP_SUPPORT = 1 << 0,
+	IBV_ODP_SUPPORT_IMPLICIT = 1 << 1,
+};
+
 enum ibv_odp_transport_cap_bits {
 	IBV_ODP_SUPPORT_SEND     = 1 << 0,
 	IBV_ODP_SUPPORT_RECV     = 1 << 1,
@@ -235,6 +240,8 @@ enum ibv_odp_transport_cap_bits {
 	IBV_ODP_SUPPORT_READ     = 1 << 3,
 	IBV_ODP_SUPPORT_ATOMIC   = 1 << 4,
 	IBV_ODP_SUPPORT_SRQ_RECV = 1 << 5,
+	IBV_ODP_SUPPORT_FLUSH    = 1 << 6,
+	IBV_ODP_SUPPORT_ATOMIC_WRITE = 1 << 7,
 };
 
 struct ibv_odp_caps {
@@ -246,11 +253,6 @@ struct ibv_odp_caps {
 	} per_transport_caps;
 };
 
-enum ibv_odp_general_caps {
-	IBV_ODP_SUPPORT = 1 << 0,
-	IBV_ODP_SUPPORT_IMPLICIT = 1 << 1,
-};
-
 struct ibv_tso_caps {
 	uint32_t max_tso;
 	uint32_t supported_qpts;
diff --git a/pyverbs/device.pyx b/pyverbs/device.pyx
index 23f84b0fd..82baefcb6 100644
--- a/pyverbs/device.pyx
+++ b/pyverbs/device.pyx
@@ -567,7 +567,9 @@ cdef class ODPCaps(PyverbsObject):
              e.IBV_ODP_SUPPORT_WRITE: 'IBV_ODP_SUPPORT_WRITE',
              e.IBV_ODP_SUPPORT_READ: 'IBV_ODP_SUPPORT_READ',
              e.IBV_ODP_SUPPORT_ATOMIC: 'IBV_ODP_SUPPORT_ATOMIC',
-             e.IBV_ODP_SUPPORT_SRQ_RECV: 'IBV_ODP_SUPPORT_SRQ_RECV'}
+             e.IBV_ODP_SUPPORT_SRQ_RECV: 'IBV_ODP_SUPPORT_SRQ_RECV',
+             e.IBV_ODP_SUPPORT_FLUSH: 'IBV_ODP_SUPPORT_FLUSH',
+             e.IBV_ODP_SUPPORT_ATOMIC_WRITE: 'IBV_ODP_SUPPORT_ATOMIC_WRITE'}
 
         print_format = '{}: {}\n'
         return print_format.format('ODP General caps', str_from_flags(self.general_caps, general_caps)) +\
diff --git a/pyverbs/libibverbs_enums.pxd b/pyverbs/libibverbs_enums.pxd
index 17bb3ef32..692396421 100644
--- a/pyverbs/libibverbs_enums.pxd
+++ b/pyverbs/libibverbs_enums.pxd
@@ -417,6 +417,8 @@ cdef extern from '<infiniband/verbs.h>':
         IBV_ODP_SUPPORT_READ
         IBV_ODP_SUPPORT_ATOMIC
         IBV_ODP_SUPPORT_SRQ_RECV
+        IBV_ODP_SUPPORT_FLUSH
+        IBV_ODP_SUPPORT_ATOMIC_WRITE
 
     cpdef enum ibv_device_cap_flags:
         IBV_DEVICE_RESIZE_MAX_WR
diff --git a/tests/test_odp.py b/tests/test_odp.py
index 6d891f11f..a9885298c 100644
--- a/tests/test_odp.py
+++ b/tests/test_odp.py
@@ -7,6 +7,7 @@
 from pyverbs.mr import MR
 import pyverbs.enums as e
 import tests.utils as u
+import unittest
 
 HUGE_PAGE_SIZE = 0x200000
 
@@ -112,6 +113,56 @@ def create_mr(self):
                                   flags=MAP_ANONYMOUS_| MAP_PRIVATE_)
         self.mr = u.create_custom_mr(self, e.IBV_ACCESS_ON_DEMAND, user_addr=self.user_addr)
 
+class OdpQpExRC(RCResources):
+    def __init__(self, dev_name, ib_port, gid_index, is_huge=False,
+                 request_user_addr=False, use_mr_prefetch=None, is_implicit=False,
+                 prefetch_advice=e._IBV_ADVISE_MR_ADVICE_PREFETCH_WRITE,
+                 msg_size=8, odp_caps=e.IBV_ODP_SUPPORT_SEND | e.IBV_ODP_SUPPORT_RECV,
+                 use_mixed_mr=False):
+
+        ''' For object descriptions, refer to OdpRC class '''
+        self.request_user_addr = request_user_addr
+        self.is_implicit = is_implicit
+        self.odp_caps = odp_caps
+        self.access = e.IBV_ACCESS_LOCAL_WRITE | e.IBV_ACCESS_ON_DEMAND | \
+            e.IBV_ACCESS_REMOTE_ATOMIC | e.IBV_ACCESS_REMOTE_READ | \
+            e.IBV_ACCESS_REMOTE_WRITE
+        self.user_addr = None
+        super(OdpQpExRC, self).__init__(dev_name=dev_name, ib_port=ib_port,
+                                         gid_index=gid_index)
+        self.msg_size = msg_size
+
+        if self.odp_caps & e.IBV_ODP_SUPPORT_FLUSH:
+            self.ptype = e.IBV_FLUSH_GLOBAL
+            self.level = e.IBV_FLUSH_RANGE
+
+    def create_qps(self):
+        if self.odp_caps & e.IBV_ODP_SUPPORT_ATOMIC_WRITE:
+            u.create_qp_ex(self, e.IBV_QPT_RC, e.IBV_QP_EX_WITH_ATOMIC_WRITE)
+        elif self.odp_caps & e.IBV_ODP_SUPPORT_FLUSH:
+            u.create_qp_ex(self, e.IBV_QPT_RC, e.IBV_QP_EX_WITH_FLUSH | e.IBV_QP_EX_WITH_RDMA_WRITE)
+        else:
+            raise unittest.SkipTest('There is no qpex test for the specified ODP caps.')
+
+    def create_mr(self):
+        u.odp_supported(self.ctx, 'rc', self.odp_caps)
+        if self.odp_caps & e.IBV_ODP_SUPPORT_ATOMIC_WRITE:
+            access = self.access
+            if self.request_user_addr:
+                mmap_flags = MAP_ANONYMOUS_| MAP_PRIVATE_
+                length = self.msg_size
+                self.user_addr = mmap(length=length, flags=mmap_flags)
+            self.mr = MR(self.pd, self.msg_size, access, address=self.user_addr,
+                         implicit=self.is_implicit)
+        elif self.odp_caps & e.IBV_ODP_SUPPORT_FLUSH:
+            try:
+                self.mr = u.create_custom_mr(self, e.IBV_ACCESS_FLUSH_GLOBAL | e.IBV_ACCESS_REMOTE_WRITE | e.IBV_ACCESS_ON_DEMAND)
+            except PyverbsRDMAError as ex:
+                if ex.error_code == errno.EINVAL:
+                    raise unittest.SkipTest('Create mr with IBV_ACCESS_FLUSH_GLOBAL access flag is not supported in kernel')
+                raise ex
+        else:
+            raise unittest.SkipTest('There is no qpex test for the specified ODP caps.')
 
 class OdpTestCase(RDMATestCase):
     def setUp(self):
@@ -150,6 +201,28 @@ def test_odp_rc_mixed_mr(self):
                             use_mixed_mr=True)
         u.traffic(**self.traffic_args)
 
+    def test_odp_qp_ex_rc_atomic_write(self):
+        super().create_players(OdpQpExRC, request_user_addr=self.force_page_faults,
+                               msg_size=8, odp_caps=e.IBV_ODP_SUPPORT_ATOMIC_WRITE)
+        self.client.msg_size = 8
+        self.server.msg_size = 8
+        u.rdma_traffic(**self.traffic_args,
+                       new_send=True, send_op=e.IBV_WR_ATOMIC_WRITE)
+
+    def test_odp_qp_ex_rc_flush(self):
+        super().create_players(OdpQpExRC, request_user_addr=self.force_page_faults,
+                               odp_caps=e.IBV_ODP_SUPPORT_FLUSH)
+        wcs = u.flush_traffic(**self.traffic_args, new_send=True,
+                              send_op=e.IBV_WR_FLUSH)
+        if wcs[0].status != e.IBV_WC_SUCCESS:
+            raise PyverbsError(f'Unexpected {wc_status_to_str(wcs[0].status)}')
+
+        self.client.level = e.IBV_FLUSH_MR
+        wcs = u.flush_traffic(**self.traffic_args, new_send=True,
+                              send_op=e.IBV_WR_FLUSH)
+        if wcs[0].status != e.IBV_WC_SUCCESS:
+            raise PyverbsError(f'Unexpected {wc_status_to_str(wcs[0].status)}')
+
     def test_odp_rc_atomic_cmp_and_swp(self):
         self.create_players(OdpRC, request_user_addr=self.force_page_faults,
                             msg_size=8, odp_caps=e.IBV_ODP_SUPPORT_ATOMIC)
diff --git a/tests/test_qpex.py b/tests/test_qpex.py
index 21906bfb4..899b0519e 100644
--- a/tests/test_qpex.py
+++ b/tests/test_qpex.py
@@ -12,51 +12,14 @@
 from tests.base import UDResources, RCResources, RDMATestCase, XRCResources
 import tests.utils as u
 
-
-def create_qp_ex(agr_obj, qp_type, send_flags):
-    if qp_type == e.IBV_QPT_XRC_SEND:
-        cap = QPCap(max_send_wr=agr_obj.num_msgs, max_recv_wr=0, max_recv_sge=0,
-                    max_send_sge=1)
-    else:
-        cap = QPCap(max_send_wr=agr_obj.num_msgs, max_recv_wr=agr_obj.num_msgs,
-                    max_recv_sge=1, max_send_sge=1)
-    qia = QPInitAttrEx(cap=cap, qp_type=qp_type, scq=agr_obj.cq,
-                       rcq=agr_obj.cq, pd=agr_obj.pd, send_ops_flags=send_flags,
-                       comp_mask=e.IBV_QP_INIT_ATTR_PD |
-                                 e.IBV_QP_INIT_ATTR_SEND_OPS_FLAGS)
-    qp_attr = QPAttr(port_num=agr_obj.ib_port)
-    if qp_type == e.IBV_QPT_UD:
-        qp_attr.qkey = agr_obj.UD_QKEY
-        qp_attr.pkey_index = agr_obj.UD_PKEY_INDEX
-    if qp_type == e.IBV_QPT_RC:
-        qp_attr.qp_access_flags = e.IBV_ACCESS_REMOTE_WRITE | \
-                                  e.IBV_ACCESS_REMOTE_READ | \
-                                  e.IBV_ACCESS_REMOTE_ATOMIC | \
-                                  e.IBV_ACCESS_FLUSH_GLOBAL | \
-                                  e.IBV_ACCESS_FLUSH_PERSISTENT
-    try:
-        # We don't have capability bits for this
-        qp = QPEx(agr_obj.ctx, qia, qp_attr)
-    except PyverbsRDMAError as ex:
-        if ex.error_code == errno.EOPNOTSUPP:
-            raise unittest.SkipTest('Extended QP is not supported on this device')
-        raise ex
-    if qp_type != e.IBV_QPT_XRC_SEND:
-        agr_obj.qps.append(qp)
-        agr_obj.qps_num.append(qp.qp_num)
-        agr_obj.psns.append(random.getrandbits(24))
-    else:
-        return qp
-
-
 class QpExUDSend(UDResources):
     def create_qps(self):
-        create_qp_ex(self, e.IBV_QPT_UD, e.IBV_QP_EX_WITH_SEND)
+        u.create_qp_ex(self, e.IBV_QPT_UD, e.IBV_QP_EX_WITH_SEND)
 
 
 class QpExRCSend(RCResources):
     def create_qps(self):
-        create_qp_ex(self, e.IBV_QPT_RC, e.IBV_QP_EX_WITH_SEND)
+        u.create_qp_ex(self, e.IBV_QPT_RC, e.IBV_QP_EX_WITH_SEND)
 
 
 class QpExXRCSend(XRCResources):
@@ -72,7 +35,7 @@ def create_qps(self):
             recv_qp = QP(self.ctx, attr_ex, qp_attr)
             self.rqp_lst.append(recv_qp)
 
-            send_qp = create_qp_ex(self, e.IBV_QPT_XRC_SEND, e.IBV_QP_EX_WITH_SEND)
+            send_qp = u.create_qp_ex(self, e.IBV_QPT_XRC_SEND, e.IBV_QP_EX_WITH_SEND)
             self.sqp_lst.append(send_qp)
             self.qps_num.append((recv_qp.qp_num, send_qp.qp_num))
             self.psns.append(random.getrandbits(24))
@@ -80,12 +43,12 @@ def create_qps(self):
 
 class QpExUDSendImm(UDResources):
     def create_qps(self):
-        create_qp_ex(self, e.IBV_QPT_UD, e.IBV_QP_EX_WITH_SEND_WITH_IMM)
+        u.create_qp_ex(self, e.IBV_QPT_UD, e.IBV_QP_EX_WITH_SEND_WITH_IMM)
 
 
 class QpExRCSendImm(RCResources):
     def create_qps(self):
-        create_qp_ex(self, e.IBV_QPT_RC, e.IBV_QP_EX_WITH_SEND_WITH_IMM)
+        u.create_qp_ex(self, e.IBV_QPT_RC, e.IBV_QP_EX_WITH_SEND_WITH_IMM)
 
 
 class QpExXRCSendImm(XRCResources):
@@ -101,7 +64,7 @@ def create_qps(self):
             recv_qp = QP(self.ctx, attr_ex, qp_attr)
             self.rqp_lst.append(recv_qp)
 
-            send_qp = create_qp_ex(self, e.IBV_QPT_XRC_SEND,
+            send_qp = u.create_qp_ex(self, e.IBV_QPT_XRC_SEND,
                                    e.IBV_QP_EX_WITH_SEND_WITH_IMM)
             self.sqp_lst.append(send_qp)
             self.qps_num.append((recv_qp.qp_num, send_qp.qp_num))
@@ -112,7 +75,7 @@ class QpExRCFlush(RCResources):
     ptype = e.IBV_FLUSH_GLOBAL
     level = e.IBV_FLUSH_RANGE
     def create_qps(self):
-        create_qp_ex(self, e.IBV_QPT_RC, e.IBV_QP_EX_WITH_FLUSH | e.IBV_QP_EX_WITH_RDMA_WRITE)
+        u.create_qp_ex(self, e.IBV_QPT_RC, e.IBV_QP_EX_WITH_FLUSH | e.IBV_QP_EX_WITH_RDMA_WRITE)
 
     def create_mr(self):
         try:
@@ -125,7 +88,7 @@ def create_mr(self):
 
 class QpExRCAtomicWrite(RCResources):
     def create_qps(self):
-        create_qp_ex(self, e.IBV_QPT_RC, e.IBV_QP_EX_WITH_ATOMIC_WRITE)
+        u.create_qp_ex(self, e.IBV_QPT_RC, e.IBV_QP_EX_WITH_ATOMIC_WRITE)
 
     def create_mr(self):
         self.mr = u.create_custom_mr(self, e.IBV_ACCESS_REMOTE_WRITE)
@@ -133,7 +96,7 @@ def create_mr(self):
 
 class QpExRCRDMAWrite(RCResources):
     def create_qps(self):
-        create_qp_ex(self, e.IBV_QPT_RC, e.IBV_QP_EX_WITH_RDMA_WRITE)
+        u.create_qp_ex(self, e.IBV_QPT_RC, e.IBV_QP_EX_WITH_RDMA_WRITE)
 
     def create_mr(self):
         self.mr = u.create_custom_mr(self, e.IBV_ACCESS_REMOTE_WRITE)
@@ -141,7 +104,7 @@ def create_mr(self):
 
 class QpExRCRDMAWriteImm(RCResources):
     def create_qps(self):
-        create_qp_ex(self, e.IBV_QPT_RC,
+        u.create_qp_ex(self, e.IBV_QPT_RC,
                                e.IBV_QP_EX_WITH_RDMA_WRITE_WITH_IMM)
 
     def create_mr(self):
@@ -150,7 +113,7 @@ def create_mr(self):
 
 class QpExRCRDMARead(RCResources):
     def create_qps(self):
-        create_qp_ex(self, e.IBV_QPT_RC, e.IBV_QP_EX_WITH_RDMA_READ)
+        u.create_qp_ex(self, e.IBV_QPT_RC, e.IBV_QP_EX_WITH_RDMA_READ)
 
     def create_mr(self):
         self.mr = u.create_custom_mr(self, e.IBV_ACCESS_REMOTE_READ)
@@ -158,21 +121,21 @@ def create_mr(self):
 
 class QpExRCAtomicCmpSwp(RCResources):
     def create_qps(self):
-        create_qp_ex(self, e.IBV_QPT_RC,
+        u.create_qp_ex(self, e.IBV_QPT_RC,
                                e.IBV_QP_EX_WITH_ATOMIC_CMP_AND_SWP)
         self.mr = u.create_custom_mr(self, e.IBV_ACCESS_REMOTE_ATOMIC)
 
 
 class QpExRCAtomicFetchAdd(RCResources):
     def create_qps(self):
-        create_qp_ex(self, e.IBV_QPT_RC,
+        u.create_qp_ex(self, e.IBV_QPT_RC,
                                e.IBV_QP_EX_WITH_ATOMIC_FETCH_AND_ADD)
         self.mr = u.create_custom_mr(self, e.IBV_ACCESS_REMOTE_ATOMIC)
 
 
 class QpExRCBindMw(RCResources):
     def create_qps(self):
-        create_qp_ex(self, e.IBV_QPT_RC, e.IBV_QP_EX_WITH_RDMA_WRITE |
+        u.create_qp_ex(self, e.IBV_QPT_RC, e.IBV_QP_EX_WITH_RDMA_WRITE |
                      e.IBV_QP_EX_WITH_BIND_MW)
 
     def create_mr(self):
diff --git a/tests/utils.py b/tests/utils.py
index bb2444efa..94ddb2891 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -359,6 +359,42 @@ def get_qp_init_attr(cq, attr):
     return QPInitAttr(scq=cq, rcq=cq, cap=qp_cap, sq_sig_all=sig)
 
 
+def create_qp_ex(agr_obj, qp_type, send_flags):
+    if qp_type == e.IBV_QPT_XRC_SEND:
+        cap = QPCap(max_send_wr=agr_obj.num_msgs, max_recv_wr=0, max_recv_sge=0,
+                    max_send_sge=1)
+    else:
+        cap = QPCap(max_send_wr=agr_obj.num_msgs, max_recv_wr=agr_obj.num_msgs,
+                    max_recv_sge=1, max_send_sge=1)
+    qia = QPInitAttrEx(cap=cap, qp_type=qp_type, scq=agr_obj.cq,
+                       rcq=agr_obj.cq, pd=agr_obj.pd, send_ops_flags=send_flags,
+                       comp_mask=e.IBV_QP_INIT_ATTR_PD |
+                                 e.IBV_QP_INIT_ATTR_SEND_OPS_FLAGS)
+    qp_attr = QPAttr(port_num=agr_obj.ib_port)
+    if qp_type == e.IBV_QPT_UD:
+        qp_attr.qkey = agr_obj.UD_QKEY
+        qp_attr.pkey_index = agr_obj.UD_PKEY_INDEX
+    if qp_type == e.IBV_QPT_RC:
+        qp_attr.qp_access_flags = e.IBV_ACCESS_REMOTE_WRITE | \
+                                  e.IBV_ACCESS_REMOTE_READ | \
+                                  e.IBV_ACCESS_REMOTE_ATOMIC | \
+                                  e.IBV_ACCESS_FLUSH_GLOBAL | \
+                                  e.IBV_ACCESS_FLUSH_PERSISTENT
+    try:
+        # We don't have capability bits for this
+        qp = QPEx(agr_obj.ctx, qia, qp_attr)
+    except PyverbsRDMAError as ex:
+        if ex.error_code == errno.EOPNOTSUPP:
+            raise unittest.SkipTest('Extended QP is not supported on this device')
+        raise ex
+    if qp_type != e.IBV_QPT_XRC_SEND:
+        agr_obj.qps.append(qp)
+        agr_obj.qps_num.append(qp.qp_num)
+        agr_obj.psns.append(random.getrandbits(24))
+    else:
+        return qp
+
+
 def wc_status_to_str(status):
     try:
         return \


Reviews:
user: login: rleon
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-10T06:57:07Z
user: login: ddmatsu
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-10T11:30:37Z
user: login: ddmatsu
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-18T05:23:50Z
user: login: ddmatsu
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-22T06:46:45Z

Review comments:
diff_hunk: @@ -235,6 +235,8 @@ enum ibv_odp_transport_cap_bits {
 	IBV_ODP_SUPPORT_READ     = 1 << 3,
 	IBV_ODP_SUPPORT_ATOMIC   = 1 << 4,
 	IBV_ODP_SUPPORT_SRQ_RECV = 1 << 5,
+	IBV_ODP_SUPPORT_FLUSH    = 1 << 6,
path: libibverbs/verbs.h
user: login: rleon
body: Why aren't these bits coming from UAPI?
created_at: 2025-04-10T06:57:07Z
author_association: MEMBER
diff_hunk: @@ -235,6 +235,8 @@ enum ibv_odp_transport_cap_bits {
 	IBV_ODP_SUPPORT_READ     = 1 << 3,
 	IBV_ODP_SUPPORT_ATOMIC   = 1 << 4,
 	IBV_ODP_SUPPORT_SRQ_RECV = 1 << 5,
+	IBV_ODP_SUPPORT_FLUSH    = 1 << 6,
path: libibverbs/verbs.h
user: login: ddmatsu
body: Okay. I will send a kernel patch to add `IB_UVERBS_ODP_*` flags in `include/uapi/rdma/ib_user_verbs.h` and update this PR accordingly.
created_at: 2025-04-10T11:30:37Z
author_association: CONTRIBUTOR
diff_hunk: @@ -235,6 +235,8 @@ enum ibv_odp_transport_cap_bits {
 	IBV_ODP_SUPPORT_READ     = 1 << 3,
 	IBV_ODP_SUPPORT_ATOMIC   = 1 << 4,
 	IBV_ODP_SUPPORT_SRQ_RECV = 1 << 5,
+	IBV_ODP_SUPPORT_FLUSH    = 1 << 6,
path: libibverbs/verbs.h
user: login: ddmatsu
body: @rleon 
I've submitted the kernel patch and updated this PR.

Thanks
created_at: 2025-04-18T05:23:50Z
author_association: CONTRIBUTOR
diff_hunk: @@ -235,6 +235,8 @@ enum ibv_odp_transport_cap_bits {
 	IBV_ODP_SUPPORT_READ     = 1 << 3,
 	IBV_ODP_SUPPORT_ATOMIC   = 1 << 4,
 	IBV_ODP_SUPPORT_SRQ_RECV = 1 << 5,
+	IBV_ODP_SUPPORT_FLUSH    = 1 << 6,
path: libibverbs/verbs.h
user: login: ddmatsu
body: I revised the kernel-headers commit after all the kernel-side patches were merged.
created_at: 2025-04-22T06:46:45Z
author_association: CONTRIBUTOR

PR comments:
user: login: ddmatsu
created_at: 2025-04-09T08:22:37Z
body: The kernel-side enablement was merged to for-next.
I have confirmed that tests added in this PR run successfully with the latest for-next kernel.
author_association: CONTRIBUTOR
user: login: rleon
created_at: 2025-04-10T06:57:54Z
body: General comment to all patches. Please write commit messages for all commits.

Thanks
author_association: MEMBER
user: login: rleon
created_at: 2025-04-22T07:53:27Z
body: CI must pass for this PR
author_association: MEMBER
user: login: ddmatsu
created_at: 2025-04-25T21:08:33Z
body: I think it is now okay.
author_association: CONTRIBUTOR