Repository: kvcache-ai/Mooncake (2026-07-14T05:30:50Z)

Issue:
RdmaContext accepts an invalid empty completion topology
## Problem

`RdmaContext::construct` does not reject zero completion queues or completion channels before resource setup.

A context with an empty CQ list can later reach `RdmaContext::cq()`, which selects a queue using modulo by `cq_list_.size()`. When the list is empty, the operation is undefined behavior. Empty completion-channel lists and devices exposing no completion vectors have equivalent invalid preconditions.

## Reachability

Environment configuration normally rejects zero values, but the path remains reachable through:

- direct calls to `RdmaContext::construct`;
- process-global configuration clamping during device initialization;
- a device reporting no valid completion vectors.

## Reproduction evidence

Two hardware-independent regression cases call construction with:

- zero CQs and one completion channel;
- one CQ and zero completion channels.

Against the current implementation, both enter resource setup and return `ERR_CONTEXT`. The endpoint store has already been allocated.

## Expected behavior

- Reject zero CQ or completion-channel counts before allocating resources.
- Reject an opened verbs context with no completion vectors.
- Propagate construction failure through the existing rail-disable path.
- Never expose a partially initialized context to endpoint creation.

## Impact

The invalid topology can cause modulo-by-zero undefined behavior during endpoint or CQ creation.



Discussion:
Thanks for opening this issue, @morluto!

| Field | Value |
|-------|-------|
| **Issue** | #2889 |
| **GitHub user ID** | `76467478` |
| **Reporter** | @morluto |

A maintainer will triage this when possible. To help us respond faster, please include:

- Mooncake version or commit SHA
- Environment (OS, CUDA/driver, RDMA stack if relevant)
- Steps to reproduce and expected vs. actual behavior

Useful links: [Documentation](https://kvcache-ai.github.io/Mooncake/) · [Contributing guide](https://github.com/kvcache-ai/Mooncake/blob/main/CONTRIBUTING.md)

> This message was posted automatically by the issue bot.



Patch:
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 3fefbf6af2..d6df05b60d 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -47,7 +47,7 @@ repos:
     hooks:
       - id: codespell
         exclude: '^(extern/|FAST25-release/)'
-        args: ['--ignore-words-list=te,mooncake,KVCache,cann']
+        args: ['--ignore-words-list=te,mooncake,KVCache,cann,hsa']
 
   - repo: https://github.com/pre-commit/mirrors-clang-format
     rev: v20.1.8
diff --git a/mooncake-transfer-engine/src/transport/rdma_transport/rdma_context.cpp b/mooncake-transfer-engine/src/transport/rdma_transport/rdma_context.cpp
index 4e7659d07b..de3d23f662 100644
--- a/mooncake-transfer-engine/src/transport/rdma_transport/rdma_context.cpp
+++ b/mooncake-transfer-engine/src/transport/rdma_transport/rdma_context.cpp
@@ -181,6 +181,13 @@ RdmaContext::~RdmaContext() {
 int RdmaContext::construct(size_t num_cq_list, size_t num_comp_channels,
                            uint8_t port, int gid_index, size_t max_cqe,
                            int max_endpoints) {
+    if (num_cq_list == 0 || num_comp_channels == 0) {
+        LOG(ERROR) << "Invalid RDMA completion configuration for device "
+                   << device_name_ << ": num_cq_list=" << num_cq_list
+                   << ", num_comp_channels=" << num_comp_channels;
+        return ERR_INVALID_ARGUMENT;
+    }
+
     // Create endpoint store based on configuration
     auto &config = globalConfig();
     switch (config.endpoint_store_type) {
@@ -202,6 +209,12 @@ int RdmaContext::construct(size_t num_cq_list, size_t num_comp_channels,
         return ERR_CONTEXT;
     }
 
+    if (context_->num_comp_vectors <= 0) {
+        LOG(ERROR) << "RDMA device " << device_name_
+                   << " exposes no completion vectors";
+        return ERR_CONTEXT;
+    }
+
     pd_ = ibv_alloc_pd(context_);
     if (!pd_) {
         PLOG(ERROR) << "Failed to allocate new protection domain on device "


Test patch:
diff --git a/mooncake-transfer-engine/tests/rdma_context_reprobe_test.cpp b/mooncake-transfer-engine/tests/rdma_context_reprobe_test.cpp
index 319d57b242..35f8706d74 100644
--- a/mooncake-transfer-engine/tests/rdma_context_reprobe_test.cpp
+++ b/mooncake-transfer-engine/tests/rdma_context_reprobe_test.cpp
@@ -19,6 +19,10 @@
 #include <memory>
 #include <string>
 
+#ifdef __linux__
+#include <limits>
+#endif
+
 #include "common.h"
 #include "error.h"
 #include "transfer_metadata.h"
@@ -39,6 +43,105 @@
 
 using namespace mooncake;
 
+#ifdef __linux__
+namespace {
+
+struct FakeVerbsDevice {
+    bool enabled = false;
+    ibv_device device = {};
+    ibv_device *device_list[2] = {&device, nullptr};
+    ibv_context context = {};
+    size_t alloc_pd_calls = 0;
+};
+
+FakeVerbsDevice fake_verbs;
+
+class FakeVerbsDeviceScope {
+   public:
+    explicit FakeVerbsDeviceScope(int num_comp_vectors) {
+        fake_verbs.enabled = true;
+        fake_verbs.context = {};
+        fake_verbs.context.num_comp_vectors = num_comp_vectors;
+        fake_verbs.alloc_pd_calls = 0;
+    }
+
+    ~FakeVerbsDeviceScope() { fake_verbs.enabled = false; }
+
+    size_t allocPdCalls() const { return fake_verbs.alloc_pd_calls; }
+};
+
+}  // namespace
+
+#undef ibv_query_port
+
+// Interpose the libibverbs boundary for this test binary so construct() can
+// exercise device validation without RDMA hardware.
+extern "C" {
+
+ibv_device **ibv_get_device_list(int *num_devices) {
+    if (!fake_verbs.enabled) {
+        *num_devices = 0;
+        return nullptr;
+    }
+    *num_devices = 1;
+    return fake_verbs.device_list;
+}
+
+void ibv_free_device_list(ibv_device **) {}
+
+const char *ibv_get_device_name(ibv_device *device) {
+    if (fake_verbs.enabled && device == &fake_verbs.device)
+        return "nonexistent-device";
+    return "";
+}
+
+ibv_context *ibv_open_device(ibv_device *device) {
+    if (fake_verbs.enabled && device == &fake_verbs.device)
+        return &fake_verbs.context;
+    return nullptr;
+}
+
+int ibv_query_port(ibv_context *context, uint8_t,
+                   _compat_ibv_port_attr *compat_port_attr) {
+    if (!fake_verbs.enabled || context != &fake_verbs.context) return EINVAL;
+    auto *port_attr = reinterpret_cast<ibv_port_attr *>(compat_port_attr);
+    *port_attr = {};
+    port_attr->state = IBV_PORT_ACTIVE;
+    port_attr->lid = 1;
+    port_attr->active_mtu = IBV_MTU_4096;
+    return 0;
+}
+
+int ibv_query_device(ibv_context *context, ibv_device_attr *device_attr) {
+    if (!fake_verbs.enabled || context != &fake_verbs.context) return EINVAL;
+    *device_attr = {};
+    device_attr->max_qp = std::numeric_limits<int>::max();
+    device_attr->max_cq = std::numeric_limits<int>::max();
+    device_attr->max_qp_wr = std::numeric_limits<int>::max();
+    device_attr->max_sge = std::numeric_limits<int>::max();
+    device_attr->max_cqe = std::numeric_limits<int>::max();
+    device_attr->max_mr_size = std::numeric_limits<uint64_t>::max();
+    return 0;
+}
+
+int ibv_query_gid(ibv_context *context, uint8_t, int, ibv_gid *gid) {
+    if (!fake_verbs.enabled || context != &fake_verbs.context) return EINVAL;
+    *gid = {};
+    gid->raw[15] = 1;
+    return 0;
+}
+
+ibv_pd *ibv_alloc_pd(ibv_context *context) {
+    if (!fake_verbs.enabled || context != &fake_verbs.context) return nullptr;
+    ++fake_verbs.alloc_pd_calls;
+    return nullptr;
+}
+
+int ibv_close_device(ibv_context *) { return 0; }
+
+}  // extern "C"
+#endif  // __linux__
+
 namespace mooncake {
 
 class RdmaTransportTestPeer {
@@ -53,6 +156,10 @@ class RdmaTransportTestPeer {
 
 class RdmaContextTestPeer {
    public:
+    static bool hasEndpointStore(const RdmaContext &context) {
+        return context.endpoint_store_ != nullptr;
+    }
+
     static void seedAutoGidState(RdmaContext &context, ibv_context *verbs_ctx,
                                  uint8_t port, uint16_t lid, const ibv_gid &gid,
                                  int gid_index) {
@@ -73,6 +180,53 @@ class RdmaContextTestPeer {
 
 namespace {
 
+class RdmaContextConstructionTest : public ::testing::Test {
+   protected:
+    void SetUp() override {
+        // RdmaTransport teardown requires metadata initialized by install().
+        // These tests only need the constructor reference, so match the
+        // existing uninstalled-transport test setup below.
+        transport_ = new RdmaTransport();
+        MC_LSAN_IGNORE_OBJECT(transport_);
+        context_ =
+            std::make_unique<RdmaContext>(*transport_, "nonexistent-device");
+    }
+
+    RdmaTransport *transport_ = nullptr;
+    std::unique_ptr<RdmaContext> context_;
+};
+
+TEST_F(RdmaContextConstructionTest, RejectsZeroCompletionQueuesBeforeSetup) {
+    EXPECT_EQ(context_->construct(/*num_cq_list=*/0,
+                                  /*num_comp_channels=*/1),
+              ERR_INVALID_ARGUMENT);
+    EXPECT_FALSE(RdmaContextTestPeer::hasEndpointStore(*context_));
+}
+
+TEST_F(RdmaContextConstructionTest, RejectsZeroCompletionChannelsBeforeSetup) {
+    EXPECT_EQ(context_->construct(/*num_cq_list=*/1,
+                                  /*num_comp_channels=*/0),
+              ERR_INVALID_ARGUMENT);
+    EXPECT_FALSE(RdmaContextTestPeer::hasEndpointStore(*context_));
+}
+
+TEST_F(RdmaContextConstructionTest,
+       RejectsDeviceWithoutCompletionVectorsBeforeAllocatingResources) {
+#ifdef __linux__
+    FakeVerbsDeviceScope fake_device(/*num_comp_vectors=*/0);
+
+    EXPECT_EQ(context_->construct(/*num_cq_list=*/1,
+                                  /*num_comp_channels=*/1,
+                                  /*port=*/1,
+                                  /*gid_index=*/0),
+              ERR_CONTEXT);
+    EXPECT_EQ(fake_device.allocPdCalls(), 0);
+    RdmaContextTestPeer::disableContextForTeardown(*context_);
+#else
+    GTEST_SKIP() << "Requires Linux libibverbs symbol interposition";
+#endif
+}
+
 ibv_gid makeGid(const std::array<uint8_t, 16> &bytes) {
     ibv_gid gid = {};
     std::memcpy(gid.raw, bytes.data(), bytes.size());