Repository: apache/doris
Pull request #49223: [improve](cloud-mow) MS can choose use new way or old way when processing delete bitmap lock
Author: login: hust-hhb
Labels: name: approved
description: Indicates a PR has been approved by one committer.
name: reviewed
name: dev/3.1.0-merged
Created: 2025-03-18T14:00:14Z | Merged: 2025-04-18T09:45:33Z

MS can choose use new way or old way when processing delete bitmap lock
### What problem does this PR solve?

Issue Number: close #xxx

Related PR: #xxx

Problem Summary:

### Release note

None

### Check List (For Author)

- Test <!-- At least one of them must be included. -->
    - [ ] Regression test
    - [ ] Unit Test
    - [ ] Manual test (add detailed scripts or steps below)
    - [ ] No need to test or manual test. Explain why:
        - [ ] This is a refactor/code format and no logic has been changed.
        - [ ] Previous test can cover this change.
        - [ ] No code files have been changed.
        - [ ] Other reason <!-- Add your reason?  -->

- Behavior changed:
    - [ ] No.
    - [ ] Yes. <!-- Explain the behavior change -->

- Does this need documentation?
    - [ ] No.
    - [ ] Yes. <!-- Add document PR link here. eg: https://github.com/apache/doris-website/pull/1214 -->

### Check List (For Reviewer who merge this PR)

- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR should merge into -->




Diff:
diff --git a/cloud/src/common/config.h b/cloud/src/common/config.h
index 9f89433fe1bfe4..baa7425e0e3661 100644
--- a/cloud/src/common/config.h
+++ b/cloud/src/common/config.h
@@ -261,6 +261,12 @@ CONF_Bool(enable_check_instance_id, "true");
 
 // Check if ip eq 127.0.0.1, ms/recycler exit
 CONF_Bool(enable_loopback_address_for_ms, "false");
+
+// delete_bitmap_lock version config
+CONF_mString(use_delete_bitmap_lock_version, "v1");
+// FOR DEBUGGING
+CONF_mBool(use_delete_bitmap_lock_random_version, "false");
+
 // Which vaults should be recycled. If empty, recycle all vaults.
 // Comma seprated list: recycler_storage_vault_white_list="aaa,bbb,ccc"
 CONF_Strings(recycler_storage_vault_white_list, "");
diff --git a/cloud/src/meta-service/meta_service.cpp b/cloud/src/meta-service/meta_service.cpp
index 5ba8ac617f10c4..d7fd57c8f218d6 100644
--- a/cloud/src/meta-service/meta_service.cpp
+++ b/cloud/src/meta-service/meta_service.cpp
@@ -1746,7 +1746,7 @@ static bool check_delete_bitmap_lock(MetaServiceCode& code, std::string& msg, st
                                      std::unique_ptr<Transaction>& txn, std::string& instance_id,
                                      int64_t table_id, int64_t lock_id, int64_t lock_initiator,
                                      std::string& lock_key, DeleteBitmapUpdateLockPB& lock_info,
-                                     std::string log = "") {
+                                     std::string use_version, std::string log = "") {
     std::string lock_val;
     LOG(INFO) << "check_delete_bitmap_lock, table_id=" << table_id << " lock_id=" << lock_id
               << " initiator=" << lock_initiator << " key=" << hex(lock_key) << log;
@@ -1775,7 +1775,7 @@ static bool check_delete_bitmap_lock(MetaServiceCode& code, std::string& msg, st
         code = MetaServiceCode::LOCK_EXPIRED;
         return false;
     }
-    if (lock_id == COMPACTION_DELETE_BITMAP_LOCK_ID) {
+    if (use_version == "v2" && lock_id == COMPACTION_DELETE_BITMAP_LOCK_ID) {
         std::string tablet_compaction_key =
                 mow_tablet_compaction_key({instance_id, table_id, lock_initiator});
         std::string tablet_compaction_val;
@@ -1811,6 +1811,15 @@ static bool check_delete_bitmap_lock(MetaServiceCode& code, std::string& msg, st
     }
     return true;
 }
+static bool use_new_version_random() {
+    std::mt19937 gen {std::random_device {}()};
+    auto p = 0.5;
+    std::bernoulli_distribution inject_fault {p};
+    if (inject_fault(gen)) {
+        return true;
+    }
+    return false;
+}
 
 static bool remove_pending_delete_bitmap(MetaServiceCode& code, std::string& msg,
                                          std::stringstream& ss, std::unique_ptr<Transaction>& txn,
@@ -1953,6 +1962,10 @@ void MetaServiceImpl::update_delete_bitmap(google::protobuf::RpcController* cont
                                            const UpdateDeleteBitmapRequest* request,
                                            UpdateDeleteBitmapResponse* response,
                                            ::google::protobuf::Closure* done) {
+    std::string use_version = config::use_delete_bitmap_lock_version;
+    if (config::use_delete_bitmap_lock_random_version && !use_new_version_random()) {
+        use_version = "v1";
+    }
     RPC_PREPROCESS(update_delete_bitmap);
     std::string cloud_unique_id = request->has_cloud_unique_id() ? request->cloud_unique_id() : "";
     if (cloud_unique_id.empty()) {
@@ -1991,7 +2004,8 @@ void MetaServiceImpl::update_delete_bitmap(google::protobuf::RpcController* cont
         std::string lock_key = meta_delete_bitmap_update_lock_key({instance_id, table_id, -1});
         DeleteBitmapUpdateLockPB lock_info;
         if (!check_delete_bitmap_lock(code, msg, ss, txn, instance_id, table_id, request->lock_id(),
-                                      request->initiator(), lock_key, lock_info, log)) {
+                                      request->initiator(), lock_key, lock_info, use_version,
+                                      log)) {
             LOG(WARNING) << "failed to check delete bitmap lock, table_id=" << table_id
                          << " request lock_id=" << request->lock_id()
                          << " request initiator=" << request->initiator() << " msg " << msg;
@@ -2143,7 +2157,7 @@ void MetaServiceImpl::update_delete_bitmap(google::protobuf::RpcController* cont
                 DeleteBitmapUpdateLockPB lock_info;
                 if (!check_delete_bitmap_lock(code, msg, ss, txn, instance_id, table_id,
                                               request->lock_id(), request->initiator(), lock_key,
-                                              lock_info, log)) {
+                                              lock_info, use_version, log)) {
                     LOG(WARNING) << "failed to check delete bitmap lock, table_id=" << table_id
                                  << " request lock_id=" << request->lock_id()
                                  << " request initiator=" << request->initiator() << " msg " << msg;
@@ -2415,10 +2429,12 @@ static bool put_delete_bitmap_update_lock_key(MetaServiceCode& code, std::string
     return true;
 }
 
-void MetaServiceImpl::get_delete_bitmap_update_lock(google::protobuf::RpcController* controller,
-                                                    const GetDeleteBitmapUpdateLockRequest* request,
-                                                    GetDeleteBitmapUpdateLockResponse* response,
-                                                    ::google::protobuf::Closure* done) {
+void MetaServiceImpl::get_delete_bitmap_update_lock_v2(
+        google::protobuf::RpcController* controller,
+        const GetDeleteBitmapUpdateLockRequest* request,
+        GetDeleteBitmapUpdateLockResponse* response, ::google::protobuf::Closure* done) {
+    VLOG_DEBUG << "get delete bitmap update lock in v2 for table=" << request->table_id()
+               << ",lock id=" << request->lock_id() << ",initiator=" << request->initiator();
     RPC_PREPROCESS(get_delete_bitmap_update_lock);
     std::string cloud_unique_id = request->has_cloud_unique_id() ? request->cloud_unique_id() : "";
     if (cloud_unique_id.empty()) {
@@ -2746,7 +2762,7 @@ void MetaServiceImpl::get_delete_bitmap_update_lock(google::protobuf::RpcControl
 
     DeleteBitmapUpdateLockPB lock_info_tmp;
     if (!check_delete_bitmap_lock(code, msg, ss, txn, instance_id, table_id, request->lock_id(),
-                                  request->initiator(), lock_key, lock_info_tmp)) {
+                                  request->initiator(), lock_key, lock_info_tmp, "v2")) {
         LOG(WARNING) << "failed to check delete bitmap lock after get tablet stats and tablet "
                         "states, table_id="
                      << table_id << " request lock_id=" << request->lock_id()
@@ -2755,10 +2771,200 @@ void MetaServiceImpl::get_delete_bitmap_update_lock(google::protobuf::RpcControl
     }
 }
 
-void MetaServiceImpl::remove_delete_bitmap_update_lock(
+void MetaServiceImpl::get_delete_bitmap_update_lock_v1(
+        google::protobuf::RpcController* controller,
+        const GetDeleteBitmapUpdateLockRequest* request,
+        GetDeleteBitmapUpdateLockResponse* response, ::google::protobuf::Closure* done) {
+    VLOG_DEBUG << "get delete bitmap update lock in v1 for table=" << request->table_id()
+               << ",lock id=" << request->lock_id() << ",initiator=" << request->initiator();
+    RPC_PREPROCESS(get_delete_bitmap_update_lock);
+    std::string cloud_unique_id = request->has_cloud_unique_id() ? request->cloud_unique_id() : "";
+    if (cloud_unique_id.empty()) {
+        code = MetaServiceCode::INVALID_ARGUMENT;
+        msg = "cloud unique id not set";
+        return;
+    }
+    instance_id = get_instance_id(resource_mgr_, cloud_unique_id);
+    if (instance_id.empty()) {
+        code = MetaServiceCode::INVALID_ARGUMENT;
+        msg = "empty instance_id";
+        LOG(INFO) << msg << ", cloud_unique_id=" << cloud_unique_id;
+        return;
+    }
+
+    RPC_RATE_LIMIT(get_delete_bitmap_update_lock)
+    std::unique_ptr<Transaction> txn;
+    TxnErrorCode err = txn_kv_->create_txn(&txn);
+    if (err != TxnErrorCode::TXN_OK) {
+        code = cast_as<ErrCategory::CREATE>(err);
+        msg = "failed to init txn";
+        return;
+    }
+    auto table_id = request->table_id();
+    std::string lock_key = meta_delete_bitmap_update_lock_key({instance_id, table_id, -1});
+    std::string lock_val;
+    DeleteBitmapUpdateLockPB lock_info;
+    err = txn->get(lock_key, &lock_val);
+    if (err != TxnErrorCode::TXN_OK && err != TxnErrorCode::TXN_KEY_NOT_FOUND) {
+        ss << "failed to get delete bitmap update lock, instance_id=" << instance_id
+           << " table_id=" << table_id << " key=" << hex(lock_key) << " err=" << err;
+        msg = ss.str();
+        code = MetaServiceCode::KV_TXN_GET_ERR;
+        return;
+    }
+    using namespace std::chrono;
+    int64_t now = duration_cast<seconds>(system_clock::now().time_since_epoch()).count();
+    if (err == TxnErrorCode::TXN_OK) {
+        if (!lock_info.ParseFromString(lock_val)) [[unlikely]] {
+            code = MetaServiceCode::PROTOBUF_PARSE_ERR;
+            msg = "failed to parse DeleteBitmapUpdateLockPB";
+            return;
+        }
+        if (lock_info.expiration() > 0 && lock_info.expiration() < now) {
+            LOG(INFO) << "delete bitmap lock expired, continue to process. lock_id="
+                      << lock_info.lock_id() << " table_id=" << table_id << " now=" << now;
+            lock_info.clear_initiators();
+        } else if (lock_info.lock_id() != request->lock_id()) {
+            ss << "already be locked. request lock_id=" << request->lock_id()
+               << " locked by lock_id=" << lock_info.lock_id() << " table_id=" << table_id
+               << " now=" << now << " expiration=" << lock_info.expiration();
+            msg = ss.str();
+            code = MetaServiceCode::LOCK_CONFLICT;
+            return;
+        }
+    }
+
+    lock_info.set_lock_id(request->lock_id());
+    lock_info.set_expiration(now + request->expiration());
+    bool found = false;
+    for (auto initiator : lock_info.initiators()) {
+        if (request->initiator() == initiator) {
+            found = true;
+            break;
+        }
+    }
+    if (!found) {
+        lock_info.add_initiators(request->initiator());
+    }
+    lock_info.SerializeToString(&lock_val);
+    if (lock_val.empty()) {
+        code = MetaServiceCode::PROTOBUF_SERIALIZE_ERR;
+        msg = "pb serialization error";
+        return;
+    }
+    txn->put(lock_key, lock_val);
+    LOG(INFO) << "xxx put lock_key=" << hex(lock_key) << " table_id=" << table_id
+              << " lock_id=" << request->lock_id() << " initiator=" << request->initiator()
+              << " initiators_size=" << lock_info.initiators_size();
+
+    err = txn->commit();
+    if (err != TxnErrorCode::TXN_OK) {
+        code = cast_as<ErrCategory::COMMIT>(err);
+        ss << "failed to get_delete_bitmap_update_lock, err=" << err;
+        msg = ss.str();
+        return;
+    }
+
+    bool require_tablet_stats =
+            request->has_require_compaction_stats() ? request->require_compaction_stats() : false;
+    if (!require_tablet_stats) return;
+    // this request is from fe when it commits txn for MOW table, we send the compaction stats
+    // along with the GetDeleteBitmapUpdateLockResponse which will be sent to BE later to let
+    // BE eliminate unnecessary sync_rowsets() calls if possible
+    // 1. hold the delete bitmap update lock in MS(update lock_info.lock_id to current load's txn id)
+    // 2. read tablets' stats
+    // 3. check whether we still hold the delete bitmap update lock
+    // these steps can be done in different fdb txns
+
+    StopWatch read_stats_sw;
+    err = txn_kv_->create_txn(&txn);
+    if (err != TxnErrorCode::TXN_OK) {
+        code = cast_as<ErrCategory::CREATE>(err);
+        msg = "failed to init txn";
+        return;
+    }
+    for (const auto& tablet_idx : request->tablet_indexes()) {
+        // 1. get compaction cnts
+        TabletStatsPB tablet_stat;
+        std::string stats_key =
+                stats_tablet_key({instance_id, tablet_idx.table_id(), tablet_idx.index_id(),
+                                  tablet_idx.partition_id(), tablet_idx.tablet_id()});
+        std::string stats_val;
+        TxnErrorCode err = txn->get(stats_key, &stats_val);
+        TEST_SYNC_POINT_CALLBACK("get_delete_bitmap_update_lock.get_compaction_cnts_inject_error",
+                                 &err);
+        if (err == TxnErrorCode::TXN_TOO_OLD) {
+            code = MetaServiceCode::OK;
+            err = txn_kv_->create_txn(&txn);
+            if (err != TxnErrorCode::TXN_OK) {
+                code = cast_as<ErrCategory::CREATE>(err);
+                ss << "failed to init txn when get tablet stats";
+                msg = ss.str();
+                return;
+            }
+            err = txn->get(stats_key, &stats_val);
+        }
+        if (err != TxnErrorCode::TXN_OK) {
+            code = cast_as<ErrCategory::READ>(err);
+            msg = fmt::format("failed to get tablet stats, err={} tablet_id={}", err,
+                              tablet_idx.tablet_id());
+            return;
+        }
+        if (!tablet_stat.ParseFromArray(stats_val.data(), stats_val.size())) {
+            code = MetaServiceCode::PROTOBUF_PARSE_ERR;
+            msg = fmt::format("marformed tablet stats value, key={}", hex(stats_key));
+            return;
+        }
+        response->add_base_compaction_cnts(tablet_stat.base_compaction_cnt());
+        response->add_cumulative_compaction_cnts(tablet_stat.cumulative_compaction_cnt());
+        response->add_cumulative_points(tablet_stat.cumulative_point());
+        // 2. get tablet states
+        std::string tablet_meta_key =
+                meta_tablet_key({instance_id, tablet_idx.table_id(), tablet_idx.index_id(),
+                                 tablet_idx.partition_id(), tablet_idx.tablet_id()});
+        std::string tablet_meta_val;
+        err = txn->get(tablet_meta_key, &tablet_meta_val);
+        if (err != TxnErrorCode::TXN_OK) {
+            ss << "failed to get tablet meta"
+               << (err == TxnErrorCode::TXN_KEY_NOT_FOUND ? " (not found)" : "")
+               << " instance_id=" << instance_id << " tablet_id=" << tablet_idx.tablet_id()
+               << " key=" << hex(tablet_meta_key) << " err=" << err;
+            msg = ss.str();
+            code = err == TxnErrorCode::TXN_KEY_NOT_FOUND ? MetaServiceCode::TABLET_NOT_FOUND
+                                                          : cast_as<ErrCategory::READ>(err);
+            return;
+        }
+        doris::TabletMetaCloudPB tablet_meta;
+        if (!tablet_meta.ParseFromString(tablet_meta_val)) {
+            code = MetaServiceCode::PROTOBUF_PARSE_ERR;
+            msg = "malformed tablet meta";
+            return;
+        }
+        response->add_tablet_states(
+                static_cast<std::underlying_type_t<TabletStatePB>>(tablet_meta.tablet_state()));
+    }
+    read_stats_sw.pause();
+    LOG(INFO) << fmt::format(
+            "tablet_idxes.size()={}, read tablet compaction cnts and tablet states cost={} ms",
+            request->tablet_indexes().size(), read_stats_sw.elapsed_us() / 1000);
+
+    DeleteBitmapUpdateLockPB lock_info_tmp;
+    if (!check_delete_bitmap_lock(code, msg, ss, txn, instance_id, table_id, request->lock_id(),
+                                  request->initiator(), lock_key, lock_info_tmp, "v1")) {
+        LOG(WARNING) << "failed to check delete bitmap lock after get tablet stats and tablet "
+                        "states, table_id="
+                     << table_id << " request lock_id=" << request->lock_id()
+                     << " request initiator=" << request->initiator() << " code=" << code
+                     << " msg=" << msg;
+    }
+}
+
+void MetaServiceImpl::remove_delete_bitmap_update_lock_v2(
         google::protobuf::RpcController* controller,
         const RemoveDeleteBitmapUpdateLockRequest* request,
         RemoveDeleteBitmapUpdateLockResponse* response, ::google::protobuf::Closure* done) {
+    VLOG_DEBUG << "remove delete bitmap update lock in v2 for table=" << request->table_id()
+               << ",lock id=" << request->lock_id() << ",initiator=" << request->initiator();
     RPC_PREPROCESS(remove_delete_bitmap_update_lock);
     std::string cloud_unique_id = request->has_cloud_unique_id() ? request->cloud_unique_id() : "";
     if (cloud_unique_id.empty()) {
@@ -2797,7 +3003,7 @@ void MetaServiceImpl::remove_delete_bitmap_update_lock(
         DeleteBitmapUpdateLockPB lock_info;
         if (!check_delete_bitmap_lock(code, msg, ss, txn, instance_id, request->table_id(),
                                       request->lock_id(), request->initiator(), lock_key, lock_info,
-                                      ", remove lock")) {
+                                      "v2", ", remove lock")) {
             LOG(WARNING) << "failed to check delete bitmap tablet lock"
                          << " table_id=" << request->table_id()
                          << " tablet_id=" << request->tablet_id()
@@ -2845,6 +3051,116 @@ void MetaServiceImpl::remove_delete_bitmap_update_lock(
     }
 }
 
+void MetaServiceImpl::remove_delete_bitmap_update_lock_v1(
+        google::protobuf::RpcController* controller,
+        const RemoveDeleteBitmapUpdateLockRequest* request,
+        RemoveDeleteBitmapUpdateLockResponse* response, ::google::protobuf::Closure* done) {
+    VLOG_DEBUG << "remove delete bitmap update lock in v1 for table=" << request->table_id()
+               << ",lock id=" << request->lock_id() << ",initiator=" << request->initiator();
+    RPC_PREPROCESS(remove_delete_bitmap_update_lock);
+    std::string cloud_unique_id = request->has_cloud_unique_id() ? request->cloud_unique_id() : "";
+    if (cloud_unique_id.empty()) {
+        code = MetaServiceCode::INVALID_ARGUMENT;
+        msg = "cloud unique id not set";
+        return;
+    }
+    instance_id = get_instance_id(resource_mgr_, cloud_unique_id);
+    if (instance_id.empty()) {
+        code = MetaServiceCode::INVALID_ARGUMENT;
+        msg = "empty instance_id";
+        LOG(INFO) << msg << ", cloud_unique_id=" << cloud_unique_id;
+        return;
+    }
+    RPC_RATE_LIMIT(remove_delete_bitmap_update_lock)
+    std::unique_ptr<Transaction> txn;
+    TxnErrorCode err = txn_kv_->create_txn(&txn);
+    if (err != TxnErrorCode::TXN_OK) {
+        code = cast_as<ErrCategory::CREATE>(err);
+        msg = "failed to init txn";
+        return;
+    }
+    std::string lock_key =
+            meta_delete_bitmap_update_lock_key({instance_id, request->table_id(), -1});
+    std::string lock_val;
+    DeleteBitmapUpdateLockPB lock_info;
+    if (!check_delete_bitmap_lock(code, msg, ss, txn, instance_id, request->table_id(),
+                                  request->lock_id(), request->initiator(), lock_key, lock_info,
+                                  "v1")) {
+        LOG(WARNING) << "failed to check delete bitmap tablet lock"
+                     << " table_id=" << request->table_id() << " tablet_id=" << request->tablet_id()
+                     << " request lock_id=" << request->lock_id()
+                     << " request initiator=" << request->initiator() << " msg " << msg;
+        return;
+    }
+    bool modify_initiators = false;
+    auto initiators = lock_info.mutable_initiators();
+    for (auto iter = initiators->begin(); iter != initiators->end(); iter++) {
+        if (*iter == request->initiator()) {
+            initiators->erase(iter);
+            modify_initiators = true;
+            break;
+        }
+    }
+    if (!modify_initiators) {
+        LOG(INFO) << "initiators don't have initiator=" << request->initiator()
+                  << ",initiators_size=" << lock_info.initiators_size() << ",just return";
+        return;
+    } else if (initiators->empty()) {
+        LOG(INFO) << "remove delete bitmap lock, table_id=" << request->table_id()
+                  << " lock_id=" << request->lock_id() << " key=" << hex(lock_key);
+        txn->remove(lock_key);
+    } else {
+        lock_info.SerializeToString(&lock_val);
+        if (lock_val.empty()) {
+            LOG(WARNING) << "failed to seiralize lock_info, table_id=" << request->table_id()
+                         << " key=" << hex(lock_key);
+            return;
+        }
+        LOG(INFO) << "remove delete bitmap lock initiator, table_id=" << request->table_id()
+                  << ", key=" << hex(lock_key) << " lock_id=" << request->lock_id()
+                  << " initiator=" << request->initiator()
+                  << " initiators_size=" << lock_info.initiators_size();
+        txn->put(lock_key, lock_val);
+    }
+    err = txn->commit();
+    if (err != TxnErrorCode::TXN_OK) {
+        code = cast_as<ErrCategory::COMMIT>(err);
+        ss << "failed to remove delete bitmap tablet lock , err=" << err;
+        msg = ss.str();
+        return;
+    }
+}
+
+void MetaServiceImpl::get_delete_bitmap_update_lock(google::protobuf::RpcController* controller,
+                                                    const GetDeleteBitmapUpdateLockRequest* request,
+                                                    GetDeleteBitmapUpdateLockResponse* response,
+                                                    ::google::protobuf::Closure* done) {
+    std::string use_version = config::use_delete_bitmap_lock_version;
+    if (config::use_delete_bitmap_lock_random_version && !use_new_version_random()) {
+        use_version = "v1";
+    }
+    if (use_version == "v2") {
+        get_delete_bitmap_update_lock_v2(controller, request, response, done);
+    } else {
+        get_delete_bitmap_update_lock_v1(controller, request, response, done);
+    }
+}
+
+void MetaServiceImpl::remove_delete_bitmap_update_lock(
+        google::protobuf::RpcController* controller,
+        const RemoveDeleteBitmapUpdateLockRequest* request,
+        RemoveDeleteBitmapUpdateLockResponse* response, ::google::protobuf::Closure* done) {
+    std::string use_version = config::use_delete_bitmap_lock_version;
+    if (config::use_delete_bitmap_lock_random_version && !use_new_version_random()) {
+        use_version = "v1";
+    }
+    if (use_version == "v2") {
+        remove_delete_bitmap_update_lock_v2(controller, request, response, done);
+    } else {
+        remove_delete_bitmap_update_lock_v1(controller, request, response, done);
+    }
+}
+
 void MetaServiceImpl::remove_delete_bitmap(google::protobuf::RpcController* controller,
                                            const RemoveDeleteBitmapRequest* request,
                                            RemoveDeleteBitmapResponse* response,
diff --git a/cloud/src/meta-service/meta_service.h b/cloud/src/meta-service/meta_service.h
index 57f88d51dfe904..ae2dc2997ad02b 100644
--- a/cloud/src/meta-service/meta_service.h
+++ b/cloud/src/meta-service/meta_service.h
@@ -321,6 +321,26 @@ class MetaServiceImpl : public cloud::MetaService {
             const AlterInstanceRequest* request,
             std::function<std::pair<MetaServiceCode, std::string>(InstanceInfoPB*)> action);
 
+    void get_delete_bitmap_update_lock_v2(google::protobuf::RpcController* controller,
+                                          const GetDeleteBitmapUpdateLockRequest* request,
+                                          GetDeleteBitmapUpdateLockResponse* response,
+                                          ::google::protobuf::Closure* done);
+
+    void get_delete_bitmap_update_lock_v1(google::protobuf::RpcController* controller,
+                                          const GetDeleteBitmapUpdateLockRequest* request,
+                                          GetDeleteBitmapUpdateLockResponse* response,
+                                          ::google::protobuf::Closure* done);
+
+    void remove_delete_bitmap_update_lock_v2(google::protobuf::RpcController* controller,
+                                             const RemoveDeleteBitmapUpdateLockRequest* request,
+                                             RemoveDeleteBitmapUpdateLockResponse* response,
+                                             ::google::protobuf::Closure* done);
+
+    void remove_delete_bitmap_update_lock_v1(google::protobuf::RpcController* controller,
+                                             const RemoveDeleteBitmapUpdateLockRequest* request,
+                                             RemoveDeleteBitmapUpdateLockResponse* response,
+                                             ::google::protobuf::Closure* done);
+
     std::shared_ptr<TxnKv> txn_kv_;
     std::shared_ptr<ResourceManager> resource_mgr_;
     std::shared_ptr<RateLimiter> rate_limiter_;
diff --git a/cloud/src/meta-service/meta_service_job.cpp b/cloud/src/meta-service/meta_service_job.cpp
index a86a6c5d56f1dd..81d92e3a0d62bb 100644
--- a/cloud/src/meta-service/meta_service_job.cpp
+++ b/cloud/src/meta-service/meta_service_job.cpp
@@ -475,12 +475,13 @@ void MetaServiceImpl::start_tablet_job(::google::protobuf::RpcController* contro
     }
 }
 
-static bool check_and_remove_delete_bitmap_update_lock(MetaServiceCode& code, std::string& msg,
-                                                       std::stringstream& ss,
-                                                       std::unique_ptr<Transaction>& txn,
-                                                       std::string& instance_id, int64_t table_id,
-                                                       int64_t tablet_id, int64_t lock_id,
-                                                       int64_t lock_initiator) {
+static bool check_and_remove_delete_bitmap_update_lock(
+        MetaServiceCode& code, std::string& msg, std::stringstream& ss,
+        std::unique_ptr<Transaction>& txn, std::string& instance_id, int64_t table_id,
+        int64_t tablet_id, int64_t lock_id, int64_t lock_initiator, std::string use_version) {
+    VLOG_DEBUG << "check_and_remove_delete_bitmap_update_lock table_id=" << table_id
+               << " tablet_id=" << tablet_id << " lock_id=" << lock_id
+               << " initiator=" << lock_initiator << " use_version=" << use_version;
     std::string lock_key = meta_delete_bitmap_update_lock_key({instance_id, table_id, -1});
     std::string lock_val;
     TxnErrorCode err = txn->get(lock_key, &lock_val);
@@ -506,7 +507,7 @@ static bool check_and_remove_delete_bitmap_update_lock(MetaServiceCode& code, st
         code = MetaServiceCode::LOCK_EXPIRED;
         return false;
     }
-    if (lock_id == COMPACTION_DELETE_BITMAP_LOCK_ID) {
+    if (use_version == "v2" && lock_id == COMPACTION_DELETE_BITMAP_LOCK_ID) {
         // when upgrade ms, prevent old ms get delete bitmap update lock
         if (lock_info.initiators_size() > 0) {
             ss << "compaction lock has " << lock_info.initiators_size() << " initiators";
@@ -636,8 +637,11 @@ static void remove_delete_bitmap_update_lock_v1(std::unique_ptr<Transaction>& tx
 static void remove_delete_bitmap_update_lock(std::unique_ptr<Transaction>& txn,
                                              const std::string& instance_id, int64_t table_id,
                                              int64_t tablet_id, int64_t lock_id,
-                                             int64_t lock_initiator) {
-    if (lock_id == COMPACTION_DELETE_BITMAP_LOCK_ID) {
+                                             int64_t lock_initiator, std::string use_version) {
+    VLOG_DEBUG << "remove_delete_bitmap_update_lock table_id=" << table_id
+               << " initiator=" << lock_initiator << " tablet_id=" << tablet_id
+               << " lock_id=" << lock_id << " use_version=" << use_version;
+    if (use_version == "v2" && lock_id == COMPACTION_DELETE_BITMAP_LOCK_ID) {
         std::string tablet_compaction_key =
                 mow_tablet_compaction_key({instance_id, table_id, lock_initiator});
         std::string tablet_compaction_val;
@@ -662,6 +666,16 @@ static void remove_delete_bitmap_update_lock(std::unique_ptr<Transaction>& txn,
     }
 }
 
+static bool use_new_version_random() {
+    std::mt19937 gen {std::random_device {}()};
+    auto p = 0.5;
+    std::bernoulli_distribution inject_fault {p};
+    if (inject_fault(gen)) {
+        return true;
+    }
+    return false;
+}
+
 void process_compaction_job(MetaServiceCode& code, std::string& msg, std::stringstream& ss,
                             std::unique_ptr<Transaction>& txn,
                             const FinishTabletJobRequest* request,
@@ -737,9 +751,13 @@ void process_compaction_job(MetaServiceCode& code, std::string& msg, std::string
         INSTANCE_LOG(INFO) << "abort tablet compaction job, tablet_id=" << tablet_id
                            << " key=" << hex(job_key);
         if (compaction.has_delete_bitmap_lock_initiator()) {
-            remove_delete_bitmap_update_lock(txn, instance_id, table_id, tablet_id,
-                                             COMPACTION_DELETE_BITMAP_LOCK_ID,
-                                             compaction.delete_bitmap_lock_initiator());
+            std::string use_version = config::use_delete_bitmap_lock_version;
+            if (config::use_delete_bitmap_lock_random_version && !use_new_version_random()) {
+                use_version = "v1";
+            }
+            remove_delete_bitmap_update_lock(
+                    txn, instance_id, table_id, tablet_id, COMPACTION_DELETE_BITMAP_LOCK_ID,
+                    compaction.delete_bitmap_lock_initiator(), use_version);
         }
         need_commit = true;
         return;
@@ -889,9 +907,14 @@ void process_compaction_job(MetaServiceCode& code, std::string& msg, std::string
 
     // remove delete bitmap update lock for MoW table
     if (compaction.has_delete_bitmap_lock_initiator()) {
+        std::string use_version = config::use_delete_bitmap_lock_version;
+        if (config::use_delete_bitmap_lock_random_version && !use_new_version_random()) {
+            use_version = "v1";
+        }
         bool success = check_and_remove_delete_bitmap_update_lock(
                 code, msg, ss, txn, instance_id, table_id, tablet_id,
-                COMPACTION_DELETE_BITMAP_LOCK_ID, compaction.delete_bitmap_lock_initiator());
+                COMPACTION_DELETE_BITMAP_LOCK_ID, compaction.delete_bitmap_lock_initiator(),
+                use_version);
         if (!success) {
             return;
         }
@@ -1386,9 +1409,14 @@ void process_schema_change_job(MetaServiceCode& code, std::string& msg, std::str
 
     // process mow table, check lock
     if (new_tablet_meta.enable_unique_key_merge_on_write()) {
+        std::string use_version = config::use_delete_bitmap_lock_version;
+        if (config::use_delete_bitmap_lock_random_version && !use_new_version_random()) {
+            use_version = "v1";
+        }
         bool success = check_and_remove_delete_bitmap_update_lock(
                 code, msg, ss, txn, instance_id, new_table_id, new_tablet_id,
-                SCHEMA_CHANGE_DELETE_BITMAP_LOCK_ID, schema_change.delete_bitmap_lock_initiator());
+                SCHEMA_CHANGE_DELETE_BITMAP_LOCK_ID, schema_change.delete_bitmap_lock_initiator(),
+                use_version);
         if (!success) {
             return;
         }
diff --git a/cloud/test/meta_service_job_test.cpp b/cloud/test/meta_service_job_test.cpp
index d905300766e4f6..8e91a70e233e46 100644
--- a/cloud/test/meta_service_job_test.cpp
+++ b/cloud/test/meta_service_job_test.cpp
@@ -1090,6 +1090,805 @@ TEST(MetaServiceJobTest, CompactionJobTest) {
     ASSERT_NO_FATAL_FAILURE(test_abort_compaction_job(1, 2, 3, 7));
 }
 
+TEST(MetaServiceJobTest, DeleteBitmapUpdateLockCompatibilityTest) {
+    auto meta_service = get_meta_service();
+    auto sp = SyncPoint::get_instance();
+    std::unique_ptr<int, std::function<void(int*)>> defer(
+            (int*)0x01, [](int*) { SyncPoint::get_instance()->clear_all_call_backs(); });
+    sp->set_call_back("get_instance_id", [&](auto&& args) {
+        auto* ret = try_any_cast_ret<std::string>(args);
+        ret->first = instance_id;
+        ret->second = true;
+    });
+    sp->enable_processing();
+
+    brpc::Controller cntl;
+
+    auto test_start_compaction_job = [&](int64_t table_id, int64_t index_id, int64_t partition_id,
+                                         int64_t tablet_id,
+                                         TabletCompactionJobPB::CompactionType type,
+                                         std::string job_id = "job_id123") {
+        StartTabletJobResponse res;
+
+        auto index_key = meta_tablet_idx_key({instance_id, tablet_id});
+        TabletIndexPB idx_pb;
+        idx_pb.set_table_id(table_id);
+        idx_pb.set_index_id(index_id);
+        idx_pb.set_partition_id(partition_id);
+        idx_pb.set_tablet_id(tablet_id);
+        std::string idx_val = idx_pb.SerializeAsString();
+        std::unique_ptr<Transaction> txn;
+        ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK);
+        txn->put(index_key, idx_val);
+        std::string stats_key =
+                stats_tablet_key({instance_id, table_id, index_id, partition_id, tablet_id});
+        TabletStatsPB stats;
+        stats.set_base_compaction_cnt(9);
+        stats.set_cumulative_compaction_cnt(19);
+        txn->put(stats_key, stats.SerializeAsString());
+        ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK);
+        start_compaction_job(meta_service.get(), tablet_id, job_id, "ip:port", 9, 19, type, res);
+        ASSERT_EQ(res.status().code(), MetaServiceCode::OK);
+    };
+    FinishTabletJobResponse res;
+    auto test_commit_compaction_job = [&](int64_t table_id, int64_t index_id, int64_t partition_id,
+                                          int64_t tablet_id,
+                                          TabletCompactionJobPB::CompactionType type,
+                                          int64_t initiator = 12345,
+                                          std::string job_id = "job_id123") {
+        FinishTabletJobRequest req;
+
+        auto compaction = req.mutable_job()->add_compaction();
+        compaction->set_id(job_id);
+        compaction->set_initiator("ip:port");
+        compaction->set_base_compaction_cnt(10);
+        compaction->set_cumulative_compaction_cnt(20);
+        compaction->set_delete_bitmap_lock_initiator(initiator);
+        req.mutable_job()->mutable_idx()->set_table_id(table_id);
+        req.mutable_job()->mutable_idx()->set_index_id(index_id);
+        req.mutable_job()->mutable_idx()->set_partition_id(partition_id);
+        req.mutable_job()->mutable_idx()->set_tablet_id(tablet_id);
+        req.set_action(FinishTabletJobRequest::COMMIT);
+        req.set_cloud_unique_id("test_cloud_unique_id");
+
+        auto tablet_meta_key =
+                meta_tablet_key({instance_id, table_id, index_id, partition_id, tablet_id});
+        std::unique_ptr<Transaction> txn;
+        ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK);
+        doris::TabletMetaCloudPB tablet_meta_pb;
+        tablet_meta_pb.set_table_id(table_id);
+        tablet_meta_pb.set_index_id(index_id);
+        tablet_meta_pb.set_partition_id(partition_id);
+        tablet_meta_pb.set_tablet_id(tablet_id);
+        tablet_meta_pb.set_cumulative_layer_point(50);
+        std::string tablet_meta_val = tablet_meta_pb.SerializeAsString();
+        ASSERT_FALSE(tablet_meta_val.empty());
+        txn->put(tablet_meta_key, tablet_meta_val);
+        ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK);
+
+        // Create tablet stats, compaction job will update stats
+        auto tablet_stats_key =
+                stats_tablet_key({instance_id, table_id, index_id, partition_id, tablet_id});
+        TabletStatsPB tablet_stats_pb;
+        tablet_stats_pb.mutable_idx()->set_table_id(table_id);
+        tablet_stats_pb.mutable_idx()->set_index_id(index_id);
+        tablet_stats_pb.mutable_idx()->set_partition_id(partition_id);
+        tablet_stats_pb.mutable_idx()->set_tablet_id(tablet_id);
+
+        std::mt19937 rng(std::chrono::system_clock::now().time_since_epoch().count());
+        std::uniform_int_distribution<int> dist(1, 10000); // Positive numbers
+
+        compaction->set_output_cumulative_point(tablet_stats_pb.cumulative_point() + dist(rng));
+        compaction->set_num_output_rows(dist(rng));
+        compaction->set_num_output_rowsets(dist(rng));
+        compaction->set_num_output_segments(dist(rng));
+        compaction->set_num_input_rows(dist(rng));
+        compaction->set_num_input_rowsets(dist(rng));
+        compaction->set_num_input_segments(dist(rng));
+        compaction->set_size_input_rowsets(dist(rng));
+        compaction->set_size_output_rowsets(dist(rng));
+        compaction->set_type(type);
+
+        tablet_stats_pb.set_cumulative_compaction_cnt(dist(rng));
+        tablet_stats_pb.set_base_compaction_cnt(dist(rng));
+        tablet_stats_pb.set_cumulative_point(tablet_meta_pb.cumulative_layer_point());
+        // MUST let data stats be larger than input data size
+        tablet_stats_pb.set_num_rows(dist(rng) + compaction->num_input_rows());
+        tablet_stats_pb.set_data_size(dist(rng) + compaction->size_input_rowsets());
+        tablet_stats_pb.set_num_rowsets(dist(rng) + compaction->num_input_rowsets());
+        tablet_stats_pb.set_num_segments(dist(rng) + compaction->num_input_segments());
+        tablet_stats_pb.set_index_size(dist(rng) + compaction->index_size_input_rowsets());
+        tablet_stats_pb.set_segment_size(dist(rng) + compaction->segment_size_input_rowsets());
+
+        std::string tablet_stats_val = tablet_stats_pb.SerializeAsString();
+        ASSERT_FALSE(tablet_stats_val.empty());
+        ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK);
+        txn->put(tablet_stats_key, tablet_stats_val);
+        ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK);
+
+        // Provide input and output rowset info
+        int64_t input_version_start = dist(rng);
+        int64_t input_version_end = input_version_start + 100;
+        compaction->add_input_versions(input_version_start);
+        compaction->add_input_versions(input_version_end);
+        compaction->add_output_versions(input_version_end);
+        compaction->add_output_rowset_ids("output rowset id");
+
+        // Provide input rowset KVs, boundary test, 5 input rowsets
+        ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK);
+        // clang-format off
+        std::vector<std::string> input_rowset_keys = {
+                meta_rowset_key({instance_id, tablet_id, input_version_start - 1}),
+                meta_rowset_key({instance_id, tablet_id, input_version_start}),
+                meta_rowset_key({instance_id, tablet_id, input_version_start + 1}),
+                meta_rowset_key({instance_id, tablet_id, (input_version_start + input_version_end) / 2}),
+                meta_rowset_key({instance_id, tablet_id, input_version_end - 1}),
+                meta_rowset_key({instance_id, tablet_id, input_version_end}),
+                meta_rowset_key({instance_id, tablet_id, input_version_end + 1}),
+        };
+        // clang-format on
+        std::vector<std::unique_ptr<std::string>> input_rowset_vals;
+        for (auto& i : input_rowset_keys) {
+            doris::RowsetMetaCloudPB rs_pb;
+            rs_pb.set_rowset_id(0);
+            rs_pb.set_rowset_id_v2(hex(i));
+            input_rowset_vals.emplace_back(new std::string(rs_pb.SerializeAsString()));
+            txn->put(i, *input_rowset_vals.back());
+        }
+        ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK);
+
+        int64_t txn_id = dist(rng);
+        compaction->add_txn_id(txn_id);
+
+        // Provide output rowset meta
+        auto tmp_rowset_key = meta_rowset_tmp_key({instance_id, txn_id, tablet_id});
+        doris::RowsetMetaCloudPB tmp_rs_pb;
+        tmp_rs_pb.set_rowset_id(0);
+        tmp_rs_pb.set_txn_id(10086);
+        auto tmp_rowset_val = tmp_rs_pb.SerializeAsString();
+        ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK);
+        txn->put(tmp_rowset_key, tmp_rowset_val);
+        ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK);
+
+        meta_service->finish_tablet_job(reinterpret_cast<::google::protobuf::RpcController*>(&cntl),
+                                        &req, &res, nullptr);
+    };
+
+    auto clear_rowsets = [&](int64_t tablet_id) {
+        std::unique_ptr<Transaction> txn;
+        ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK);
+        std::string key1 = meta_rowset_key({instance_id, tablet_id, 1});
+        std::string key2 = meta_rowset_key({instance_id, tablet_id, 10001});
+        txn->remove(key1, key2);
+        ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK);
+    };
+
+    config::use_delete_bitmap_lock_random_version = false;
+    int64_t table_id = 111;
+    remove_delete_bitmap_lock(meta_service.get(), table_id);
+    clear_rowsets(table_id);
+
+    // case 1: lock key does not exist, get and remove load lock in new way, success
+    config::use_delete_bitmap_lock_version = "v2";
+    auto res_code = get_delete_bitmap_lock(meta_service.get(), table_id, 123, -1);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, 123, -1);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+
+    // case 2: lock key does not exist, get and remove load lock in old way, success
+    config::use_delete_bitmap_lock_version = "v1";
+    res_code = get_delete_bitmap_lock(meta_service.get(), table_id, 123, -1);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, 123, -1);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+
+    // case 3: lock key does not exist, get and remove compaction lock in new way, success
+    config::use_delete_bitmap_lock_version = "v2";
+    test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE);
+    res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 12345);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    test_commit_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE);
+    ASSERT_EQ(res.status().code(), MetaServiceCode::OK);
+
+    // case 4: lock key does not exist, get and remove compaction lock in old way, success
+    config::use_delete_bitmap_lock_version = "v1";
+    test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE);
+    res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 12345);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    test_commit_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE);
+    ASSERT_EQ(res.status().code(), MetaServiceCode::OK);
+
+    // case 5:
+    // 5.1 lock key does not exist
+    remove_delete_bitmap_lock(meta_service.get(), table_id);
+    clear_rowsets(table_id);
+    // 5.2 load get lock in new way
+    config::use_delete_bitmap_lock_version = "v2";
+    res_code = get_delete_bitmap_lock(meta_service.get(), table_id, 222, -1);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    // 5.3 compaction get lock in new way, failed
+    res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 111);
+    ASSERT_EQ(res_code, MetaServiceCode::LOCK_CONFLICT);
+    // 5.4 load remove lock in old way, success
+    config::use_delete_bitmap_lock_version = "v1";
+    res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, 222, -1);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+
+    // case 6:
+    // 6.1 lock key does not exist
+    remove_delete_bitmap_lock(meta_service.get(), table_id);
+    clear_rowsets(table_id);
+    // 6.2 load get lock in old way
+    config::use_delete_bitmap_lock_version = "v1";
+    res_code = get_delete_bitmap_lock(meta_service.get(), table_id, 222, -1);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    // 6.3 compaction get lock in old way, failed
+    res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 111);
+    ASSERT_EQ(res_code, MetaServiceCode::LOCK_CONFLICT);
+    // 6.4 load remove lock in new way, success
+    config::use_delete_bitmap_lock_version = "v2";
+    res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, 222, -1);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+
+    // case 7:
+    // 7.1 lock key does not exist
+    remove_delete_bitmap_lock(meta_service.get(), table_id);
+    clear_rowsets(table_id);
+    // 7.2 compaction get lock in new way
+    remove_delete_bitmap_lock(meta_service.get(), table_id);
+    config::use_delete_bitmap_lock_version = "v2";
+    res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 777);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    // 7.3 load get lock in new way
+    res_code = get_delete_bitmap_lock(meta_service.get(), table_id, 123, -1);
+    ASSERT_EQ(res_code, MetaServiceCode::LOCK_CONFLICT);
+    // 7.4 compaction remove lock in old way, failed
+    config::use_delete_bitmap_lock_version = "v1";
+    test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE);
+    test_commit_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE, 777);
+    ASSERT_EQ(res.status().code(), MetaServiceCode::LOCK_EXPIRED);
+
+    // case 8:
+    // 8.1 lock key does not exist
+    remove_delete_bitmap_lock(meta_service.get(), table_id);
+    clear_rowsets(table_id);
+    // 8.2 compaction get lock in old way
+    config::use_delete_bitmap_lock_version = "v1";
+    res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 888);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    // 8.3 load get lock in old way
+    res_code = get_delete_bitmap_lock(meta_service.get(), table_id, 124, -1);
+    ASSERT_EQ(res_code, MetaServiceCode::LOCK_CONFLICT);
+    // 8.4 compaction remove lock in new way, failed
+    config::use_delete_bitmap_lock_version = "v2";
+    test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE);
+    test_commit_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE, 888);
+    ASSERT_EQ(res.status().code(), MetaServiceCode::LOCK_EXPIRED);
+
+    // case 9:
+    // 9.1 lock key does not exist
+    remove_delete_bitmap_lock(meta_service.get(), table_id);
+    clear_rowsets(table_id);
+    // 9.2 compaction1 get lock in new way
+    config::use_delete_bitmap_lock_version = "v2";
+    res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 901);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    // 9.3 compaction2 get and remove lock in old way
+    config::use_delete_bitmap_lock_version = "v1";
+    res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 902);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE);
+    test_commit_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE, 902);
+    ASSERT_EQ(res.status().code(), MetaServiceCode::OK);
+    // 9.4 load get and remove lock in old way
+    config::use_delete_bitmap_lock_version = "v1";
+    res_code = get_delete_bitmap_lock(meta_service.get(), table_id, 199, -1);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, 199, -1);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    // 9.5 compaction3 get lock in new way
+    config::use_delete_bitmap_lock_version = "v2";
+    res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 903);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    // 9.6 compaction1 remove lock in new way, failed
+    test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE);
+    test_commit_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE, 901);
+    ASSERT_EQ(res.status().code(), MetaServiceCode::LOCK_EXPIRED);
+
+    // case 10:
+    // 10.1 lock key does not exist
+    remove_delete_bitmap_lock(meta_service.get(), table_id);
+    clear_rowsets(table_id);
+    // 10.2 compaction1 get lock in new way
+    config::use_delete_bitmap_lock_version = "v2";
+    res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 1001);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    // 10.3 compaction2 get and remove lock in old way
+    config::use_delete_bitmap_lock_version = "v1";
+    res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 1002);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE);
+    test_commit_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE, 1002);
+    ASSERT_EQ(res.status().code(), MetaServiceCode::OK);
+    // 10.4 load get and remove lock in old way
+    config::use_delete_bitmap_lock_version = "v1";
+    res_code = get_delete_bitmap_lock(meta_service.get(), table_id, 1910, -1);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, 1910, -1);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    // 10.5 compaction3 get lock in old way
+    config::use_delete_bitmap_lock_version = "v1";
+    res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 1003);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    // 10.6 compaction1 remove lock in new way, failed
+    config::use_delete_bitmap_lock_version = "v2";
+    test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE);
+    test_commit_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE, 1001);
+    ASSERT_EQ(res.status().code(), MetaServiceCode::LOCK_EXPIRED);
+
+    // case 11:
+    // 11.1 lock key does not exist
+    remove_delete_bitmap_lock(meta_service.get(), table_id);
+    clear_rowsets(table_id);
+    // 11.2 compaction1 get lock in new way
+    config::use_delete_bitmap_lock_version = "v2";
+    res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 1101);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    // 11.3 compaction2 get and remove lock in old way
+    config::use_delete_bitmap_lock_version = "v1";
+    res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 1102);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE);
+    test_commit_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE, 1102);
+    ASSERT_EQ(res.status().code(), MetaServiceCode::OK);
+    // 11.4 sc get and remove lock in old way
+    res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -2, -1);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, -2, -1);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    // 11.5 compaction3 get lock in new way
+    config::use_delete_bitmap_lock_version = "v2";
+    res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 1103);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    // 11.6 compaction1 remove lock in new way, failed
+    config::use_delete_bitmap_lock_version = "v2";
+    test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE);
+    test_commit_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE, 1101);
+    ASSERT_EQ(res.status().code(), MetaServiceCode::LOCK_EXPIRED);
+
+    // case 12:
+    // 12.1 lock key does not exist
+    remove_delete_bitmap_lock(meta_service.get(), table_id);
+    clear_rowsets(table_id);
+    // 12.2 compaction1 get lock in new way
+    config::use_delete_bitmap_lock_version = "v2";
+    res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 1201);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    // 12.3 compaction2 get and remove lock in old way
+    config::use_delete_bitmap_lock_version = "v1";
+    res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 1202);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE);
+    test_commit_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE, 1202);
+    ASSERT_EQ(res.status().code(), MetaServiceCode::OK);
+    // 12.4 sc get and remove lock in old way
+    config::use_delete_bitmap_lock_version = "v1";
+    res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -2, -1);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, -2, -1);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    // 12.5 compaction3 get lock in old way
+    config::use_delete_bitmap_lock_version = "v1";
+    res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 1203);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    // 12.6 compaction1 remove lock in new way, failed
+    config::use_delete_bitmap_lock_version = "v2";
+    test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE);
+    test_commit_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE, 1201);
+    ASSERT_EQ(res.status().code(), MetaServiceCode::LOCK_EXPIRED);
+
+    // case 13:
+    // 13.1 lock key does not exist
+    remove_delete_bitmap_lock(meta_service.get(), table_id);
+    clear_rowsets(table_id);
+    // 13.2 compaction1 get lock in new way
+    config::use_delete_bitmap_lock_version = "v2";
+    res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 1301);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    // 13.3 compaction2 get and remove lock in old way
+    config::use_delete_bitmap_lock_version = "v1";
+    res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 1302);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE);
+    test_commit_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE, 1302);
+    ASSERT_EQ(res.status().code(), MetaServiceCode::OK);
+    // 13.4 load get and remove lock in old way
+    config::use_delete_bitmap_lock_version = "v1";
+    res_code = get_delete_bitmap_lock(meta_service.get(), table_id, 1390, -1);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, 1390, -1);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    // 13.5 compaction3 get lock in new way
+    config::use_delete_bitmap_lock_version = "v2";
+    res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 1303);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    // 13.6 compaction1 remove lock in new way, failed
+    config::use_delete_bitmap_lock_version = "v2";
+    test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE);
+    test_commit_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE, 1301);
+    ASSERT_EQ(res.status().code(), MetaServiceCode::LOCK_EXPIRED);
+
+    // case 14:
+    // 14.1 lock key does not exist
+    remove_delete_bitmap_lock(meta_service.get(), table_id);
+    clear_rowsets(table_id);
+    // 14.2 compaction1 get lock in new way
+    config::use_delete_bitmap_lock_version = "v2";
+    res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 1401);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    // 14.3 compaction2 get and remove lock in old way
+    config::use_delete_bitmap_lock_version = "v1";
+    res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 1402);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE);
+    test_commit_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE, 1402);
+    ASSERT_EQ(res.status().code(), MetaServiceCode::OK);
+    // 14.4 load get and remove lock in old way
+    config::use_delete_bitmap_lock_version = "v1";
+    res_code = get_delete_bitmap_lock(meta_service.get(), table_id, 1490, -1);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, 1490, -1);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    // 14.5 compaction3 get lock in old way
+    config::use_delete_bitmap_lock_version = "v1";
+    res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 1403);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    // 14.6 compaction1 remove lock in old way, failed
+    config::use_delete_bitmap_lock_version = "v1";
+    test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE);
+    test_commit_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE, 1401);
+    ASSERT_EQ(res.status().code(), MetaServiceCode::LOCK_EXPIRED);
+
+    // case 15:
+    // 15.1 lock key does not exist
+    remove_delete_bitmap_lock(meta_service.get(), table_id);
+    clear_rowsets(table_id);
+    // 15.2 compaction1 get lock in new way
+    config::use_delete_bitmap_lock_version = "v2";
+    res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 1501);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    // 15.3 compaction2 get and remove lock in old way
+    config::use_delete_bitmap_lock_version = "v1";
+    res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 1502);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE);
+    test_commit_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE, 1502);
+    ASSERT_EQ(res.status().code(), MetaServiceCode::OK);
+    // 15.4 sc get and remove lock in old way
+    config::use_delete_bitmap_lock_version = "v1";
+    res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -2, -1);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, -2, -1);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    // 15.5 compaction3 get lock in new way
+    config::use_delete_bitmap_lock_version = "v2";
+    res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 1503);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    // 15.6 compaction1 remove lock in old way, failed
+    config::use_delete_bitmap_lock_version = "v1";
+    test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE);
+    test_commit_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE, 1501);
+    ASSERT_EQ(res.status().code(), MetaServiceCode::LOCK_EXPIRED);
+
+    // case 16:
+    // 16.1 lock key does not exist
+    remove_delete_bitmap_lock(meta_service.get(), table_id);
+    clear_rowsets(table_id);
+    // 16.2 compaction1 get lock in new way
+    config::use_delete_bitmap_lock_version = "v2";
+    res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 1601);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    // 16.3 compaction2 get and remove lock in old way
+    config::use_delete_bitmap_lock_version = "v1";
+    res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 1602);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE);
+    test_commit_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE, 1602);
+    ASSERT_EQ(res.status().code(), MetaServiceCode::OK);
+    // 16.4 sc get and remove lock in old way
+    config::use_delete_bitmap_lock_version = "v1";
+    res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -2, -1);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, -2, -1);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    // 16.5 compaction3 get lock in old way
+    config::use_delete_bitmap_lock_version = "v1";
+    res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 1603);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    // 16.6 compaction1 remove lock in old way, failed
+    config::use_delete_bitmap_lock_version = "v1";
+    test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE);
+    test_commit_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE, 1601);
+    ASSERT_EQ(res.status().code(), MetaServiceCode::LOCK_EXPIRED);
+
+    // case 17:
+    // 17.1 lock key does not exist
+    remove_delete_bitmap_lock(meta_service.get(), table_id);
+    clear_rowsets(table_id);
+    // 17.2 compaction1 get lock in old way
+    config::use_delete_bitmap_lock_version = "v1";
+    res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 1701);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    // 17.3 load get and remove lock in new way
+    config::use_delete_bitmap_lock_version = "v2";
+    res_code = get_delete_bitmap_lock(meta_service.get(), table_id, 1793, -1);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, 1793, -1);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    // 17.4 compaction2 get lock in new way
+    config::use_delete_bitmap_lock_version = "v2";
+    res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 1702);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    // 17.5 compaction1 remove lock in new way, failed
+    config::use_delete_bitmap_lock_version = "v2";
+    test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE);
+    test_commit_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE, 1701);
+    ASSERT_EQ(res.status().code(), MetaServiceCode::LOCK_EXPIRED);
+
+    // case 18:
+    // 18.1 lock key does not exist
+    remove_delete_bitmap_lock(meta_service.get(), table_id);
+    clear_rowsets(table_id);
+    // 18.2 compaction1 get lock in old way
+    config::use_delete_bitmap_lock_version = "v1";
+    res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 1801);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    // 18.3 load get and remove lock in new way
+    config::use_delete_bitmap_lock_version = "v2";
+    res_code = get_delete_bitmap_lock(meta_service.get(), table_id, 1893, -1);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, 1893, -1);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    // 18.4 compaction2 get lock in old way
+    config::use_delete_bitmap_lock_version = "v1";
+    res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 1802);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    // 18.5 compaction1 remove lock in new way, failed
+    config::use_delete_bitmap_lock_version = "v2";
+    test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE);
+    test_commit_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE, 1801);
+    ASSERT_EQ(res.status().code(), MetaServiceCode::LOCK_EXPIRED);
+
+    // case 19:
+    // 19.1 lock key does not exist
+    remove_delete_bitmap_lock(meta_service.get(), table_id);
+    clear_rowsets(table_id);
+    // 19.2 compaction1 get lock in old way
+    config::use_delete_bitmap_lock_version = "v1";
+    res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 1901);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    // 19.3 sc get and remove lock in new way
+    config::use_delete_bitmap_lock_version = "v2";
+    res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -2, -1);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, -2, -1);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    // 19.4 compaction2 get lock in new way
+    config::use_delete_bitmap_lock_version = "v2";
+    res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 1902);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    // 19.5 compaction1 remove lock in new way, failed
+    config::use_delete_bitmap_lock_version = "v2";
+    test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE);
+    test_commit_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE, 1901);
+    ASSERT_EQ(res.status().code(), MetaServiceCode::LOCK_EXPIRED);
+
+    // case 20:
+    // 20.1 lock key does not exist
+    remove_delete_bitmap_lock(meta_service.get(), table_id);
+    clear_rowsets(table_id);
+    // 20.2 compaction1 get lock in old way
+    config::use_delete_bitmap_lock_version = "v1";
+    res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 2001);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    // 20.3 sc get and remove lock in new way
+    config::use_delete_bitmap_lock_version = "v2";
+    res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -2, -1);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, -2, -1);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    // 20.4 compaction2 get lock in old way
+    config::use_delete_bitmap_lock_version = "v1";
+    res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 2002);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    // 20.5 compaction1 remove lock in new way, failed
+    config::use_delete_bitmap_lock_version = "v2";
+    test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE);
+    test_commit_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE, 2001);
+    ASSERT_EQ(res.status().code(), MetaServiceCode::LOCK_EXPIRED);
+
+    // case 21:
+    // 21.1 lock key does not exist
+    remove_delete_bitmap_lock(meta_service.get(), table_id);
+    clear_rowsets(table_id);
+    // 21.2 compaction1 get lock in old way
+    config::use_delete_bitmap_lock_version = "v1";
+    res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 2101);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    // 21.3 load get and remove lock in new way
+    config::use_delete_bitmap_lock_version = "v2";
+    res_code = get_delete_bitmap_lock(meta_service.get(), table_id, 2190, -1);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, 2190, -1);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    // 21.4 compaction2 get lock in new way
+    config::use_delete_bitmap_lock_version = "v2";
+    res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 2102);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    // 21.5 compaction1 remove lock in old way, failed
+    config::use_delete_bitmap_lock_version = "v1";
+    test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE);
+    test_commit_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE, 2101);
+    ASSERT_EQ(res.status().code(), MetaServiceCode::LOCK_EXPIRED);
+
+    // case 22:
+    // 22.1 lock key does not exist
+    remove_delete_bitmap_lock(meta_service.get(), table_id);
+    clear_rowsets(table_id);
+    // 22.2 compaction1 get lock in old way
+    config::use_delete_bitmap_lock_version = "v1";
+    res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 2201);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    // 22.3 load get and remove lock in new way
+    config::use_delete_bitmap_lock_version = "v2";
+    res_code = get_delete_bitmap_lock(meta_service.get(), table_id, 2290, -1);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, 2290, -1);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    // 22.4 compaction2 get lock in old way
+    config::use_delete_bitmap_lock_version = "v1";
+    res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 2202);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    // 22.5 compaction1 remove lock in old way, failed
+    config::use_delete_bitmap_lock_version = "v1";
+    test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE);
+    test_commit_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE, 2201);
+    ASSERT_EQ(res.status().code(), MetaServiceCode::LOCK_EXPIRED);
+
+    // case 23:
+    // 23.1 lock key does not exist
+    remove_delete_bitmap_lock(meta_service.get(), table_id);
+    clear_rowsets(table_id);
+    // 23.2 compaction1 get lock in old way
+    config::use_delete_bitmap_lock_version = "v1";
+    res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 2301);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    // 23.3 sc get and remove lock in new way
+    config::use_delete_bitmap_lock_version = "v2";
+    res_code = get_delete_bitmap_lock(meta_service.get(), table_id, 2390, -1);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, 2390, -1);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    // 23.4 compaction2 get lock in new way
+    config::use_delete_bitmap_lock_version = "v2";
+    res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 2302);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    // 23.5 compaction1 remove lock in old way, failed
+    config::use_delete_bitmap_lock_version = "v1";
+    test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE);
+    test_commit_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE, 2301);
+    ASSERT_EQ(res.status().code(), MetaServiceCode::LOCK_EXPIRED);
+
+    // case 24:
+    // 24.1 lock key does not exist
+    remove_delete_bitmap_lock(meta_service.get(), table_id);
+    clear_rowsets(table_id);
+    // 24.2 compaction1 get lock in old way
+    config::use_delete_bitmap_lock_version = "v1";
+    res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 2401);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    // 24.3 sc get and remove lock in new way
+    config::use_delete_bitmap_lock_version = "v2";
+    res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -2, -1);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, -2, -1);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    // 24.4 compaction2 get lock in old way
+    config::use_delete_bitmap_lock_version = "v1";
+    res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 2402);
+    ASSERT_EQ(res_code, MetaServiceCode::OK);
+    // 24.5 compaction1 remove lock in old way, failed
+    config::use_delete_bitmap_lock_version = "v1";
+    test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE);
+    test_commit_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE, 2401);
+    ASSERT_EQ(res.status().code(), MetaServiceCode::LOCK_EXPIRED);
+
+    // fuzzy case 1:
+    // 1.1 lock key does not exist
+    // 1.2 compaction1 get lock in old way
+    // 1.3 sc、load、compaction get and remove lock in new or old way
+    // 1.4 compaction2 get lock in old way
+    // 1.5 if sc or load succeed，compaction1 remove lock in old way failed
+
+    // fuzzy case 2:
+    // 2.1 lock key does not exist
+    // 2.2 compaction1 get lock in new way
+    // 2.3 sc、load、compaction get and remove lock in new or old way
+    // 2.4 compaction2 get lock in new way
+    // 2.5 if sc or load succeed，compaction1 remove lock in new way failed
+    table_id = 222;
+    for (int i = 1; i <= 2; i++) {
+        std::string lock_version = "v1";
+        if (i == 1) {
+            lock_version = "v1";
+        } else {
+            lock_version = "v2";
+        }
+        // 1 lock key does not exist
+        remove_delete_bitmap_lock(meta_service.get(), table_id);
+        clear_rowsets(table_id);
+        // 2 compaction1 get lock in old/new way
+        config::use_delete_bitmap_lock_version = lock_version;
+        res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 2501);
+        ASSERT_EQ(res_code, MetaServiceCode::OK);
+        // 3 sc、load、compaction get and remove lock in new or old way
+        bool load_or_sc_succeed = false;
+        std::srand(std::time(0));
+        for (int i = 0; i < 10; i++) {
+            int num = std::rand() % 3;
+            std::string use_version = (std::rand() % 2 == 0 ? "v2" : "v1");
+            config::use_delete_bitmap_lock_version = use_version;
+            LOG(INFO) << "i=" << i << ",num=" << num << ",use_version=" << use_version;
+            switch (num) {
+            case 0: {
+                res_code = get_delete_bitmap_lock(meta_service.get(), table_id, 2590, -1);
+                if (res_code == MetaServiceCode::OK) {
+                    res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, 2590, -1);
+                    if (res_code == MetaServiceCode::OK) {
+                        load_or_sc_succeed = true;
+                    }
+                }
+                LOG(INFO) << "i=" << i << ",load_or_sc_succeed=" << load_or_sc_succeed;
+                break;
+            }
+            case 1: {
+                res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -2, -1);
+                if (res_code == MetaServiceCode::OK) {
+                    res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, -2, -1);
+                    if (res_code == MetaServiceCode::OK) {
+                        load_or_sc_succeed = true;
+                    }
+                }
+                LOG(INFO) << "i=" << i << ",load_or_sc_succeed=" << load_or_sc_succeed;
+                break;
+            }
+            case 2: {
+                res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 6600 + i);
+                if (res_code == MetaServiceCode::OK) {
+                    test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE);
+                    test_commit_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE,
+                                               6600 + i);
+                }
+                break;
+            }
+            }
+        }
+        // 4 compaction2 get lock in old/new way
+        config::use_delete_bitmap_lock_version = lock_version;
+        res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 2502);
+        ASSERT_EQ(res_code, MetaServiceCode::OK);
+        // 5 if sc or load succeed，compaction1 remove lock in old/new way failed
+        config::use_delete_bitmap_lock_version = lock_version;
+        test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE);
+        test_commit_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE, 2501);
+        if (load_or_sc_succeed) {
+            ASSERT_EQ(res.status().code(), MetaServiceCode::LOCK_EXPIRED);
+        } else {
+            ASSERT_EQ(res.status().code(), MetaServiceCode::OK);
+        }
+    }
+}
+
 TEST(MetaServiceJobTest, CompactionJobWithMoWTest) {
     auto meta_service = get_meta_service();
     auto sp = SyncPoint::get_instance();


Reviews:
user: login: zhannngchen
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-19T06:48:49Z
user: login: zhannngchen
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-19T08:01:49Z
user: login: zhannngchen
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-19T08:05:52Z
user: login: hust-hhb
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-19T08:10:43Z
user: login: hust-hhb
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-02T01:50:28Z
user: login: zhannngchen
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-09T13:00:20Z
user: login: zhannngchen
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-09T13:49:05Z
user: login: zhannngchen
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-09T13:55:18Z
user: login: zhannngchen
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-16T15:13:16Z
user: login: zhannngchen
body: LGTM
state: DISMISSED
author_association: CONTRIBUTOR
submitted_at: 2025-04-16T15:26:30Z
user: login: zhannngchen
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-16T15:27:15Z
user: login: zhannngchen
body: LGTM
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-04-18T02:55:43Z
user: login: mymeiyi
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-04-18T09:33:43Z
user: login: dataroaring
body: LGTM
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-04-18T09:44:27Z

Review comments:
diff_hunk: @@ -264,4 +264,7 @@ CONF_Bool(enable_loopback_address_for_ms, "false");
 // Which vaults should be recycled. If empty, recycle all vaults.
 // Comma seprated list: recycler_storage_vault_white_list="aaa,bbb,ccc"
 CONF_Strings(recycler_storage_vault_white_list, "");
+
+CONF_mBool(use_delete_bitmap_lock_new_way, "true");
+CONF_mBool(use_delete_bitmap_lock_random_way, "true");
path: cloud/src/common/config.h
user: login: zhannngchen
body: The random strategy doesn’t seem to need a configuration; injection should be enough.
created_at: 2025-03-19T06:48:49Z
author_association: CONTRIBUTOR
diff_hunk: @@ -2645,7 +2659,193 @@ void MetaServiceImpl::get_delete_bitmap_update_lock(google::protobuf::RpcControl
     }
 }
 
-void MetaServiceImpl::remove_delete_bitmap_update_lock(
+void MetaServiceImpl::get_delete_bitmap_update_lock_old_way(
path: cloud/src/meta-service/meta_service.cpp
user: login: zhannngchen
body: use name v1 and v2 is better, maybe we'll have v3 in future?
created_at: 2025-03-19T08:01:49Z
author_association: CONTRIBUTOR
diff_hunk: @@ -275,6 +275,16 @@ class MetaServiceImpl : public cloud::MetaService {
                                        GetDeleteBitmapUpdateLockResponse* response,
                                        ::google::protobuf::Closure* done) override;
 
+    void get_delete_bitmap_update_lock_new_way(google::protobuf::RpcController* controller,
path: cloud/src/meta-service/meta_service.h
user: login: zhannngchen
body: the implementation of `check_delete_bitmap_lock` is also changed
created_at: 2025-03-19T08:05:52Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1775,7 +1775,7 @@ static bool check_delete_bitmap_lock(MetaServiceCode& code, std::string& msg, st
         code = MetaServiceCode::LOCK_EXPIRED;
         return false;
     }
-    if (lock_id == COMPACTION_DELETE_BITMAP_LOCK_ID) {
+    if (use_new_way && lock_id == COMPACTION_DELETE_BITMAP_LOCK_ID) {
path: cloud/src/meta-service/meta_service.cpp
user: login: hust-hhb
body: @zhannngchen  check_delete_bitmap_lock reserve the old way, so i just use this bool use_new_way to decide which branch to enter
created_at: 2025-03-19T08:10:43Z
author_association: CONTRIBUTOR
diff_hunk: @@ -264,4 +264,7 @@ CONF_Bool(enable_loopback_address_for_ms, "false");
 // Which vaults should be recycled. If empty, recycle all vaults.
 // Comma seprated list: recycler_storage_vault_white_list="aaa,bbb,ccc"
 CONF_Strings(recycler_storage_vault_white_list, "");
+
+CONF_mBool(use_delete_bitmap_lock_new_way, "true");
+CONF_mBool(use_delete_bitmap_lock_random_way, "true");
path: cloud/src/common/config.h
user: login: hust-hhb
body: In my test, injection like pr https://github.com/apache/doris/pull/47336 dosn't work in normal case
created_at: 2025-04-02T01:50:28Z
author_association: CONTRIBUTOR
diff_hunk: @@ -2305,10 +2325,12 @@ static bool put_delete_bitmap_update_lock_key(MetaServiceCode& code, std::string
     return true;
 }
 
-void MetaServiceImpl::get_delete_bitmap_update_lock(google::protobuf::RpcController* controller,
-                                                    const GetDeleteBitmapUpdateLockRequest* request,
-                                                    GetDeleteBitmapUpdateLockResponse* response,
-                                                    ::google::protobuf::Closure* done) {
+void MetaServiceImpl::get_delete_bitmap_update_lock_v2(
+        google::protobuf::RpcController* controller,
+        const GetDeleteBitmapUpdateLockRequest* request,
+        GetDeleteBitmapUpdateLockResponse* response, ::google::protobuf::Closure* done) {
+    LOG(INFO) << "get delete bitmap update lock in v2 for table=" << request->table_id()
path: cloud/src/meta-service/meta_service.cpp
user: login: zhannngchen
body: The INFO level logs might be too much？
created_at: 2025-04-09T13:00:19Z
author_association: CONTRIBUTOR
diff_hunk: @@ -2673,6 +2885,19 @@ void MetaServiceImpl::remove_delete_bitmap_update_lock(
         msg = "failed to init txn";
         return;
     }
+    std::string lock_key =
+            meta_delete_bitmap_update_lock_key({instance_id, request->table_id(), -1});
+    std::string lock_val;
+    DeleteBitmapUpdateLockPB lock_info;
+    if (!check_delete_bitmap_lock(code, msg, ss, txn, instance_id, request->table_id(),
path: cloud/src/meta-service/meta_service.cpp
user: login: zhannngchen
body: Why do we need to add a lock check here?
created_at: 2025-04-09T13:49:02Z
author_association: CONTRIBUTOR
diff_hunk: @@ -4830,6 +4830,1873 @@ TEST(MetaServiceTest, GetDeleteBitmapUpdateLock) {
     remove_delete_bitmap_lock(meta_service.get(), 2);
 }
 
+TEST(MetaServiceTest, DeleteBitmapUpdateLockCompatibilityTest) {
+    config::use_delete_bitmap_lock_random_version = false;
+    auto meta_service = get_meta_service();
+    [[maybe_unused]] auto sp = SyncPoint::get_instance();
+    std::unique_ptr<int, std::function<void(int*)>> defer(
+            (int*)0x01, [](int*) { SyncPoint::get_instance()->clear_all_call_backs(); });
+    sp->set_call_back("get_delete_bitmap_update_lock:commit:conflict", [&](auto&& args) {
+        auto* first_retry = try_any_cast<bool*>(args[0]);
+        if (*first_retry) {
+            *try_any_cast<TxnErrorCode*>(args[1]) = TxnErrorCode::TXN_CONFLICT;
+        } else {
+            *try_any_cast<TxnErrorCode*>(args[1]) = TxnErrorCode::TXN_OK;
+        }
+    });
+
+    int64_t table_id = 555;
+    // case 1: lock key does not exist, get and remove load lock in new way, success
+    config::use_delete_bitmap_lock_version = "v2";
+    brpc::Controller cntl;
+    GetDeleteBitmapUpdateLockRequest load_req;
+    GetDeleteBitmapUpdateLockResponse load_res;
+    load_req.set_cloud_unique_id("test_cloud_unique_id");
path: cloud/test/meta_service_test.cpp
user: login: zhannngchen
body: most of the request is duplicate code, why don't you abstract some methods? like `get_delete_bitmap_lock(int lock_id);`
created_at: 2025-04-09T13:55:18Z
author_association: CONTRIBUTOR
diff_hunk: @@ -275,6 +275,16 @@ class MetaServiceImpl : public cloud::MetaService {
                                        GetDeleteBitmapUpdateLockResponse* response,
                                        ::google::protobuf::Closure* done) override;
 
+    void get_delete_bitmap_update_lock_v2(google::protobuf::RpcController* controller,
path: cloud/src/meta-service/meta_service.h
user: login: zhannngchen
body: all these 4 methods should be private.
created_at: 2025-04-16T15:13:14Z
author_association: CONTRIBUTOR
diff_hunk: @@ -261,6 +261,11 @@ CONF_Bool(enable_check_instance_id, "true");
 
 // Check if ip eq 127.0.0.1, ms/recycler exit
 CONF_Bool(enable_loopback_address_for_ms, "false");
+
+// delete_bitmap_lock version config
+CONF_mString(use_delete_bitmap_lock_version, "v1");
+CONF_mBool(use_delete_bitmap_lock_random_version, "false");
path: cloud/src/common/config.h
user: login: zhannngchen
body: ```suggestion
// FOR DEBUGGING
CONF_mBool(use_delete_bitmap_lock_random_version, "false");
```
created_at: 2025-04-16T15:27:15Z
author_association: CONTRIBUTOR

PR comments:
user: login: Thearas
created_at: 2025-03-18T14:00:22Z
body: 
Thank you for your contribution to Apache Doris.
Don't know what should be done next? See [How to process your PR](https://cwiki.apache.org/confluence/display/DORIS/How+to+process+your+PR).

Please clearly describe your PR:
1. What problem was fixed (it's best to include specific error reporting information). How it was fixed.
2. Which behaviors were modified. What was the previous behavior, what is it now, why was it modified, and what possible impacts might there be.
3. What features were added. Why was this function added?
4. Which code was refactored and why was this part of the code refactored?
5. Which functions were optimized and what is the difference before and after the optimization?
    
author_association: CONTRIBUTOR
user: login: hust-hhb
created_at: 2025-03-18T14:00:27Z
body: run buildall
author_association: CONTRIBUTOR
user: login: doris-robot
created_at: 2025-03-18T14:10:55Z
body: TeamCity cloud ut coverage result:
 Function Coverage: 82.86% (1083/1307) 
 Line Coverage: 65.41% (18023/27553)
 Region Coverage: 64.72% (8887/13731)
 Branch Coverage: 54.49% (4779/8770)
 Coverage Report: http://coverage.selectdb-in.cc/coverage/afd702d0a8241f4fca09356a4f29295928f2be3c_afd702d0a8241f4fca09356a4f29295928f2be3c_cloud/report/index.html
author_association: NONE
user: login: hust-hhb
created_at: 2025-03-19T02:00:08Z
body: run buildall
author_association: CONTRIBUTOR
user: login: hust-hhb
created_at: 2025-04-02T01:52:25Z
body: run buildall
author_association: CONTRIBUTOR
user: login: doris-robot
created_at: 2025-04-02T02:04:26Z
body: TeamCity cloud ut coverage result:
 Function Coverage: 83.05% (1093/1316) 
 Line Coverage: 65.92% (18274/27720)
 Region Coverage: 65.39% (9020/13795)
 Branch Coverage: 55.15% (4860/8812)
 Coverage Report: http://coverage.selectdb-in.cc/coverage/4c15e53b1af343c891c55d6987998e3fc688ae96_4c15e53b1af343c891c55d6987998e3fc688ae96_cloud/report/index.html
author_association: NONE
user: login: cosven
created_at: 2025-04-07T13:05:22Z
body: Please rebase so we can run the correctness test again. @hust-hhb 
author_association: CONTRIBUTOR
user: login: hust-hhb
created_at: 2025-04-14T07:24:46Z
body: run buildall
author_association: CONTRIBUTOR
user: login: doris-robot
created_at: 2025-04-14T07:45:58Z
body: TeamCity cloud ut coverage result:
 Function Coverage: 83.00% (1094/1318) 
 Line Coverage: 65.72% (18276/27810)
 Region Coverage: 65.11% (9049/13897)
 Branch Coverage: 55.16% (4878/8844)
 Coverage Report: http://coverage.selectdb-in.cc/coverage/32e9e07e2e8b6ee3019f9d43785d6a3559535676_32e9e07e2e8b6ee3019f9d43785d6a3559535676_cloud/report/index.html
author_association: NONE
user: login: hust-hhb
created_at: 2025-04-16T03:35:19Z
body: run buildall
author_association: CONTRIBUTOR
user: login: doris-robot
created_at: 2025-04-16T03:44:22Z
body: TeamCity cloud ut coverage result:
 Function Coverage: 82.93% (1093/1318) 
 Line Coverage: 65.64% (18257/27814)
 Region Coverage: 65.02% (9039/13902)
 Branch Coverage: 55.03% (4869/8848)
 Coverage Report: http://coverage.selectdb-in.cc/coverage/7682d7de8cb76b95644d9283cdf39431270a1e12_7682d7de8cb76b95644d9283cdf39431270a1e12_cloud/report/index.html
author_association: NONE
user: login: github-actions[bot]
created_at: 2025-04-16T15:27:29Z
body: PR approved by at least one committer and no changes requested.
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-04-16T15:27:33Z
body: PR approved by anyone and no changes requested.
author_association: CONTRIBUTOR
user: login: hust-hhb
created_at: 2025-04-17T02:31:11Z
body: run buildall
author_association: CONTRIBUTOR
user: login: doris-robot
created_at: 2025-04-17T02:43:07Z
body: TeamCity cloud ut coverage result:
 Function Coverage: 82.93% (1093/1318) 
 Line Coverage: 65.67% (18265/27814)
 Region Coverage: 65.06% (9044/13902)
 Branch Coverage: 55.00% (4866/8848)
 Coverage Report: http://coverage.selectdb-in.cc/coverage/90707f8403bd0f25a570b9d0b94cd727c0b86a07_90707f8403bd0f25a570b9d0b94cd727c0b86a07_cloud/report/index.html
author_association: NONE
user: login: github-actions[bot]
created_at: 2025-04-18T02:56:37Z
body: PR approved by at least one committer and no changes requested.
author_association: CONTRIBUTOR