Code review comment: The most risky bug in this code is: Potential race conditions due to the lack of atomicity between checking job state (`isTxnDone`) and performing operations based on that check.  You can modify the code like this: ```java @@ -331,6 +334,7 @@ private void createLoadingTask(Database db, BrokerPendingTaskAttachment attachme          // Lock should be moved up before if (isTxnDone()) check to ensure atomicity. +       writeLock();          try {              // check if job has been completed              if (isTxnDone()) { @@ -341,7 +345,6 @@ private void createLoadingTask(Database db, BrokerPendingTaskAttachment attachme                  unprotectUpdateLoadingStatus(txnState);                  // cancel load job                  unprotectedExecuteCancel(new FailMsg(FailMsg.CancelType.LOAD_RUN_FAIL, txnStatusChangeReason), true); -                writeLock(); // This line is removed because we've moved the lock earlier.                  return;              } ``` This modification ensures that once a job's state is checked, the subsequent operations are performed atomically with respect to that state. Moving the `writeLock()` call before the `if (isTxnDone())` statement ensures that no other thread can interleave and change the state of the job between the check and the action. Additionally, the redundant `writeLock()` inside the cancellation path (after calling `unprotectedExecuteCancel`) is removed to prevent incorrect re-locking.

Code under review:
private void onLoadingTaskFinished(BrokerLoadingTaskAttachment attachment) {
        writeLock();
        try {
            
            if (isTxnDone()) {
                LOG.warn(new LogBuilder(LogKey.LOAD_JOB, id)
                        .add("state", state)
                        .add("error_msg", "this task will be ignored when job is: " + state)
                        .build());
                return;
            }

            
            if (finishedTaskIds.contains(attachment.getTaskId())) {
                LOG.warn(new LogBuilder(LogKey.LOAD_JOB, id)
                        .add("task_id", attachment.getTaskId())
                        .add("error_msg", "this is a duplicated callback of loading task").build());
                return;
            }

            
            finishedTaskIds.add(attachment.getTaskId());
            updateLoadingStatus(attachment);

            
            if (finishedTaskIds.size() != idToTasks.size()) {
                return;
            }
        } finally {
            writeUnlock();
        }

        if (LOG.isDebugEnabled()) {
            LOG.debug(new LogBuilder(LogKey.LOAD_JOB, id)
                    .add("commit_infos", Joiner.on(",").join(commitInfos))
                    .build());
        }

        
        if (!checkDataQuality()) {
            cancelJobWithoutCheck(
                    new FailMsg(FailMsg.CancelType.ETL_QUALITY_UNSATISFIED,
                            DataQualityException.QUALITY_FAIL_MSG +
                                    ". You can find detailed error message from running `TrackingSQL`."),
                    true, true);
            return;
        }
        Database db = null;
        try {
            db = getDb();
        } catch (MetaNotFoundException e) {
            LOG.warn(new LogBuilder(LogKey.LOAD_JOB, id)
                    .add("database_id", dbId)
                    .add("error_msg", "db has been deleted when job is loading")
                    .build(), e);
            cancelJobWithoutCheck(new FailMsg(FailMsg.CancelType.LOAD_RUN_FAIL, e.getMessage()), true, true);
            return;
        }
        while (true) {
            try {
                commitTransactionUnderDatabaseLock(db);
                break;
            } catch (CommitRateExceededException e) {
                
                ThreadUtil.sleepAtLeastIgnoreInterrupts(Math.max(e.getAllowCommitTime() - System.currentTimeMillis(), 0));
            } catch (UserException e) {
                LOG.warn(new LogBuilder(LogKey.LOAD_JOB, id)
                        .add("database_id", dbId)
                        .add("error_msg", "Failed to commit txn with error:" + e.getMessage())
                        .build(), e);
                cancelJobWithoutCheck(new FailMsg(FailMsg.CancelType.LOAD_RUN_FAIL, e.getMessage()), true, true);
                break;
            }
        }
    }

Line commented on: try {

Revised code:
private void onLoadingTaskFinished(BrokerLoadingTaskAttachment attachment) {
        writeLock();
        try {
            
            if (isTxnDone()) {
                LOG.warn(new LogBuilder(LogKey.LOAD_JOB, id)
                        .add("state", state)
                        .add("error_msg", "this task will be ignored when job is: " + state)
                        .build());
                return;
            }

            
            if (finishedTaskIds.contains(attachment.getTaskId())) {
                LOG.warn(new LogBuilder(LogKey.LOAD_JOB, id)
                        .add("task_id", attachment.getTaskId())
                        .add("error_msg", "this is a duplicated callback of loading task").build());
                return;
            }

            
            finishedTaskIds.add(attachment.getTaskId());
            updateLoadingStatus(attachment);

            
            if (finishedTaskIds.size() != idToTasks.size()) {
                return;
            }
        } finally {
            writeUnlock();
        }

        if (LOG.isDebugEnabled()) {
            LOG.debug(new LogBuilder(LogKey.LOAD_JOB, id)
                    .add("commit_infos", Joiner.on(",").join(commitInfos))
                    .build());
        }

        
        if (!checkDataQuality()) {
            cancelJobWithoutCheck(
                    new FailMsg(FailMsg.CancelType.ETL_QUALITY_UNSATISFIED,
                            DataQualityException.QUALITY_FAIL_MSG +
                                    ". You can find detailed error message from running `TrackingSQL`."),
                    true, true);
            return;
        }
        Database db = null;
        try {
            db = getDb();
        } catch (MetaNotFoundException e) {
            LOG.warn(new LogBuilder(LogKey.LOAD_JOB, id)
                    .add("database_id", dbId)
                    .add("error_msg", "db has been deleted when job is loading")
                    .build(), e);
            cancelJobWithoutCheck(new FailMsg(FailMsg.CancelType.LOAD_RUN_FAIL, e.getMessage()), true, true);
            return;
        }
        while (true) {
            try {
                commitTransactionUnderDatabaseLock(db);
                break;
            } catch (CommitRateExceededException e) {
                
                ThreadUtil.sleepAtLeastIgnoreInterrupts(Math.max(e.getAllowCommitTime() - System.currentTimeMillis(), 0));
            } catch (UserException e) {
                cancelJobWithoutCheck(new FailMsg(FailMsg.CancelType.LOAD_RUN_FAIL, e.getMessage()), true, true);
                break;
            }
        }
    }