Repository: apache/doris
Pull request #49753: [refactor](pipeline) Re-construct ownership of pipeline components
Author: login: Gabriel39
Labels: name: approved
description: Indicates a PR has been approved by one committer.
name: reviewed
name: dev/3.1.0-merged
name: dev/3.0.8-merged
Created: 2025-04-02T02:41:19Z | Merged: 2025-04-02T12:57:41Z

### What problem does this PR solve?

In this PR, ownership of core components in pipeline engine is re-constructed following the rules below:
![image](https://github.com/user-attachments/assets/04f842b7-f052-4e3c-9005-7df3103b4d7b)


### Check List (For Author)

- Test <!-- At least one of them must be included. -->
    - [ ] Regression test
    - [ ] Unit Test
    - [ ] Manual test (add detailed scripts or steps below)
    - [ ] No need to test or manual test. Explain why:
        - [ ] This is a refactor/code format and no logic has been changed.
        - [ ] Previous test can cover this change.
        - [ ] No code files have been changed.
        - [ ] Other reason <!-- Add your reason?  -->

- Behavior changed:
    - [ ] No.
    - [ ] Yes. <!-- Explain the behavior change -->

- Does this need documentation?
    - [ ] No.
    - [ ] Yes. <!-- Add document PR link here. eg: https://github.com/apache/doris-website/pull/1214 -->

### Check List (For Reviewer who merge this PR)

- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR should merge into -->




Diff:
diff --git a/be/src/pipeline/dependency.cpp b/be/src/pipeline/dependency.cpp
index dcecee295dedc3..6266bfb9ef8f1c 100644
--- a/be/src/pipeline/dependency.cpp
+++ b/be/src/pipeline/dependency.cpp
@@ -57,8 +57,9 @@ Dependency* BasicSharedState::create_sink_dependency(int dest_id, int node_id,
     return sink_deps.back().get();
 }
 
-void Dependency::_add_block_task(PipelineTask* task) {
-    DCHECK(_blocked_task.empty() || _blocked_task[_blocked_task.size() - 1] != task)
+void Dependency::_add_block_task(std::shared_ptr<PipelineTask> task) {
+    DCHECK(_blocked_task.empty() || _blocked_task[_blocked_task.size() - 1].lock() == nullptr ||
+           _blocked_task[_blocked_task.size() - 1].lock().get() != task.get())
             << "Duplicate task: " << task->debug_string();
     _blocked_task.push_back(task);
 }
@@ -68,7 +69,7 @@ void Dependency::set_ready() {
         return;
     }
     _watcher.stop();
-    std::vector<PipelineTask*> local_block_task {};
+    std::vector<std::weak_ptr<PipelineTask>> local_block_task {};
     {
         std::unique_lock<std::mutex> lc(_task_lock);
         if (_ready) {
@@ -77,13 +78,15 @@ void Dependency::set_ready() {
         _ready = true;
         local_block_task.swap(_blocked_task);
     }
-    for (auto* task : local_block_task) {
-        std::unique_lock<std::mutex> lc(_task_lock);
-        THROW_IF_ERROR(task->wake_up(this));
+    for (auto task : local_block_task) {
+        if (auto t = task.lock()) {
+            std::unique_lock<std::mutex> lc(_task_lock);
+            THROW_IF_ERROR(t->wake_up(this));
+        }
     }
 }
 
-Dependency* Dependency::is_blocked_by(PipelineTask* task) {
+Dependency* Dependency::is_blocked_by(std::shared_ptr<PipelineTask> task) {
     std::unique_lock<std::mutex> lc(_task_lock);
     auto ready = _ready.load();
     if (!ready && task) {
diff --git a/be/src/pipeline/dependency.h b/be/src/pipeline/dependency.h
index aab39cb01a685c..e483e2a2a6b6a7 100644
--- a/be/src/pipeline/dependency.h
+++ b/be/src/pipeline/dependency.h
@@ -111,7 +111,7 @@ class Dependency : public std::enable_shared_from_this<Dependency> {
     [[nodiscard]] int64_t watcher_elapse_time() { return _watcher.elapsed_time(); }
 
     // Which dependency current pipeline task is blocked by. `nullptr` if this dependency is ready.
-    [[nodiscard]] Dependency* is_blocked_by(PipelineTask* task = nullptr);
+    [[nodiscard]] Dependency* is_blocked_by(std::shared_ptr<PipelineTask> task = nullptr);
     // Notify downstream pipeline tasks this dependency is ready.
     virtual void set_ready();
     void set_ready_to_read(int channel_id = 0) {
@@ -148,7 +148,7 @@ class Dependency : public std::enable_shared_from_this<Dependency> {
     }
 
 protected:
-    void _add_block_task(PipelineTask* task);
+    void _add_block_task(std::shared_ptr<PipelineTask> task);
 
     const int _id;
     const int _node_id;
@@ -159,7 +159,7 @@ class Dependency : public std::enable_shared_from_this<Dependency> {
     MonotonicStopWatch _watcher;
 
     std::mutex _task_lock;
-    std::vector<PipelineTask*> _blocked_task;
+    std::vector<std::weak_ptr<PipelineTask>> _blocked_task;
 
     // If `_always_ready` is true, `block()` will never block tasks.
     std::atomic<bool> _always_ready = false;
diff --git a/be/src/pipeline/pipeline_fragment_context.cpp b/be/src/pipeline/pipeline_fragment_context.cpp
index 0acfd1d216ee8b..e141113df148c9 100644
--- a/be/src/pipeline/pipeline_fragment_context.cpp
+++ b/be/src/pipeline/pipeline_fragment_context.cpp
@@ -139,18 +139,17 @@ PipelineFragmentContext::~PipelineFragmentContext() {
     // The memory released by the query end is recorded in the query mem tracker.
     SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(_query_ctx->query_mem_tracker());
     auto st = _query_ctx->exec_status();
-    _query_ctx.reset();
     for (size_t i = 0; i < _tasks.size(); i++) {
         if (!_tasks[i].empty()) {
             _call_back(_tasks[i].front()->runtime_state(), &st);
         }
     }
-    _tasks.clear();
     for (auto& runtime_states : _task_runtime_states) {
         for (auto& runtime_state : runtime_states) {
             runtime_state.reset();
         }
     }
+    _tasks.clear();
     _dag.clear();
     _pip_id_to_pipeline.clear();
     _pipelines.clear();
@@ -160,6 +159,7 @@ PipelineFragmentContext::~PipelineFragmentContext() {
     _runtime_filter_states.clear();
     _runtime_filter_mgr_map.clear();
     _op_id_to_shared_state.clear();
+    _query_ctx.reset();
 }
 
 bool PipelineFragmentContext::is_timeout(timespec now) const {
@@ -457,8 +457,9 @@ Status PipelineFragmentContext::_build_pipeline_tasks(const doris::TPipelineFrag
                 auto cur_task_id = _total_tasks++;
                 task_runtime_state->set_task_id(cur_task_id);
                 task_runtime_state->set_task_num(pipeline->num_tasks());
-                auto task = std::make_unique<PipelineTask>(
-                        pipeline, cur_task_id, task_runtime_state.get(), this,
+                auto task = std::make_shared<PipelineTask>(
+                        pipeline, cur_task_id, task_runtime_state.get(),
+                        std::dynamic_pointer_cast<PipelineFragmentContext>(shared_from_this()),
                         pipeline_id_to_profile[pip_idx].get(), get_shared_state(pipeline), i);
                 pipeline->incr_created_tasks(i, task.get());
                 pipeline_id_to_task.insert({pipeline->id(), task.get()});
@@ -1697,7 +1698,7 @@ Status PipelineFragmentContext::submit() {
     auto* scheduler = _query_ctx->get_pipe_exec_scheduler();
     for (auto& task : _tasks) {
         for (auto& t : task) {
-            st = scheduler->schedule_task(t.get());
+            st = scheduler->schedule_task(t);
             if (!st) {
                 cancel(Status::InternalError("submit context to executor fail"));
                 std::lock_guard<std::mutex> l(_task_mutex);
diff --git a/be/src/pipeline/pipeline_fragment_context.h b/be/src/pipeline/pipeline_fragment_context.h
index 9306721cf8d5ae..08d988499bfe57 100644
--- a/be/src/pipeline/pipeline_fragment_context.h
+++ b/be/src/pipeline/pipeline_fragment_context.h
@@ -236,7 +236,7 @@ class PipelineFragmentContext : public TaskExecutionContext {
 
     OperatorPtr _root_op = nullptr;
     // this is a [n * m] matrix. n is parallelism of pipeline engine and m is the number of pipelines.
-    std::vector<std::vector<std::unique_ptr<PipelineTask>>> _tasks;
+    std::vector<std::vector<std::shared_ptr<PipelineTask>>> _tasks;
 
     // TODO: remove the _sink and _multi_cast_stream_sink_senders to set both
     // of it in pipeline task not the fragment_context
diff --git a/be/src/pipeline/pipeline_task.cpp b/be/src/pipeline/pipeline_task.cpp
index 14f6bdbfc6c8b2..ed5b5f74fdc7df 100644
--- a/be/src/pipeline/pipeline_task.cpp
+++ b/be/src/pipeline/pipeline_task.cpp
@@ -54,13 +54,19 @@ class RuntimeState;
 namespace doris::pipeline {
 
 PipelineTask::PipelineTask(PipelinePtr& pipeline, uint32_t task_id, RuntimeState* state,
-                           PipelineFragmentContext* fragment_context,
+                           std::shared_ptr<PipelineFragmentContext> fragment_context,
                            RuntimeProfile* parent_profile,
                            std::map<int, std::pair<std::shared_ptr<BasicSharedState>,
                                                    std::vector<std::shared_ptr<Dependency>>>>
                                    shared_state_map,
                            int task_idx)
-        : _index(task_id),
+        :
+#ifdef BE_TEST
+          _query_id(fragment_context ? fragment_context->get_query_id() : TUniqueId()),
+#else
+          _query_id(fragment_context->get_query_id()),
+#endif
+          _index(task_id),
           _pipeline(pipeline),
           _opened(false),
           _state(state),
@@ -121,8 +127,10 @@ Status PipelineTask::prepare(const std::vector<TScanRangeParams>& scan_range, co
         std::unique_lock<std::mutex> lc(_dependency_lock);
         filter_dependencies.swap(_filter_dependencies);
     }
-    if (query_context()->is_cancelled()) {
-        terminate();
+    if (auto fragment = _fragment_context.lock()) {
+        if (fragment->get_query_ctx()->is_cancelled()) {
+            terminate();
+        }
     }
     _block = doris::vectorized::Block::create_unique();
     return _state_transition(State::RUNNABLE);
@@ -233,12 +241,12 @@ bool PipelineTask::_wait_to_start() {
     // Before task starting, we should make sure
     // 1. Execution dependency is ready (which is controlled by FE 2-phase commit)
     // 2. Runtime filter dependencies are ready
-    if (_execution_dep->is_blocked_by(this)) {
+    if (_execution_dep->is_blocked_by(shared_from_this())) {
         return true;
     }
 
     for (auto* dep : _filter_dependencies) {
-        if (dep->is_blocked_by(this)) {
+        if (dep->is_blocked_by(shared_from_this())) {
             return true;
         }
     }
@@ -248,12 +256,12 @@ bool PipelineTask::_wait_to_start() {
 bool PipelineTask::_is_pending_finish() {
     // Spilling may be in progress if eos is true.
     for (auto* dep : _spill_dependencies) {
-        if (dep->is_blocked_by(this)) {
+        if (dep->is_blocked_by(shared_from_this())) {
             return true;
         }
     }
     for (auto* dep : _finish_dependencies) {
-        if (dep->is_blocked_by(this)) {
+        if (dep->is_blocked_by(shared_from_this())) {
             return true;
         }
     }
@@ -262,11 +270,11 @@ bool PipelineTask::_is_pending_finish() {
 
 bool PipelineTask::_is_blocked() {
     for (auto* dep : _spill_dependencies) {
-        if (dep->is_blocked_by(this)) {
+        if (dep->is_blocked_by(shared_from_this())) {
             return true;
         }
     }
-    if (_memory_sufficient_dependency->is_blocked_by(this)) {
+    if (_memory_sufficient_dependency->is_blocked_by(shared_from_this())) {
         return true;
     }
     // `_dry_run = true` means we do not need data from source operator.
@@ -274,7 +282,7 @@ bool PipelineTask::_is_blocked() {
         for (int i = _read_dependencies.size() - 1; i >= 0; i--) {
             // `_read_dependencies` is organized according to operators. For each operator, running condition is met iff all dependencies are ready.
             for (auto* dep : _read_dependencies[i]) {
-                if (dep->is_blocked_by(this)) {
+                if (dep->is_blocked_by(shared_from_this())) {
                     return true;
                 }
             }
@@ -285,7 +293,7 @@ bool PipelineTask::_is_blocked() {
         }
     }
     for (auto* dep : _write_dependencies) {
-        if (dep->is_blocked_by(this)) {
+        if (dep->is_blocked_by(shared_from_this())) {
             return true;
         }
     }
@@ -295,8 +303,9 @@ bool PipelineTask::_is_blocked() {
 void PipelineTask::terminate() {
     // We use a lock to assure all dependencies are not deconstructed here.
     std::unique_lock<std::mutex> lc(_dependency_lock);
-    if (!is_finalized()) {
-        DCHECK(_wake_up_early || _fragment_context->is_canceled());
+    auto fragment = _fragment_context.lock();
+    if (!is_finalized() && fragment) {
+        DCHECK(_wake_up_early || fragment->is_canceled());
         for (auto* dep : _spill_dependencies) {
             dep->set_always_ready();
         }
@@ -338,6 +347,8 @@ void PipelineTask::terminate() {
 Status PipelineTask::execute(bool* done) {
     DCHECK(_exec_state == State::RUNNABLE) << debug_string();
     DCHECK(_blocked_dep == nullptr) << debug_string();
+    auto fragment_context = _fragment_context.lock();
+    DCHECK(fragment_context);
     int64_t time_spent = 0;
     ThreadCpuStopWatch cpu_time_stop_watch;
     cpu_time_stop_watch.start();
@@ -347,7 +358,8 @@ Status PipelineTask::execute(bool* done) {
         }
         int64_t delta_cpu_time = cpu_time_stop_watch.elapsed_time();
         _task_cpu_timer->update(delta_cpu_time);
-        query_context()->resource_ctx()->cpu_context()->update_cpu_cost_ms(delta_cpu_time);
+        fragment_context->get_query_ctx()->resource_ctx()->cpu_context()->update_cpu_cost_ms(
+                delta_cpu_time);
 
         // If task is woke up early, we should terminate all operators, and this task could be closed immediately.
         if (_wake_up_early) {
@@ -357,7 +369,7 @@ Status PipelineTask::execute(bool* done) {
             _eos = true;
             *done = true;
         } else if (_eos && !_spilling &&
-                   (_fragment_context->is_canceled() || !_is_pending_finish())) {
+                   (fragment_context->is_canceled() || !_is_pending_finish())) {
             *done = true;
         }
         // If this run is pended by a spilling request, the block will be output in next run.
@@ -396,7 +408,7 @@ Status PipelineTask::execute(bool* done) {
     }
 
     // The status must be runnable
-    if (!_opened && !_fragment_context->is_canceled()) {
+    if (!_opened && !fragment_context->is_canceled()) {
         DBUG_EXECUTE_IF("PipelineTask::execute.open_sleep", {
             auto required_pipeline_id =
                     DebugPoints::instance()->get_debug_param_or_default<int32_t>(
@@ -412,7 +424,7 @@ Status PipelineTask::execute(bool* done) {
         RETURN_IF_ERROR(_open());
     }
 
-    while (!_fragment_context->is_canceled()) {
+    while (!fragment_context->is_canceled()) {
         Defer defer {[&]() {
             // If this run is pended by a spilling request, the block will be output in next run.
             if (!_spilling) {
@@ -427,7 +439,7 @@ Status PipelineTask::execute(bool* done) {
         /// When a task is cancelled,
         /// its blocking state will be cleared and it will transition to a ready state (though it is not truly ready).
         /// Here, checking whether it is cancelled to prevent tasks in a blocking state from being re-executed.
-        if (_fragment_context->is_canceled()) {
+        if (fragment_context->is_canceled()) {
             break;
         }
 
@@ -595,16 +607,25 @@ Status PipelineTask::execute(bool* done) {
         }
     }
 
-    RETURN_IF_ERROR(get_task_queue()->push_back(this));
+    RETURN_IF_ERROR(get_task_queue()->push_back(shared_from_this()));
     return Status::OK();
 }
 
 Status PipelineTask::finalize() {
+    auto fragment = _fragment_context.lock();
+    if (!fragment) {
+        return Status::OK();
+    }
+    SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(fragment->get_query_ctx()->query_mem_tracker());
     std::unique_lock<std::mutex> lc(_dependency_lock);
     RETURN_IF_ERROR(_state_transition(State::FINALIZED));
     _sink_shared_state.reset();
     _op_shared_states.clear();
     _shared_state_map.clear();
+    _block.reset();
+    _operators.clear();
+    _sink.reset();
+    _pipeline.reset();
     return Status::OK();
 }
 
@@ -645,22 +666,28 @@ Status PipelineTask::close(Status exec_status, bool close_sink) {
 std::string PipelineTask::debug_string() {
     fmt::memory_buffer debug_string_buffer;
 
-    fmt::format_to(debug_string_buffer, "QueryId: {}\n", print_id(query_context()->query_id()));
+    fmt::format_to(debug_string_buffer, "QueryId: {}\n", print_id(_query_id));
     fmt::format_to(debug_string_buffer, "InstanceId: {}\n",
                    print_id(_state->fragment_instance_id()));
 
+    fmt::format_to(debug_string_buffer,
+                   "PipelineTask[this = {}, id = {}, open = {}, eos = {}, state = {}, dry run = "
+                   "{}, _wake_up_early = {}, time elapsed since last state changing = {}s, spilling"
+                   " = {}, is running = {}]",
+                   (void*)this, _index, _opened, _eos, _to_string(_exec_state), _dry_run,
+                   _wake_up_early.load(), _state_change_watcher.elapsed_time() / NANOS_PER_SEC,
+                   _spilling, is_running());
     std::unique_lock<std::mutex> lc(_dependency_lock);
     auto* cur_blocked_dep = _blocked_dep;
-    auto elapsed = _fragment_context->elapsed_time() / NANOS_PER_SEC;
-    fmt::format_to(
-            debug_string_buffer,
-            "PipelineTask[this = {}, id = {}, open = {}, eos = {}, state = {}, dry run = "
-            "{}, elapse time = {}s, _wake_up_early = {}], time elapsed since last state "
-            "changing = {}s, block dependency = [{}], spilling = {}, is running = {}\noperators: ",
-            (void*)this, _index, _opened, _eos, _to_string(_exec_state), _dry_run, elapsed,
-            _wake_up_early.load(), _state_change_watcher.elapsed_time() / NANOS_PER_SEC,
-            cur_blocked_dep && !is_finalized() ? cur_blocked_dep->debug_string() : "NULL",
-            _spilling, is_running());
+    auto fragment = _fragment_context.lock();
+    if (is_finalized() || !fragment) {
+        return fmt::to_string(debug_string_buffer);
+    }
+    auto elapsed = fragment->elapsed_time() / NANOS_PER_SEC;
+    fmt::format_to(debug_string_buffer,
+                   " elapse time = {}s, block dependency = [{}]\noperators: ", elapsed,
+                   cur_blocked_dep && !is_finalized() ? cur_blocked_dep->debug_string() : "NULL");
+
     for (size_t i = 0; i < _operators.size(); i++) {
         fmt::format_to(debug_string_buffer, "\n{}",
                        _opened && !is_finalized() ? _operators[i]->debug_string(_state, i)
@@ -669,9 +696,6 @@ std::string PipelineTask::debug_string() {
     fmt::format_to(debug_string_buffer, "\n{}\n",
                    _opened && !is_finalized() ? _sink->debug_string(_state, _operators.size())
                                               : _sink->debug_string(_operators.size()));
-    if (is_finalized()) {
-        return fmt::to_string(debug_string_buffer);
-    }
 
     fmt::format_to(debug_string_buffer, "\nRead Dependency Information: \n");
 
@@ -745,11 +769,56 @@ Status PipelineTask::wake_up(Dependency* dep) {
     // call by dependency
     DCHECK_EQ(_blocked_dep, dep) << "dep : " << dep->debug_string(0) << "task: " << debug_string();
     _blocked_dep = nullptr;
+    auto holder = std::dynamic_pointer_cast<PipelineTask>(shared_from_this());
     RETURN_IF_ERROR(_state_transition(PipelineTask::State::RUNNABLE));
-    return get_task_queue()->push_back(this);
+    RETURN_IF_ERROR(get_task_queue()->push_back(holder));
+    return Status::OK();
 }
 
-QueryContext* PipelineTask::query_context() {
-    return _fragment_context->get_query_ctx();
+Status PipelineTask::_state_transition(State new_state) {
+    if (_exec_state != new_state) {
+        _state_change_watcher.reset();
+        _state_change_watcher.start();
+    }
+    _task_profile->add_info_string("TaskState", _to_string(new_state));
+    _task_profile->add_info_string("BlockedByDependency", _blocked_dep ? _blocked_dep->name() : "");
+    switch (new_state) {
+    case State::RUNNABLE:
+        if (_exec_state != State::RUNNABLE && _exec_state != State::BLOCKED &&
+            _exec_state != State::INITED) {
+            return Status::InternalError(
+                    "Task state transition from {} to {} is not allowed! Task info: {}",
+                    _to_string(_exec_state), _to_string(new_state), debug_string());
+        }
+        break;
+    case State::BLOCKED:
+        if (_exec_state != State::RUNNABLE && _exec_state != State::FINISHED) {
+            return Status::InternalError(
+                    "Task state transition from {} to {} is not allowed! Task info: {}",
+                    _to_string(_exec_state), _to_string(new_state), debug_string());
+        }
+        break;
+    case State::FINISHED:
+        if (_exec_state != State::RUNNABLE) {
+            return Status::InternalError(
+                    "Task state transition from {} to {} is not allowed! Task info: {}",
+                    _to_string(_exec_state), _to_string(new_state), debug_string());
+        }
+        break;
+    case State::FINALIZED:
+        if (_exec_state != State::FINISHED && _exec_state != State::INITED) {
+            return Status::InternalError(
+                    "Task state transition from {} to {} is not allowed! Task info: {}",
+                    _to_string(_exec_state), _to_string(new_state), debug_string());
+        }
+        break;
+    default:
+        return Status::InternalError(
+                "Task state transition from {} to {} is not allowed! Task info: {}",
+                _to_string(_exec_state), _to_string(new_state), debug_string());
+    }
+    _exec_state = new_state;
+    return Status::OK();
 }
+
 } // namespace doris::pipeline
diff --git a/be/src/pipeline/pipeline_task.h b/be/src/pipeline/pipeline_task.h
index bf54d22c0ebc0d..f70486fd577dbd 100644
--- a/be/src/pipeline/pipeline_task.h
+++ b/be/src/pipeline/pipeline_task.h
@@ -44,10 +44,11 @@ class MultiCoreTaskQueue;
 class PriorityTaskQueue;
 class Dependency;
 
-class PipelineTask {
+class PipelineTask : public std::enable_shared_from_this<PipelineTask> {
 public:
     PipelineTask(PipelinePtr& pipeline, uint32_t task_id, RuntimeState* state,
-                 PipelineFragmentContext* fragment_context, RuntimeProfile* parent_profile,
+                 std::shared_ptr<PipelineFragmentContext> fragment_context,
+                 RuntimeProfile* parent_profile,
                  std::map<int, std::pair<std::shared_ptr<BasicSharedState>,
                                          std::vector<std::shared_ptr<Dependency>>>>
                          shared_state_map,
@@ -62,9 +63,7 @@ class PipelineTask {
     // must be call after all pipeline task is finish to release resource
     Status close(Status exec_status, bool close_sink = true);
 
-    PipelineFragmentContext* fragment_context() { return _fragment_context; }
-
-    QueryContext* query_context();
+    std::weak_ptr<PipelineFragmentContext>& fragment_context() { return _fragment_context; }
 
     int get_core_id() const { return _core_id; }
 
@@ -187,15 +186,16 @@ class PipelineTask {
 
     RuntimeState* runtime_state() const { return _state; }
 
-    RuntimeProfile* get_task_profile() const { return _task_profile.get(); }
-
     std::string task_name() const { return fmt::format("task{}({})", _index, _pipeline->_name); }
 
     // TODO: Maybe we do not need this safe code anymore
     void stop_if_finished() {
-        if (_sink->is_finished(_state)) {
-            set_wake_up_early();
-            terminate();
+        std::unique_lock<std::mutex> lc(_dependency_lock);
+        if (!is_finalized()) {
+            if (_sink->is_finished(_state)) {
+                set_wake_up_early();
+                terminate();
+            }
         }
     }
 
@@ -223,7 +223,8 @@ class PipelineTask {
     void _fresh_profile_counter();
     Status _open();
 
-    uint32_t _index;
+    const TUniqueId _query_id;
+    const uint32_t _index;
     PipelinePtr _pipeline;
     bool _has_exceed_timeout = false;
     bool _opened;
@@ -232,7 +233,7 @@ class PipelineTask {
     uint32_t _schedule_time = 0;
     std::unique_ptr<vectorized::Block> _block;
 
-    PipelineFragmentContext* _fragment_context = nullptr;
+    std::weak_ptr<PipelineFragmentContext> _fragment_context;
     MultiCoreTaskQueue* _task_queue = nullptr;
 
     // used for priority queue
@@ -330,55 +331,13 @@ class PipelineTask {
         }
     }
 
-    Status _state_transition(State new_state) {
-        if (_exec_state != new_state) {
-            _state_change_watcher.reset();
-            _state_change_watcher.start();
-        }
-        _task_profile->add_info_string("TaskState", _to_string(new_state));
-        _task_profile->add_info_string("BlockedByDependency",
-                                       _blocked_dep ? _blocked_dep->name() : "");
-        switch (new_state) {
-        case State::RUNNABLE:
-            if (_exec_state != State::RUNNABLE && _exec_state != State::BLOCKED &&
-                _exec_state != State::INITED) {
-                return Status::InternalError(
-                        "Task state transition from {} to {} is not allowed! Task info: {}",
-                        _to_string(_exec_state), _to_string(new_state), debug_string());
-            }
-            break;
-        case State::BLOCKED:
-            if (_exec_state != State::RUNNABLE && _exec_state != State::FINISHED) {
-                return Status::InternalError(
-                        "Task state transition from {} to {} is not allowed! Task info: {}",
-                        _to_string(_exec_state), _to_string(new_state), debug_string());
-            }
-            break;
-        case State::FINISHED:
-            if (_exec_state != State::RUNNABLE) {
-                return Status::InternalError(
-                        "Task state transition from {} to {} is not allowed! Task info: {}",
-                        _to_string(_exec_state), _to_string(new_state), debug_string());
-            }
-            break;
-        case State::FINALIZED:
-            if (_exec_state != State::FINISHED && _exec_state != State::INITED) {
-                return Status::InternalError(
-                        "Task state transition from {} to {} is not allowed! Task info: {}",
-                        _to_string(_exec_state), _to_string(new_state), debug_string());
-            }
-            break;
-        default:
-            return Status::InternalError(
-                    "Task state transition from {} to {} is not allowed! Task info: {}",
-                    _to_string(_exec_state), _to_string(new_state), debug_string());
-        }
-        _exec_state = new_state;
-        return Status::OK();
-    }
+    Status _state_transition(State new_state);
     std::atomic<State> _exec_state = State::INITED;
     MonotonicStopWatch _state_change_watcher;
     std::atomic<bool> _spilling = false;
 };
 
+using PipelineTaskSPtr = std::shared_ptr<PipelineTask>;
+using PipelineTaskWPtr = std::weak_ptr<PipelineTask>;
+
 } // namespace doris::pipeline
diff --git a/be/src/pipeline/task_queue.cpp b/be/src/pipeline/task_queue.cpp
index 390707766a4892..e708961134d28a 100644
--- a/be/src/pipeline/task_queue.cpp
+++ b/be/src/pipeline/task_queue.cpp
@@ -29,7 +29,7 @@
 namespace doris::pipeline {
 #include "common/compile_check_begin.h"
 
-PipelineTask* SubTaskQueue::try_take(bool is_steal) {
+PipelineTaskSPtr SubTaskQueue::try_take(bool is_steal) {
     if (_queue.empty()) {
         return nullptr;
     }
@@ -54,7 +54,7 @@ void PriorityTaskQueue::close() {
     _wait_task.notify_all();
 }
 
-PipelineTask* PriorityTaskQueue::_try_take_unprotected(bool is_steal) {
+PipelineTaskSPtr PriorityTaskQueue::_try_take_unprotected(bool is_steal) {
     if (_total_task_size == 0 || _closed) {
         return nullptr;
     }
@@ -90,13 +90,13 @@ int PriorityTaskQueue::_compute_level(uint64_t runtime) {
     return SUB_QUEUE_LEVEL - 1;
 }
 
-PipelineTask* PriorityTaskQueue::try_take(bool is_steal) {
+PipelineTaskSPtr PriorityTaskQueue::try_take(bool is_steal) {
     // TODO other efficient lock? e.g. if get lock fail, return null_ptr
     std::unique_lock<std::mutex> lock(_work_size_mutex);
     return _try_take_unprotected(is_steal);
 }
 
-PipelineTask* PriorityTaskQueue::take(uint32_t timeout_ms) {
+PipelineTaskSPtr PriorityTaskQueue::take(uint32_t timeout_ms) {
     std::unique_lock<std::mutex> lock(_work_size_mutex);
     auto task = _try_take_unprotected(false);
     if (task) {
@@ -111,7 +111,7 @@ PipelineTask* PriorityTaskQueue::take(uint32_t timeout_ms) {
     }
 }
 
-Status PriorityTaskQueue::push(PipelineTask* task) {
+Status PriorityTaskQueue::push(PipelineTaskSPtr task) {
     if (_closed) {
         return Status::InternalError("WorkTaskQueue closed");
     }
@@ -145,8 +145,8 @@ void MultiCoreTaskQueue::close() {
                           [](auto& prio_task_queue) { prio_task_queue.close(); });
 }
 
-PipelineTask* MultiCoreTaskQueue::take(int core_id) {
-    PipelineTask* task = nullptr;
+PipelineTaskSPtr MultiCoreTaskQueue::take(int core_id) {
+    PipelineTaskSPtr task = nullptr;
     while (!_closed) {
         DCHECK(_prio_task_queues.size() > core_id)
                 << " list size: " << _prio_task_queues.size() << " core_id: " << core_id
@@ -170,7 +170,7 @@ PipelineTask* MultiCoreTaskQueue::take(int core_id) {
     return task;
 }
 
-PipelineTask* MultiCoreTaskQueue::_steal_take(int core_id) {
+PipelineTaskSPtr MultiCoreTaskQueue::_steal_take(int core_id) {
     DCHECK(core_id < _core_size);
     int next_id = core_id;
     for (int i = 1; i < _core_size; ++i) {
@@ -187,7 +187,7 @@ PipelineTask* MultiCoreTaskQueue::_steal_take(int core_id) {
     return nullptr;
 }
 
-Status MultiCoreTaskQueue::push_back(PipelineTask* task) {
+Status MultiCoreTaskQueue::push_back(PipelineTaskSPtr task) {
     int core_id = task->get_core_id();
     if (core_id < 0) {
         core_id = _next_core.fetch_add(1) % _core_size;
@@ -195,7 +195,7 @@ Status MultiCoreTaskQueue::push_back(PipelineTask* task) {
     return push_back(task, core_id);
 }
 
-Status MultiCoreTaskQueue::push_back(PipelineTask* task, int core_id) {
+Status MultiCoreTaskQueue::push_back(PipelineTaskSPtr task, int core_id) {
     DCHECK(core_id < _core_size);
     task->put_in_runnable_queue();
     return _prio_task_queues[core_id].push(task);
diff --git a/be/src/pipeline/task_queue.h b/be/src/pipeline/task_queue.h
index 2218d70fde61dc..b3f867fa2a6079 100644
--- a/be/src/pipeline/task_queue.h
+++ b/be/src/pipeline/task_queue.h
@@ -38,9 +38,9 @@ class SubTaskQueue {
     friend class PriorityTaskQueue;
 
 public:
-    void push_back(PipelineTask* task) { _queue.emplace(task); }
+    void push_back(PipelineTaskSPtr task) { _queue.emplace(task); }
 
-    PipelineTask* try_take(bool is_steal);
+    PipelineTaskSPtr try_take(bool is_steal);
 
     void set_level_factor(double level_factor) { _level_factor = level_factor; }
 
@@ -58,7 +58,7 @@ class SubTaskQueue {
     bool empty() { return _queue.empty(); }
 
 private:
-    std::queue<PipelineTask*> _queue;
+    std::queue<PipelineTaskSPtr> _queue;
     // depends on LEVEL_QUEUE_TIME_FACTOR
     double _level_factor = 1;
 
@@ -72,18 +72,18 @@ class PriorityTaskQueue {
 
     void close();
 
-    PipelineTask* try_take(bool is_steal);
+    PipelineTaskSPtr try_take(bool is_steal);
 
-    PipelineTask* take(uint32_t timeout_ms = 0);
+    PipelineTaskSPtr take(uint32_t timeout_ms = 0);
 
-    Status push(PipelineTask* task);
+    Status push(PipelineTaskSPtr task);
 
     void inc_sub_queue_runtime(int level, uint64_t runtime) {
         _sub_queues[level].inc_runtime(runtime);
     }
 
 private:
-    PipelineTask* _try_take_unprotected(bool is_steal);
+    PipelineTaskSPtr _try_take_unprotected(bool is_steal);
     static constexpr auto LEVEL_QUEUE_TIME_FACTOR = 2;
     static constexpr size_t SUB_QUEUE_LEVEL = 6;
     SubTaskQueue _sub_queues[SUB_QUEUE_LEVEL];
@@ -110,24 +110,24 @@ class MultiCoreTaskQueue {
 #ifndef BE_TEST
     ~MultiCoreTaskQueue();
     // Get the task by core id.
-    PipelineTask* take(int core_id);
+    PipelineTaskSPtr take(int core_id);
 #else
     virtual ~MultiCoreTaskQueue();
-    virtual PipelineTask* take(int core_id);
+    virtual PipelineTaskSPtr take(int core_id);
 #endif
     void close();
 
     // TODO combine these methods to `push_back(task, core_id = -1)`
-    Status push_back(PipelineTask* task);
+    Status push_back(PipelineTaskSPtr task);
 
-    Status push_back(PipelineTask* task, int core_id);
+    Status push_back(PipelineTaskSPtr task, int core_id);
 
     void update_statistics(PipelineTask* task, int64_t time_spent);
 
     int cores() const { return _core_size; }
 
 private:
-    PipelineTask* _steal_take(int core_id);
+    PipelineTaskSPtr _steal_take(int core_id);
 
     std::vector<PriorityTaskQueue> _prio_task_queues;
     std::atomic<uint32_t> _next_core = 0;
diff --git a/be/src/pipeline/task_scheduler.cpp b/be/src/pipeline/task_scheduler.cpp
index 905ad97f842eee..1e82ef262b6a9f 100644
--- a/be/src/pipeline/task_scheduler.cpp
+++ b/be/src/pipeline/task_scheduler.cpp
@@ -70,35 +70,34 @@ Status TaskScheduler::start() {
     return Status::OK();
 }
 
-Status TaskScheduler::schedule_task(PipelineTask* task) {
+Status TaskScheduler::schedule_task(PipelineTaskSPtr task) {
     return _task_queue.push_back(task);
 }
 
 // after close_task, task maybe destructed.
-void close_task(PipelineTask* task, Status exec_status) {
+void close_task(PipelineTask* task, Status exec_status, PipelineFragmentContext* ctx) {
     // Has to attach memory tracker here, because the close task will also release some memory.
     // Should count the memory to the query or the query's memory will not decrease when part of
     // task finished.
     SCOPED_ATTACH_TASK(task->runtime_state());
     if (!exec_status.ok()) {
-        task->fragment_context()->cancel(exec_status);
+        ctx->cancel(exec_status);
         LOG(WARNING) << fmt::format("Pipeline task failed. query_id: {} reason: {}",
-                                    print_id(task->query_context()->query_id()),
-                                    exec_status.to_string());
+                                    print_id(ctx->get_query_id()), exec_status.to_string());
     }
     Status status = task->close(exec_status);
     if (!status.ok()) {
-        task->fragment_context()->cancel(status);
+        ctx->cancel(status);
     }
     status = task->finalize();
     if (!status.ok()) {
-        task->fragment_context()->cancel(status);
+        ctx->cancel(status);
     }
 }
 
 void TaskScheduler::_do_work(int index) {
     while (!_need_to_stop) {
-        auto* task = _task_queue.take(index);
+        auto task = _task_queue.take(index);
         if (!task) {
             continue;
         }
@@ -113,18 +112,21 @@ void TaskScheduler::_do_work(int index) {
         if (task->is_finalized()) {
             continue;
         }
+        auto fragment_context = task->fragment_context().lock();
+        if (!fragment_context) {
+            // Fragment already finishedquery
+            continue;
+        }
         task->set_running(true);
         bool done = false;
         auto status = Status::OK();
         Defer task_running_defer {[&]() {
             // If fragment is finished, fragment context will be de-constructed with all tasks in it.
             if (done || !status.ok()) {
-                // decrement_running_task may delete fragment context and will core in some defer
-                // code, because the defer code will access fragment context itself.
-                auto lock_for_context = task->fragment_context()->shared_from_this();
-                close_task(task, status);
+                auto id = task->pipeline_id();
+                close_task(task.get(), status, fragment_context.get());
                 task->set_running(false);
-                task->fragment_context()->decrement_running_task(task->pipeline_id());
+                fragment_context->decrement_running_task(id);
             } else {
                 task->set_running(false);
             }
@@ -132,12 +134,11 @@ void TaskScheduler::_do_work(int index) {
         task->set_task_queue(&_task_queue);
         task->log_detail_if_need();
 
-        auto* fragment_ctx = task->fragment_context();
-        bool canceled = fragment_ctx->is_canceled();
+        bool canceled = fragment_context->is_canceled();
 
         // Close task if canceled
         if (canceled) {
-            status = fragment_ctx->get_query_ctx()->exec_status();
+            status = fragment_context->get_query_ctx()->exec_status();
             DCHECK(!status.ok());
             continue;
         }
@@ -147,7 +148,7 @@ void TaskScheduler::_do_work(int index) {
         ASSIGN_STATUS_IF_CATCH_EXCEPTION(
                 //TODO: use a better enclose to abstracting these
                 if (ExecEnv::GetInstance()->pipeline_tracer_context()->enabled()) {
-                    TUniqueId query_id = task->query_context()->query_id();
+                    TUniqueId query_id = fragment_context->get_query_id();
                     std::string task_name = task->task_name();
 
                     std::thread::id tid = std::this_thread::get_id();
@@ -162,7 +163,7 @@ void TaskScheduler::_do_work(int index) {
                              start_time, end_time});
                 } else { status = task->execute(&done); },
                 status);
-        fragment_ctx->trigger_report_if_necessary();
+        fragment_context->trigger_report_if_necessary();
     }
 }
 
diff --git a/be/src/pipeline/task_scheduler.h b/be/src/pipeline/task_scheduler.h
index 449f0bfbe5dd20..b8d4fe01fec3bc 100644
--- a/be/src/pipeline/task_scheduler.h
+++ b/be/src/pipeline/task_scheduler.h
@@ -50,7 +50,7 @@ class TaskScheduler {
 
     ~TaskScheduler();
 
-    Status schedule_task(PipelineTask* task);
+    Status schedule_task(PipelineTaskSPtr task);
 
     Status start();
 
diff --git a/be/test/pipeline/dummy_task_queue.h b/be/test/pipeline/dummy_task_queue.h
index bace0ecae96192..9b609f41fc8e96 100644
--- a/be/test/pipeline/dummy_task_queue.h
+++ b/be/test/pipeline/dummy_task_queue.h
@@ -22,8 +22,8 @@ namespace doris::pipeline {
 class DummyTaskQueue final : public MultiCoreTaskQueue {
     explicit DummyTaskQueue(int core_size) : MultiCoreTaskQueue(core_size) {}
     ~DummyTaskQueue() override = default;
-    PipelineTask* take(int core_id) override {
-        PipelineTask* task = nullptr;
+    std::shared_ptr<PipelineTask> take(int core_id) override {
+        std::shared_ptr<PipelineTask> task = nullptr;
         do {
             DCHECK(_prio_task_queues.size() > core_id)
                     << " list size: " << _prio_task_queues.size() << " core_id: " << core_id
diff --git a/be/test/pipeline/pipeline_test.cpp b/be/test/pipeline/pipeline_test.cpp
index 2c395737ad5183..feceab00945faf 100644
--- a/be/test/pipeline/pipeline_test.cpp
+++ b/be/test/pipeline/pipeline_test.cpp
@@ -77,7 +77,7 @@ class PipelineTest : public testing::Test {
                 parent ? std::min(parent->num_tasks(), num_instances) : num_instances,
                 parent ? parent->num_tasks() : num_instances);
         _pipelines.push_back(pip);
-        _pipeline_tasks.push_back(std::vector<std::unique_ptr<PipelineTask>> {});
+        _pipeline_tasks.push_back(std::vector<std::shared_ptr<PipelineTask>> {});
         _runtime_states.push_back(std::vector<std::unique_ptr<RuntimeState>> {});
         _pipeline_profiles.push_back(nullptr);
         if (parent) {
@@ -156,7 +156,7 @@ class PipelineTest : public testing::Test {
 
     // Task Level
     // Fragment0[Pipeline0[Task0] -> Pipeline1[Task0]] -> Fragment1[Pipeline2[Task0] -> Pipeline3[Task0]]
-    std::vector<std::vector<std::unique_ptr<PipelineTask>>> _pipeline_tasks;
+    std::vector<std::vector<std::shared_ptr<PipelineTask>>> _pipeline_tasks;
     std::vector<std::vector<std::unique_ptr<RuntimeState>>> _runtime_states;
 
     // Instance level
@@ -276,7 +276,7 @@ TEST_F(PipelineTest, HAPPY_PATH) {
                                 std::vector<std::shared_ptr<Dependency>>>>
                 shared_state_map;
         auto task = std::make_unique<PipelineTask>(
-                cur_pipe, task_id, local_runtime_state.get(), _context.back().get(),
+                cur_pipe, task_id, local_runtime_state.get(), _context.back(),
                 _pipeline_profiles[cur_pipe->id()].get(), shared_state_map, task_id);
         cur_pipe->incr_created_tasks(task_id, task.get());
         task->set_task_queue(_task_queue.get());
@@ -939,7 +939,7 @@ TEST_F(PipelineTest, PLAN_HASH_JOIN) {
                                         std::vector<std::shared_ptr<Dependency>>>>
                         shared_state_map;
                 auto task = std::make_unique<PipelineTask>(
-                        _pipelines[i], task_id, local_runtime_state.get(), _context.back().get(),
+                        _pipelines[i], task_id, local_runtime_state.get(), _context.back(),
                         _pipeline_profiles[_pipelines[i]->id()].get(), shared_state_map, j);
                 _pipelines[i]->incr_created_tasks(j, task.get());
                 task->set_task_queue(_task_queue.get());


Reviews:
user: login: yiguolei
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-02T03:48:38Z
user: login: yiguolei
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-02T03:50:44Z
user: login: yiguolei
state: DISMISSED
author_association: CONTRIBUTOR
submitted_at: 2025-04-02T06:31:06Z
user: login: yiguolei
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-04-02T09:17:48Z
user: login: Mryange
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-04-02T10:29:12Z

Review comments:
diff_hunk: @@ -745,11 +753,65 @@ Status PipelineTask::wake_up(Dependency* dep) {
     // call by dependency
     DCHECK_EQ(_blocked_dep, dep) << "dep : " << dep->debug_string(0) << "task: " << debug_string();
     _blocked_dep = nullptr;
+    auto holder = std::dynamic_pointer_cast<PipelineTask>(shared_from_this());
+    DCHECK(_scheduler);
+    _scheduler->erase_blocked_task(&_blocked_iterator);
     RETURN_IF_ERROR(_state_transition(PipelineTask::State::RUNNABLE));
-    return get_task_queue()->push_back(this);
+    RETURN_IF_ERROR(get_task_queue()->push_back(holder));
+    return Status::OK();
+}
+
+Status PipelineTask::_state_transition(State new_state) {
+    if (_exec_state != new_state) {
+        _state_change_watcher.reset();
+        _state_change_watcher.start();
+    }
+    _task_profile->add_info_string("TaskState", _to_string(new_state));
+    _task_profile->add_info_string("BlockedByDependency", _blocked_dep ? _blocked_dep->name() : "");
+    switch (new_state) {
+    case State::RUNNABLE:
+        if (_exec_state != State::RUNNABLE && _exec_state != State::BLOCKED &&
+            _exec_state != State::INITED) {
+            return Status::InternalError(
+                    "Task state transition from {} to {} is not allowed! Task info: {}",
+                    _to_string(_exec_state), _to_string(new_state), debug_string());
+        }
+        break;
+    case State::BLOCKED:
+        // If this task is blocked, the blocking dependency will hold the raw pointer of this task.
+        // To ensure this task will not be freed, we use scheduler to hold the shared pointer.
+        DCHECK(_scheduler);
+        _scheduler->hold_blocked_task(shared_from_this(), &_blocked_iterator);
+        if (_exec_state != State::RUNNABLE && _exec_state != State::FINISHED) {
+            return Status::InternalError(
+                    "Task state transition from {} to {} is not allowed! Task info: {}",
+                    _to_string(_exec_state), _to_string(new_state), debug_string());
+        }
+        break;
+    case State::FINISHED:
+        if (_exec_state != State::RUNNABLE) {
+            return Status::InternalError(
+                    "Task state transition from {} to {} is not allowed! Task info: {}",
+                    _to_string(_exec_state), _to_string(new_state), debug_string());
+        }
+        break;
+    case State::FINALIZED:
+        if (_exec_state != State::FINISHED && _exec_state != State::INITED) {
+            return Status::InternalError(
+                    "Task state transition from {} to {} is not allowed! Task info: {}",
+                    _to_string(_exec_state), _to_string(new_state), debug_string());
+        }
+        break;
+    default:
+        return Status::InternalError(
+                "Task state transition from {} to {} is not allowed! Task info: {}",
+                _to_string(_exec_state), _to_string(new_state), debug_string());
+    }
+    _exec_state = new_state;
+    return Status::OK();
 }
 
 QueryContext* PipelineTask::query_context() {
-    return _fragment_context->get_query_ctx();
+    return _fragment_context.lock()->get_query_ctx();
path: be/src/pipeline/pipeline_task.cpp
user: login: yiguolei
body: 如果lock 结果是一个空怎么办？
created_at: 2025-04-02T03:48:38Z
author_association: CONTRIBUTOR
diff_hunk: @@ -62,7 +63,7 @@ class PipelineTask {
     // must be call after all pipeline task is finish to release resource
     Status close(Status exec_status, bool close_sink = true);
 
-    PipelineFragmentContext* fragment_context() { return _fragment_context; }
+    std::shared_ptr<PipelineFragmentContext> fragment_context() { return _fragment_context.lock(); }
path: be/src/pipeline/pipeline_task.h
user: login: yiguolei
body: 最好不要这么写。
这个写法，就得外部的调用者去判断是否是nullptr，而且如果他忘记了判断nullptr，线上就是一个core。
直接返回weak ptr，这样，外部调用者得显示的lock，就能感知到lock 可能会失败。
created_at: 2025-04-02T03:50:44Z
author_association: CONTRIBUTOR

PR comments:
user: login: hello-stephen
created_at: 2025-04-02T02:41:24Z
body: 
Thank you for your contribution to Apache Doris.
Don't know what should be done next? See [How to process your PR](https://cwiki.apache.org/confluence/display/DORIS/How+to+process+your+PR).

Please clearly describe your PR:
1. What problem was fixed (it's best to include specific error reporting information). How it was fixed.
2. Which behaviors were modified. What was the previous behavior, what is it now, why was it modified, and what possible impacts might there be.
3. What features were added. Why was this function added?
4. Which code was refactored and why was this part of the code refactored?
5. Which functions were optimized and what is the difference before and after the optimization?
    
author_association: CONTRIBUTOR
user: login: Gabriel39
created_at: 2025-04-02T02:43:15Z
body: run buildall
author_association: CONTRIBUTOR
user: login: doris-robot
created_at: 2025-04-02T03:10:50Z
body: 
<details>
<summary>TPC-H: <b>Total hot run time: 34423 ms</b></summary>

```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpch-tools
Tpch sf100 test result on commit 90b544c6fa8549e046c1166c45a21691afec051c, data reload: false

------ Round 1 ----------------------------------
q1	26030	5149	5118	5118
q2	2075	273	191	191
q3	10391	1275	681	681
q4	10236	1011	523	523
q5	7528	2350	2368	2350
q6	189	166	135	135
q7	928	739	609	609
q8	9334	1292	1233	1233
q9	7077	5130	5080	5080
q10	6880	2304	1892	1892
q11	481	283	278	278
q12	357	381	222	222
q13	17769	3667	3107	3107
q14	232	229	215	215
q15	546	488	486	486
q16	639	623	582	582
q17	604	864	346	346
q18	7487	7288	7084	7084
q19	1649	997	570	570
q20	340	337	228	228
q21	4652	3538	2499	2499
q22	1055	1036	994	994
Total cold run time: 116479 ms
Total hot run time: 34423 ms

----- Round 2, with runtime_filter_mode=off -----
q1	5311	5166	5169	5166
q2	249	325	232	232
q3	2161	2713	2315	2315
q4	1425	1905	1447	1447
q5	4463	4370	4388	4370
q6	216	180	134	134
q7	1977	1942	1799	1799
q8	2601	2696	2574	2574
q9	7358	7076	7226	7076
q10	2972	3144	2738	2738
q11	589	528	486	486
q12	726	738	642	642
q13	3495	3854	3413	3413
q14	274	286	262	262
q15	512	486	462	462
q16	665	695	651	651
q17	1167	1561	1425	1425
q18	7865	7576	7533	7533
q19	818	837	982	837
q20	1992	2038	1872	1872
q21	5424	4947	4928	4928
q22	1121	1064	1031	1031
Total cold run time: 53381 ms
Total hot run time: 51393 ms
```
</details>

author_association: NONE
user: login: doris-robot
created_at: 2025-04-02T03:22:48Z
body: 
<details>
<summary>TPC-DS: <b>Total hot run time: 192820 ms</b></summary>

```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpcds-tools
TPC-DS sf100 test result on commit 90b544c6fa8549e046c1166c45a21691afec051c, data reload: false

query1	1384	1066	1041	1041
query2	6136	1942	1955	1942
query3	11147	4583	4727	4583
query4	25319	23679	22961	22961
query5	4852	635	445	445
query6	303	193	197	193
query7	3981	494	276	276
query8	293	225	215	215
query9	8501	2565	2556	2556
query10	476	314	263	263
query11	15694	15239	14699	14699
query12	165	111	110	110
query13	1571	555	391	391
query14	9634	6157	6209	6157
query15	201	188	171	171
query16	7607	630	481	481
query17	1187	756	610	610
query18	2031	421	323	323
query19	201	202	171	171
query20	141	123	122	122
query21	214	135	115	115
query22	4729	4675	4345	4345
query23	34198	33480	33492	33480
query24	8344	2433	2414	2414
query25	497	498	410	410
query26	1206	280	168	168
query27	2851	502	325	325
query28	4842	2438	2422	2422
query29	714	581	437	437
query30	276	221	194	194
query31	917	884	808	808
query32	77	66	59	59
query33	546	368	317	317
query34	809	882	518	518
query35	830	868	753	753
query36	986	1043	887	887
query37	123	100	80	80
query38	4170	4316	4036	4036
query39	1495	1445	1442	1442
query40	211	119	107	107
query41	52	56	50	50
query42	125	104	114	104
query43	518	537	489	489
query44	1301	808	810	808
query45	185	195	173	173
query46	847	1039	645	645
query47	1856	1915	1814	1814
query48	385	432	312	312
query49	772	503	461	461
query50	672	687	401	401
query51	4275	4293	4203	4203
query52	120	113	96	96
query53	230	256	203	203
query54	600	571	525	525
query55	85	89	88	88
query56	307	308	278	278
query57	1177	1195	1156	1156
query58	266	283	258	258
query59	2744	2867	2790	2790
query60	336	342	309	309
query61	129	130	138	130
query62	765	733	664	664
query63	221	181	178	178
query64	4248	1071	728	728
query65	4457	4360	4384	4360
query66	1334	413	321	321
query67	16749	15717	15478	15478
query68	9692	886	517	517
query69	472	302	269	269
query70	1188	1117	1117	1117
query71	498	337	279	279
query72	5090	4703	4610	4610
query73	697	556	352	352
query74	8930	9101	8744	8744
query75	4413	3214	2777	2777
query76	4330	1180	737	737
query77	1009	362	271	271
query78	9851	10236	9343	9343
query79	2472	802	572	572
query80	620	499	429	429
query81	474	256	220	220
query82	244	131	102	102
query83	260	259	244	244
query84	294	114	86	86
query85	824	357	318	318
query86	330	273	279	273
query87	4507	4614	4535	4535
query88	2821	2225	2212	2212
query89	416	313	280	280
query90	2142	214	217	214
query91	153	149	114	114
query92	77	64	61	61
query93	1944	928	579	579
query94	672	425	305	305
query95	375	286	287	286
query96	491	571	271	271
query97	3104	3289	3156	3156
query98	228	209	196	196
query99	1386	1382	1302	1302
Total cold run time: 283619 ms
Total hot run time: 192820 ms
```
</details>

author_association: NONE
user: login: doris-robot
created_at: 2025-04-02T03:28:15Z
body: 
<details>
<summary>ClickBench: <b>Total hot run time: 31.39 s</b></summary>

```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/clickbench-tools
ClickBench test result on commit 90b544c6fa8549e046c1166c45a21691afec051c, data reload: false

query1	0.04	0.04	0.04
query2	0.12	0.11	0.10
query3	0.25	0.20	0.20
query4	1.59	0.20	0.19
query5	0.59	0.58	0.57
query6	1.20	0.72	0.71
query7	0.02	0.02	0.01
query8	0.04	0.04	0.03
query9	0.57	0.52	0.53
query10	0.58	0.59	0.56
query11	0.16	0.11	0.11
query12	0.15	0.12	0.11
query13	0.61	0.61	0.60
query14	2.83	2.70	2.71
query15	0.93	0.86	0.84
query16	0.39	0.39	0.39
query17	1.02	1.01	1.03
query18	0.21	0.20	0.20
query19	1.87	1.91	1.84
query20	0.01	0.01	0.01
query21	15.37	0.92	0.53
query22	0.78	1.13	0.70
query23	14.97	1.37	0.60
query24	7.33	1.46	0.91
query25	0.50	0.24	0.13
query26	0.55	0.16	0.15
query27	0.06	0.05	0.05
query28	9.65	0.80	0.43
query29	12.63	4.02	3.34
query30	0.25	0.09	0.07
query31	2.82	0.60	0.38
query32	3.23	0.55	0.48
query33	3.10	3.04	3.30
query34	15.78	5.13	4.52
query35	4.55	4.50	4.55
query36	0.66	0.49	0.49
query37	0.08	0.06	0.06
query38	0.05	0.04	0.04
query39	0.03	0.03	0.02
query40	0.17	0.14	0.13
query41	0.09	0.03	0.03
query42	0.04	0.02	0.03
query43	0.03	0.03	0.03
Total cold run time: 105.9 s
Total hot run time: 31.39 s
```
</details>

author_association: NONE
user: login: Gabriel39
created_at: 2025-04-02T05:13:26Z
body: run buildall
author_association: CONTRIBUTOR
user: login: Gabriel39
created_at: 2025-04-02T05:13:51Z
body: run buildall
author_association: CONTRIBUTOR
user: login: doris-robot
created_at: 2025-04-02T06:04:49Z
body: 
<details>
<summary>TPC-H: <b>Total hot run time: 34185 ms</b></summary>

```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpch-tools
Tpch sf100 test result on commit 2ec3df019b45cb47324612c3ee8d9ee5c8a2f2fa, data reload: false

------ Round 1 ----------------------------------
q1	25790	5248	4974	4974
q2	2058	277	196	196
q3	10382	1278	692	692
q4	10233	1026	539	539
q5	7551	2388	2333	2333
q6	183	168	129	129
q7	903	747	624	624
q8	9327	1295	1123	1123
q9	6835	5206	5087	5087
q10	6850	2301	1909	1909
q11	497	291	273	273
q12	356	357	216	216
q13	17781	3700	3127	3127
q14	230	221	206	206
q15	550	490	522	490
q16	651	611	569	569
q17	594	859	379	379
q18	7550	7190	7164	7164
q19	1712	970	538	538
q20	337	331	228	228
q21	4028	3403	2413	2413
q22	1092	1036	976	976
Total cold run time: 115490 ms
Total hot run time: 34185 ms

----- Round 2, with runtime_filter_mode=off -----
q1	5139	5043	5092	5043
q2	235	318	222	222
q3	2142	2670	2324	2324
q4	1465	1837	1490	1490
q5	4493	4382	4345	4345
q6	216	168	124	124
q7	2043	2063	1808	1808
q8	2569	2557	2561	2557
q9	7312	7105	7228	7105
q10	2995	3139	2750	2750
q11	580	509	504	504
q12	685	763	611	611
q13	3507	3896	3362	3362
q14	277	304	270	270
q15	525	482	488	482
q16	636	722	666	666
q17	1149	1567	1379	1379
q18	7913	7548	7589	7548
q19	817	803	960	803
q20	1934	2006	1871	1871
q21	5353	5039	4936	4936
q22	1181	1129	1070	1070
Total cold run time: 53166 ms
Total hot run time: 51270 ms
```
</details>

author_association: NONE
user: login: doris-robot
created_at: 2025-04-02T06:16:48Z
body: 
<details>
<summary>TPC-DS: <b>Total hot run time: 193787 ms</b></summary>

```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpcds-tools
TPC-DS sf100 test result on commit 2ec3df019b45cb47324612c3ee8d9ee5c8a2f2fa, data reload: false

query1	1379	1064	1060	1060
query2	6175	2006	1965	1965
query3	11093	4645	4659	4645
query4	25634	23679	23171	23171
query5	4950	642	444	444
query6	299	201	188	188
query7	3980	491	282	282
query8	308	245	252	245
query9	8504	2630	2600	2600
query10	501	323	271	271
query11	15523	15058	14945	14945
query12	170	109	109	109
query13	1565	512	404	404
query14	9240	6106	6315	6106
query15	214	188	181	181
query16	7632	608	477	477
query17	1154	768	625	625
query18	2031	414	338	338
query19	205	195	168	168
query20	143	133	121	121
query21	215	134	107	107
query22	4579	4693	4458	4458
query23	34611	33607	33548	33548
query24	8643	2412	2412	2412
query25	504	474	408	408
query26	1190	273	148	148
query27	2823	519	335	335
query28	4682	2429	2446	2429
query29	713	568	440	440
query30	281	232	200	200
query31	937	847	848	847
query32	76	67	65	65
query33	550	374	323	323
query34	809	906	518	518
query35	840	847	809	809
query36	944	1016	888	888
query37	121	125	75	75
query38	4134	4162	4171	4162
query39	1679	1448	1652	1448
query40	223	122	115	115
query41	54	51	52	51
query42	130	112	104	104
query43	510	522	490	490
query44	1310	811	830	811
query45	182	178	172	172
query46	853	1023	647	647
query47	1806	1899	1795	1795
query48	391	426	317	317
query49	755	534	416	416
query50	675	691	416	416
query51	4234	4344	4264	4264
query52	111	110	96	96
query53	229	267	188	188
query54	600	588	525	525
query55	79	84	80	80
query56	308	305	304	304
query57	1171	1163	1131	1131
query58	297	272	280	272
query59	2783	2832	2797	2797
query60	347	328	355	328
query61	131	128	129	128
query62	737	746	681	681
query63	228	192	193	192
query64	4033	1054	707	707
query65	4432	4358	4405	4358
query66	1128	422	320	320
query67	16282	15792	15453	15453
query68	9266	889	518	518
query69	473	298	258	258
query70	1231	1112	1108	1108
query71	486	305	290	290
query72	5673	4719	4721	4719
query73	702	590	348	348
query74	8959	9043	8974	8974
query75	4324	3207	2703	2703
query76	3794	1177	741	741
query77	798	368	284	284
query78	9992	10086	9306	9306
query79	2680	825	556	556
query80	653	507	438	438
query81	509	263	224	224
query82	490	127	98	98
query83	277	252	236	236
query84	292	102	85	85
query85	787	434	309	309
query86	376	266	274	266
query87	4689	4592	4382	4382
query88	3430	2183	2180	2180
query89	406	314	272	272
query90	1889	206	208	206
query91	142	138	111	111
query92	75	61	57	57
query93	2300	933	569	569
query94	659	406	299	299
query95	364	292	324	292
query96	485	566	272	272
query97	3218	3248	3103	3103
query98	246	214	201	201
query99	1432	1399	1259	1259
Total cold run time: 284147 ms
Total hot run time: 193787 ms
```
</details>

author_association: NONE
user: login: doris-robot
created_at: 2025-04-02T06:22:07Z
body: 
<details>
<summary>ClickBench: <b>Total hot run time: 31.45 s</b></summary>

```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/clickbench-tools
ClickBench test result on commit 2ec3df019b45cb47324612c3ee8d9ee5c8a2f2fa, data reload: false

query1	0.04	0.04	0.03
query2	0.13	0.11	0.11
query3	0.25	0.19	0.19
query4	1.59	0.19	0.18
query5	0.57	0.58	0.58
query6	1.19	0.71	0.72
query7	0.02	0.02	0.02
query8	0.04	0.03	0.03
query9	0.58	0.52	0.51
query10	0.60	0.58	0.57
query11	0.15	0.11	0.12
query12	0.15	0.11	0.12
query13	0.62	0.60	0.60
query14	2.67	2.70	2.82
query15	0.94	0.85	0.85
query16	0.39	0.39	0.37
query17	1.01	1.04	1.03
query18	0.21	0.20	0.20
query19	1.82	1.85	1.92
query20	0.01	0.01	0.01
query21	15.40	0.93	0.54
query22	0.75	1.14	0.61
query23	15.09	1.37	0.63
query24	6.86	2.06	1.10
query25	0.51	0.14	0.06
query26	0.64	0.16	0.13
query27	0.05	0.05	0.06
query28	9.99	0.82	0.42
query29	12.56	3.98	3.30
query30	0.26	0.09	0.07
query31	2.82	0.57	0.38
query32	3.24	0.55	0.47
query33	3.02	3.06	3.09
query34	15.76	5.15	4.52
query35	4.54	4.55	4.55
query36	0.67	0.50	0.48
query37	0.09	0.06	0.06
query38	0.05	0.04	0.04
query39	0.03	0.02	0.02
query40	0.17	0.13	0.14
query41	0.08	0.03	0.03
query42	0.03	0.02	0.03
query43	0.04	0.03	0.02
Total cold run time: 105.63 s
Total hot run time: 31.45 s
```
</details>

author_association: NONE
user: login: github-actions[bot]
created_at: 2025-04-02T06:31:56Z
body: PR approved by at least one committer and no changes requested.
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-04-02T06:31:57Z
body: PR approved by anyone and no changes requested.
author_association: CONTRIBUTOR
user: login: Gabriel39
created_at: 2025-04-02T06:48:44Z
body: run buildall
author_association: CONTRIBUTOR
user: login: Gabriel39
created_at: 2025-04-02T06:54:05Z
body: run buildall
author_association: CONTRIBUTOR
user: login: doris-robot
created_at: 2025-04-02T07:35:38Z
body: 
<details>
<summary>TPC-H: <b>Total hot run time: 35129 ms</b></summary>

```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpch-tools
Tpch sf100 test result on commit 976f58995ca63ef05709b1fb0c5e6a9962d6ec4f, data reload: false

------ Round 1 ----------------------------------
q1	25768	5120	5066	5066
q2	2089	284	194	194
q3	10533	1299	689	689
q4	10222	1015	540	540
q5	8368	2402	2322	2322
q6	182	161	132	132
q7	922	745	602	602
q8	9330	1274	1040	1040
q9	7361	5220	5149	5149
q10	6873	2327	1912	1912
q11	481	297	291	291
q12	361	349	224	224
q13	18120	3725	3143	3143
q14	237	238	214	214
q15	546	489	494	489
q16	619	645	579	579
q17	601	851	370	370
q18	8901	7780	7597	7597
q19	6061	2208	878	878
q20	317	321	225	225
q21	3983	3447	2493	2493
q22	1064	1036	980	980
Total cold run time: 122939 ms
Total hot run time: 35129 ms

----- Round 2, with runtime_filter_mode=off -----
q1	5536	5205	5142	5142
q2	233	325	227	227
q3	2328	2812	2472	2472
q4	1563	1943	1530	1530
q5	4518	4378	4408	4378
q6	218	181	128	128
q7	2024	1911	1783	1783
q8	2594	2473	2457	2457
q9	7234	7197	7326	7197
q10	3034	3219	2776	2776
q11	595	504	500	500
q12	732	811	650	650
q13	3720	3967	3403	3403
q14	290	311	270	270
q15	523	481	478	478
q16	656	689	641	641
q17	1169	1537	1388	1388
q18	7686	7565	7224	7224
q19	793	771	854	771
q20	1948	2029	1889	1889
q21	5366	4958	4901	4901
q22	1120	1086	1042	1042
Total cold run time: 53880 ms
Total hot run time: 51247 ms
```
</details>

author_association: NONE
user: login: doris-robot
created_at: 2025-04-02T07:47:29Z
body: 
<details>
<summary>TPC-DS: <b>Total hot run time: 192533 ms</b></summary>

```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpcds-tools
TPC-DS sf100 test result on commit 976f58995ca63ef05709b1fb0c5e6a9962d6ec4f, data reload: false

query1	1398	1066	1054	1054
query2	6262	2054	1970	1970
query3	11125	4671	4572	4572
query4	25632	23404	22829	22829
query5	4655	622	449	449
query6	305	208	194	194
query7	3982	494	283	283
query8	287	239	224	224
query9	8529	2606	2584	2584
query10	450	330	264	264
query11	15507	15091	14873	14873
query12	157	110	113	110
query13	1559	514	391	391
query14	8996	6092	6142	6092
query15	204	182	174	174
query16	7423	617	453	453
query17	1170	739	599	599
query18	2008	408	320	320
query19	194	213	160	160
query20	130	129	127	127
query21	206	126	109	109
query22	4485	4686	4417	4417
query23	34660	33345	33250	33250
query24	8365	2474	2454	2454
query25	508	501	433	433
query26	769	284	168	168
query27	2826	518	341	341
query28	4634	2471	2415	2415
query29	640	601	454	454
query30	280	222	200	200
query31	956	860	827	827
query32	73	67	64	64
query33	559	370	310	310
query34	816	879	541	541
query35	813	860	758	758
query36	960	1014	924	924
query37	127	98	90	90
query38	4124	4313	4187	4187
query39	1474	1469	1422	1422
query40	216	121	109	109
query41	53	54	50	50
query42	128	107	103	103
query43	525	525	489	489
query44	1330	848	827	827
query45	184	182	167	167
query46	836	1026	643	643
query47	1850	1917	1814	1814
query48	391	438	301	301
query49	735	520	416	416
query50	662	699	410	410
query51	4233	4339	4260	4260
query52	107	107	98	98
query53	232	258	187	187
query54	588	568	503	503
query55	87	82	85	82
query56	302	307	304	304
query57	1160	1143	1160	1143
query58	273	261	268	261
query59	2726	2979	2757	2757
query60	341	337	308	308
query61	126	155	131	131
query62	766	742	658	658
query63	217	186	183	183
query64	3044	1031	716	716
query65	4338	4214	4256	4214
query66	901	401	308	308
query67	15749	15620	15265	15265
query68	8852	892	568	568
query69	483	313	259	259
query70	1199	1106	1087	1087
query71	479	310	316	310
query72	5270	4726	4613	4613
query73	685	644	344	344
query74	8858	9174	8938	8938
query75	4208	3203	2705	2705
query76	3740	1201	745	745
query77	785	362	290	290
query78	9835	10147	9234	9234
query79	1944	866	573	573
query80	594	524	439	439
query81	484	251	220	220
query82	451	125	95	95
query83	259	261	292	261
query84	245	110	92	92
query85	775	353	308	308
query86	342	313	283	283
query87	4408	4443	4434	4434
query88	3143	2256	2241	2241
query89	398	312	289	289
query90	1995	211	216	211
query91	144	141	121	121
query92	75	65	54	54
query93	1239	978	577	577
query94	701	429	276	276
query95	373	297	283	283
query96	482	571	276	276
query97	3191	3229	3147	3147
query98	223	204	201	201
query99	1418	1375	1266	1266
Total cold run time: 276626 ms
Total hot run time: 192533 ms
```
</details>

author_association: NONE
user: login: doris-robot
created_at: 2025-04-02T07:52:48Z
body: 
<details>
<summary>ClickBench: <b>Total hot run time: 30.87 s</b></summary>

```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/clickbench-tools
ClickBench test result on commit 976f58995ca63ef05709b1fb0c5e6a9962d6ec4f, data reload: false

query1	0.04	0.04	0.03
query2	0.13	0.11	0.11
query3	0.26	0.20	0.19
query4	1.58	0.20	0.19
query5	0.60	0.57	0.58
query6	1.19	0.73	0.70
query7	0.02	0.02	0.01
query8	0.04	0.04	0.04
query9	0.59	0.52	0.53
query10	0.57	0.60	0.57
query11	0.16	0.11	0.11
query12	0.15	0.12	0.12
query13	0.62	0.60	0.60
query14	2.72	2.70	2.70
query15	0.94	0.86	0.84
query16	0.38	0.39	0.37
query17	1.07	0.99	1.04
query18	0.22	0.20	0.20
query19	1.94	1.96	1.84
query20	0.01	0.02	0.01
query21	15.43	0.93	0.54
query22	0.75	1.19	0.68
query23	14.93	1.40	0.63
query24	7.52	1.94	0.40
query25	0.46	0.08	0.14
query26	0.72	0.16	0.13
query27	0.05	0.05	0.05
query28	9.64	0.84	0.43
query29	12.55	4.06	3.31
query30	0.24	0.08	0.06
query31	2.82	0.59	0.39
query32	3.23	0.55	0.47
query33	3.01	3.07	3.07
query34	15.77	5.14	4.57
query35	4.52	4.55	4.53
query36	0.67	0.50	0.50
query37	0.08	0.06	0.06
query38	0.05	0.05	0.04
query39	0.03	0.02	0.03
query40	0.17	0.13	0.13
query41	0.08	0.03	0.02
query42	0.04	0.03	0.02
query43	0.04	0.03	0.03
Total cold run time: 106.03 s
Total hot run time: 30.87 s
```
</details>

author_association: NONE
user: login: doris-robot
created_at: 2025-04-02T09:09:45Z
body: # BE UT Coverage Report
Increment line coverage `45.27% (67/148)` :tada:

[Increment coverage report](http://coverage.selectdb-in.cc/coverage/976f58995ca63ef05709b1fb0c5e6a9962d6ec4f_976f58995ca63ef05709b1fb0c5e6a9962d6ec4f/increment_report/index.html)
[Complete coverage report](http://coverage.selectdb-in.cc/coverage/976f58995ca63ef05709b1fb0c5e6a9962d6ec4f_976f58995ca63ef05709b1fb0c5e6a9962d6ec4f/report/index.html)
| Category          | Coverage           |
|-------------------|--------------------|
| Function Coverage | 52.11% (13951/26774) |
| Line Coverage     | 40.77% (119989/294310) |
| Region Coverage   | 39.55% (61204/154736) |
| Branch Coverage   | 34.21% (30613/89476) |
author_association: NONE
user: login: github-actions[bot]
created_at: 2025-04-02T09:18:51Z
body: PR approved by at least one committer and no changes requested.
author_association: CONTRIBUTOR