Repository: godotengine/godot
Pull request #103506: Use separate WorkThreadPool for shader compiler.
Author: login: bruvzg
Labels: name: bug
name: topic:core
name: topic:rendering
name: crash
name: regression
Created: 2025-03-03T13:42:59Z | Merged: 2025-03-05T18:08:45Z

Fixes https://github.com/godotengine/godot/issues/102812
Supersede https://github.com/godotengine/godot/pull/103299

Creates dedicated thread pool for `ShaderRD` compilation.

Not a perfect solution, but seems to be working pretty reliably (I was not able to get it crash or deadlock a single time).


Diff:
diff --git a/core/object/worker_thread_pool.cpp b/core/object/worker_thread_pool.cpp
index 08903d61964d..d9ff30576794 100644
--- a/core/object/worker_thread_pool.cpp
+++ b/core/object/worker_thread_pool.cpp
@@ -37,6 +37,8 @@
 
 WorkerThreadPool::Task *const WorkerThreadPool::ThreadData::YIELDING = (Task *)1;
 
+HashMap<StringName, WorkerThreadPool *> WorkerThreadPool::named_pools;
+
 void WorkerThreadPool::Task::free_template_userdata() {
 	ERR_FAIL_NULL(template_userdata);
 	ERR_FAIL_NULL(native_func_userdata);
@@ -184,25 +186,25 @@ void WorkerThreadPool::_thread_function(void *p_user) {
 	while (true) {
 		Task *task_to_process = nullptr;
 		{
-			MutexLock lock(singleton->task_mutex);
+			MutexLock lock(thread_data->pool->task_mutex);
 
-			bool exit = singleton->_handle_runlevel(thread_data, lock);
+			bool exit = thread_data->pool->_handle_runlevel(thread_data, lock);
 			if (unlikely(exit)) {
 				break;
 			}
 
 			thread_data->signaled = false;
 
-			if (singleton->task_queue.first()) {
-				task_to_process = singleton->task_queue.first()->self();
-				singleton->task_queue.remove(singleton->task_queue.first());
+			if (thread_data->pool->task_queue.first()) {
+				task_to_process = thread_data->pool->task_queue.first()->self();
+				thread_data->pool->task_queue.remove(thread_data->pool->task_queue.first());
 			} else {
 				thread_data->cond_var.wait(lock);
 			}
 		}
 
 		if (task_to_process) {
-			singleton->_process_task(task_to_process);
+			thread_data->pool->_process_task(task_to_process);
 		}
 	}
 }
@@ -497,7 +499,7 @@ void WorkerThreadPool::_wait_collaboratively(ThreadData *p_caller_pool_thread, T
 				}
 			}
 
-			if (singleton->task_queue.first()) {
+			if (p_caller_pool_thread->pool->task_queue.first()) {
 				task_to_process = task_queue.first()->self();
 				task_queue.remove(task_queue.first());
 			}
@@ -505,7 +507,9 @@ void WorkerThreadPool::_wait_collaboratively(ThreadData *p_caller_pool_thread, T
 			if (!task_to_process) {
 				p_caller_pool_thread->awaited_task = p_task;
 
-				_unlock_unlockable_mutexes();
+				if (this == singleton) {
+					_unlock_unlockable_mutexes();
+				}
 				relock_unlockables = true;
 
 				p_caller_pool_thread->cond_var.wait(lock);
@@ -514,7 +518,7 @@ void WorkerThreadPool::_wait_collaboratively(ThreadData *p_caller_pool_thread, T
 			}
 		}
 
-		if (relock_unlockables) {
+		if (relock_unlockables && this == singleton) {
 			_lock_unlockable_mutexes();
 		}
 
@@ -690,9 +694,13 @@ void WorkerThreadPool::wait_for_group_task_completion(GroupID p_group) {
 	{
 		Group *group = *groupp;
 
-		_unlock_unlockable_mutexes();
+		if (this == singleton) {
+			_unlock_unlockable_mutexes();
+		}
 		group->done_semaphore.wait();
-		_lock_unlockable_mutexes();
+		if (this == singleton) {
+			_lock_unlockable_mutexes();
+		}
 
 		uint32_t max_users = group->tasks_used + 1; // Add 1 because the thread waiting for it is also user. Read before to avoid another thread freeing task after increment.
 		uint32_t finished_users = group->finished.increment(); // fetch happens before inc, so increment later.
@@ -709,15 +717,15 @@ void WorkerThreadPool::wait_for_group_task_completion(GroupID p_group) {
 #endif
 }
 
-int WorkerThreadPool::get_thread_index() {
+int WorkerThreadPool::get_thread_index() const {
 	Thread::ID tid = Thread::get_caller_id();
-	return singleton->thread_ids.has(tid) ? singleton->thread_ids[tid] : -1;
+	return thread_ids.has(tid) ? thread_ids[tid] : -1;
 }
 
-WorkerThreadPool::TaskID WorkerThreadPool::get_caller_task_id() {
+WorkerThreadPool::TaskID WorkerThreadPool::get_caller_task_id() const {
 	int th_index = get_thread_index();
-	if (th_index != -1 && singleton->threads[th_index].current_task) {
-		return singleton->threads[th_index].current_task->self;
+	if (th_index != -1 && threads[th_index].current_task) {
+		return threads[th_index].current_task->self;
 	} else {
 		return INVALID_TASK_ID;
 	}
@@ -766,6 +774,7 @@ void WorkerThreadPool::init(int p_thread_count, float p_low_priority_task_ratio)
 
 	for (uint32_t i = 0; i < threads.size(); i++) {
 		threads[i].index = i;
+		threads[i].pool = this;
 		threads[i].thread.start(&WorkerThreadPool::_thread_function, &threads[i]);
 		thread_ids.insert(threads[i].thread.get_id(), i);
 	}
@@ -832,10 +841,33 @@ void WorkerThreadPool::_bind_methods() {
 	ClassDB::bind_method(D_METHOD("wait_for_group_task_completion", "group_id"), &WorkerThreadPool::wait_for_group_task_completion);
 }
 
-WorkerThreadPool::WorkerThreadPool() {
-	singleton = this;
+WorkerThreadPool *WorkerThreadPool::get_named_pool(const StringName &p_name) {
+	WorkerThreadPool **pool_ptr = named_pools.getptr(p_name);
+	if (pool_ptr) {
+		return *pool_ptr;
+	} else {
+		WorkerThreadPool *pool = memnew(WorkerThreadPool(false));
+		pool->init();
+		named_pools[p_name] = pool;
+		return pool;
+	}
+}
+
+WorkerThreadPool::WorkerThreadPool(bool p_singleton) {
+	if (p_singleton) {
+		singleton = this;
+	}
 }
 
 WorkerThreadPool::~WorkerThreadPool() {
 	finish();
+
+	if (this == singleton) {
+		singleton = nullptr;
+		for (KeyValue<StringName, WorkerThreadPool *> &E : named_pools) {
+			E.value->finish();
+			memdelete(E.value);
+		}
+		named_pools.clear();
+	}
 }
diff --git a/core/object/worker_thread_pool.h b/core/object/worker_thread_pool.h
index 58e86e3e48eb..76332ec8a3d3 100644
--- a/core/object/worker_thread_pool.h
+++ b/core/object/worker_thread_pool.h
@@ -119,6 +119,7 @@ class WorkerThreadPool : public Object {
 		Task *current_task = nullptr;
 		Task *awaited_task = nullptr; // Null if not awaiting the condition variable, or special value (YIELDING).
 		ConditionVariable cond_var;
+		WorkerThreadPool *pool = nullptr;
 
 		ThreadData() :
 				signaled(false),
@@ -166,6 +167,8 @@ class WorkerThreadPool : public Object {
 
 	uint64_t last_task = 1;
 
+	static HashMap<StringName, WorkerThreadPool *> named_pools;
+
 	static void _thread_function(void *p_user);
 
 	void _process_task(Task *task);
@@ -266,9 +269,12 @@ class WorkerThreadPool : public Object {
 #endif
 	}
 
+	// Note: Do not use this unless you know what you are doing, and it is absolutely necessary. Main thread pool (`get_singleton()`) should be preferred instead.
+	static WorkerThreadPool *get_named_pool(const StringName &p_name);
+
 	static WorkerThreadPool *get_singleton() { return singleton; }
-	static int get_thread_index();
-	static TaskID get_caller_task_id();
+	int get_thread_index() const;
+	TaskID get_caller_task_id() const;
 
 #ifdef THREADS_ENABLED
 	_ALWAYS_INLINE_ static uint32_t thread_enter_unlock_allowance_zone(const MutexLock<BinaryMutex> &p_lock) { return _thread_enter_unlock_allowance_zone(p_lock._get_lock()); }
@@ -285,7 +291,7 @@ class WorkerThreadPool : public Object {
 	void init(int p_thread_count = -1, float p_low_priority_task_ratio = 0.3);
 	void exit_languages_threads();
 	void finish();
-	WorkerThreadPool();
+	WorkerThreadPool(bool p_singleton = true);
 	~WorkerThreadPool();
 };
 
diff --git a/servers/rendering/renderer_rd/shader_rd.cpp b/servers/rendering/renderer_rd/shader_rd.cpp
index cbdbe151c82f..ed9428a6a351 100644
--- a/servers/rendering/renderer_rd/shader_rd.cpp
+++ b/servers/rendering/renderer_rd/shader_rd.cpp
@@ -555,7 +555,7 @@ void ShaderRD::_compile_version_start(Version *p_version, int p_group) {
 	compile_data.version = p_version;
 	compile_data.group = p_group;
 
-	WorkerThreadPool::GroupID group_task = WorkerThreadPool::get_singleton()->add_template_group_task(this, &ShaderRD::_compile_variant, compile_data, group_to_variant_map[p_group].size(), -1, true, SNAME("ShaderCompilation"));
+	WorkerThreadPool::GroupID group_task = WorkerThreadPool::get_named_pool(SNAME("ShaderCompilationPool"))->add_template_group_task(this, &ShaderRD::_compile_variant, compile_data, group_to_variant_map[p_group].size(), -1, true, SNAME("ShaderCompilation"));
 	p_version->group_compilation_tasks.write[p_group] = group_task;
 }
 
@@ -563,9 +563,8 @@ void ShaderRD::_compile_version_end(Version *p_version, int p_group) {
 	if (p_version->group_compilation_tasks.size() <= p_group || p_version->group_compilation_tasks[p_group] == 0) {
 		return;
 	}
-
 	WorkerThreadPool::GroupID group_task = p_version->group_compilation_tasks[p_group];
-	WorkerThreadPool::get_singleton()->wait_for_group_task_completion(group_task);
+	WorkerThreadPool::get_named_pool(SNAME("ShaderCompilationPool"))->wait_for_group_task_completion(group_task);
 	p_version->group_compilation_tasks.write[p_group] = 0;
 
 	bool all_valid = true;


Reviews:
user: login: stuartcarnie
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-03T18:27:03Z
user: login: bruvzg
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-03T20:25:37Z
user: login: DarioSamo
body: Overall I think this does the job considering this is a hotfix for a critical issue that has presented itself with 4.4, but I think we should consider extending WorkerThreadPool to work with a design where we're just allowed to create pools locally to the context of what we want to use. This is not entirely necessary for this PR given the nature of the issue at the moment.

For example, instead of a name, we'd just make the object inside ShaderRD's global context instead, which would make it a bit more natural and it could allow us to specify a thread count specifically for shader compilation easily.
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-03-04T14:24:26Z
user: login: clayjohn
body: Thanks for answering my questions! I think this makes sense as a workaround for now. 

I am concerned that this named WTP system will get blindly expanded in the future into some sort of generic feature, but I think for now we should go forward with this PR
state: APPROVED
author_association: MEMBER
submitted_at: 2025-03-05T00:29:37Z

Review comments:
diff_hunk: @@ -832,10 +841,33 @@ void WorkerThreadPool::_bind_methods() {
 	ClassDB::bind_method(D_METHOD("wait_for_group_task_completion", "group_id"), &WorkerThreadPool::wait_for_group_task_completion);
 }
 
-WorkerThreadPool::WorkerThreadPool() {
-	singleton = this;
+WorkerThreadPool *WorkerThreadPool::get_named_pool(const StringName &p_name) {
+	WorkerThreadPool **pool_ptr = named_pools.getptr(p_name);
+	if (pool_ptr) {
+		return *pool_ptr;
+	} else {
+		WorkerThreadPool *pool = memnew(WorkerThreadPool(false));
+		pool->init();
+		named_pools[p_name] = pool;
+		return pool;
+	}
+}
path: core/object/worker_thread_pool.cpp
user: login: stuartcarnie
body: I would advocate for a separate API to create a named pool and have this API fail if the pool doesn't exist.
created_at: 2025-03-03T18:25:46Z
author_association: CONTRIBUTOR
diff_hunk: @@ -832,10 +841,33 @@ void WorkerThreadPool::_bind_methods() {
 	ClassDB::bind_method(D_METHOD("wait_for_group_task_completion", "group_id"), &WorkerThreadPool::wait_for_group_task_completion);
 }
 
-WorkerThreadPool::WorkerThreadPool() {
-	singleton = this;
+WorkerThreadPool *WorkerThreadPool::get_named_pool(const StringName &p_name) {
+	WorkerThreadPool **pool_ptr = named_pools.getptr(p_name);
+	if (pool_ptr) {
+		return *pool_ptr;
+	} else {
+		WorkerThreadPool *pool = memnew(WorkerThreadPool(false));
+		pool->init();
+		named_pools[p_name] = pool;
+		return pool;
+	}
+}
path: core/object/worker_thread_pool.cpp
user: login: bruvzg
body: If it was a public API and intended for generic use, I would agree. But it's only intended for a single case only, and in general should be avoided.
created_at: 2025-03-03T20:25:37Z
author_association: MEMBER

PR comments:
user: login: DarioSamo
created_at: 2025-03-04T14:22:16Z
body: I'm a fan of the concept of named pools, but does this implementation mean that it'll double up the available thread count from whatever the user configured? I've looked at the PR and didn't see if we have any logic to handle on how to respect that setting.

It's still an acceptable hotfix in my eyes as the workload for shader compilation doesn't come in that frequently.
author_association: CONTRIBUTOR
user: login: bruvzg
created_at: 2025-03-04T14:30:44Z
body: > I'm a fan of the concept of named pools, but does this implementation mean that it'll double up the available thread count from whatever the user configured?

Yes, it is doubling the number of threads (we do not want to constantly spawn new threads, but simply having 2 threads per core should not cuase any problems).
author_association: MEMBER
user: login: clayjohn
created_at: 2025-03-04T16:59:58Z
body: I am a little bit unclear about the tradeoffs here. We are spawning a bunch of extra threads that will ultimately sit idle. Will they consume any resources once we finish compiling shaders? Are their any OS limitations we need to be wary of (especially on Mobile/Web)?
author_association: MEMBER
user: login: bruvzg
created_at: 2025-03-04T17:12:57Z
body: > Web

We do not have WebGPU support (RenderingDevice), so it will have zero effect on web.

> Are their any OS limitations we need to be wary of

There is a per-process handle limit, but as far as I know it's in the thousands or tens of thousands depending on platform.

> Will they consume any resources once we finish compiling shaders?

512 KiB of memory for stack + metadata per thread. At least on macOS, Xcode profiler show completely flat line for idle threads with no detectable CPU activity at all.

Also, the new pool is only initialized if there are shaders to compile, as long as it's loaded from cache it won't, so it should affect only the first start or editor.
author_association: MEMBER
user: login: bruvzg
created_at: 2025-03-05T15:52:35Z
body: > I am concerned that this named WTP system will get blindly expanded in the future into some sort of generic feature, but I think for now we should go forward with this PR

That's the main reason I do not want to create a nice API for it, and implemented it as a single function. It won't prevent it from being used, but at least have more chances for a comment advising not to use it to be noticed.
author_association: MEMBER
user: login: Repiteo
created_at: 2025-03-05T18:09:16Z
body: Thanks!
author_association: CONTRIBUTOR
user: login: akien-mga
created_at: 2025-03-12T11:22:53Z
body: Cherry-picked for 4.4.1.
author_association: MEMBER