Repository: LunarG/gfxreconstruct
Pull request #2054: Fix issues around async pipeline-handle assignment/cleanup
Author: login: fabian-lunarg
Created: 2025-03-05T12:20:49Z | Merged: 2025-03-11T15:36:36Z

changes to pipeline handle-synchronization when using async compilation.

- fixes issues in combination with `-m rebind` option and address-replacement
- fixes validation-errors for pipeline-handle leaks when using asynchronous pipeline-compilation (`--pcj X`)
- found and removed some dead code
- added debug-labels to internal VkPipelines in VulkanAddressReplacer during debugging

to reproduce the reported issue with address-replacement this sample can be used:
https://github.com/KhronosGroup/Vulkan-Samples/tree/main/samples/extensions/ray_tracing_reflection
`gfxrecon-replay -m rebind --pcj -1 ray_tracing_reflection.gfxr`


Diff:
diff --git a/framework/decode/common_handle_mapping_util.h b/framework/decode/common_handle_mapping_util.h
index 7a0b1d1477..21a8f2246c 100644
--- a/framework/decode/common_handle_mapping_util.h
+++ b/framework/decode/common_handle_mapping_util.h
@@ -48,22 +48,12 @@ static typename T::HandleType MapHandle(format::HandleId             id,
         if (info != nullptr)
         {
             handle = info->handle;
-
-            if constexpr (has_handle_future_v<T>)
-            {
-                if (info->handle == VK_NULL_HANDLE && info->future.valid())
-                {
-                    const auto& [result, async_handles] = info->future.get();
-                    handle                              = async_handles[info->future_handle_index];
-                }
-            }
         }
         else
         {
             GFXRECON_LOG_WARNING("Failed to map handle for object id %" PRIu64, id);
         }
     }
-
     return handle;
 }
 
@@ -201,32 +191,6 @@ static void AddHandleArray(format::HandleId              parent_id,
     }
 }
 
-template <typename T>
-static void AddHandleArrayAsync(format::HandleId        parent_id,
-                                const format::HandleId* ids,
-                                size_t                  ids_len,
-                                CommonObjectInfoTable*  object_info_table,
-                                void (CommonObjectInfoTable::*AddFunc)(T&&),
-                                std::shared_future<handle_create_result_t<typename T::HandleType>> future)
-{
-    static_assert(has_handle_future_v<T>, "handle-type does not support asynchronous creation");
-    assert(object_info_table != nullptr);
-
-    if (ids != nullptr)
-    {
-        for (size_t i = 0; i < ids_len; ++i)
-        {
-            T info;
-            info.handle              = VK_NULL_HANDLE; // handle does not yet exist
-            info.capture_id          = ids[i];
-            info.parent_id           = parent_id;
-            info.future              = future;
-            info.future_handle_index = i;
-            (object_info_table->*AddFunc)(std::move(info));
-        }
-    }
-}
-
 template <typename T>
 static void AddHandleArrayAsync(format::HandleId        parent_id,
                                 const format::HandleId* ids,
@@ -245,13 +209,13 @@ static void AddHandleArrayAsync(format::HandleId        parent_id,
 
         for (size_t i = 0; i < ids_len; ++i)
         {
-            auto info_iter                 = std::next(initial_infos.begin(), i);
-            info_iter->handle              = VK_NULL_HANDLE; // handle does not yet exist
-            info_iter->capture_id          = ids[i];
-            info_iter->parent_id           = parent_id;
-            info_iter->future              = future;
-            info_iter->future_handle_index = i;
-            (object_info_table->*AddFunc)(std::move(*info_iter));
+            auto& initial_info               = initial_infos[i];
+            initial_info.handle              = VK_NULL_HANDLE; // handle does not yet exist
+            initial_info.capture_id          = ids[i];
+            initial_info.parent_id           = parent_id;
+            initial_info.future              = future;
+            initial_info.future_handle_index = i;
+            (object_info_table->*AddFunc)(std::move(initial_info));
         }
     }
 }
diff --git a/framework/decode/vulkan_address_replacer.cpp b/framework/decode/vulkan_address_replacer.cpp
index bc64747774..54e456440e 100644
--- a/framework/decode/vulkan_address_replacer.cpp
+++ b/framework/decode/vulkan_address_replacer.cpp
@@ -1301,6 +1301,16 @@ bool VulkanAddressReplacer::init_pipeline()
             GFXRECON_LOG_ERROR("VulkanAddressReplacer: pipeline creation failed");
         }
 
+        if (set_debug_utils_object_name_fn_)
+        {
+            VkDebugUtilsObjectNameInfoEXT object_name_info = {};
+            object_name_info.sType                         = VK_STRUCTURE_TYPE_DEBUG_UTILS_OBJECT_NAME_INFO_EXT;
+            object_name_info.objectType                    = VK_OBJECT_TYPE_PIPELINE;
+            object_name_info.objectHandle                  = VK_HANDLE_TO_UINT64(out_pipeline);
+            object_name_info.pObjectName                   = "VulkanAddressReplacer internal pipeline";
+            set_debug_utils_object_name_fn_(device_, &object_name_info);
+        }
+
         if (compute_module != VK_NULL_HANDLE)
         {
             device_table_->DestroyShaderModule(device_, compute_module, nullptr);
diff --git a/framework/decode/vulkan_object_cleanup_util.cpp b/framework/decode/vulkan_object_cleanup_util.cpp
index 86f492efd6..3c3f8efb5c 100644
--- a/framework/decode/vulkan_object_cleanup_util.cpp
+++ b/framework/decode/vulkan_object_cleanup_util.cpp
@@ -74,7 +74,13 @@ void FreeChildObjects(CommonObjectInfoTable* table,
     // Visit all table entries and sort them by parent ID.  Using unordered_map to filter duplicate handles.
     std::unordered_map<format::HandleId, std::unordered_map<typename T::HandleType, const T*>> objects;
 
-    (table->*VisitFunc)([&](const T* info) { AddChildObject(&objects, info); });
+    (table->*VisitFunc)([&](const T* info) {
+        if constexpr (has_handle_future_v<T>)
+        {
+            sync_handle(const_cast<T*>(info));
+        }
+        AddChildObject(&objects, info);
+    });
 
     for (const auto& entry : objects)
     {
diff --git a/framework/decode/vulkan_object_info_table_base.h b/framework/decode/vulkan_object_info_table_base.h
index 0f40ddf9bd..479d4cf8c2 100644
--- a/framework/decode/vulkan_object_info_table_base.h
+++ b/framework/decode/vulkan_object_info_table_base.h
@@ -50,6 +50,23 @@ struct has_handle_future<T, decltype((void)T::future, 0)>
 template <typename T>
 inline constexpr bool has_handle_future_v = has_handle_future<T>::value;
 
+/**
+ * @brief   sync_handle will block and wait for pending handle-creation operations, then populate the object's handle.
+ *
+ * @tparam  T       object-type
+ * @param   object  a provided pointer to an asynchronous object-wrapper
+ */
+template <typename T>
+static inline void sync_handle(T* object)
+{
+    static_assert(has_handle_future_v<T>);
+    if (object != nullptr && object->handle == VK_NULL_HANDLE && object->future.valid())
+    {
+        const auto& [result, async_handles] = object->future.get();
+        object->handle                      = async_handles[object->future_handle_index];
+    }
+}
+
 // NOTE: There's nothing VulkanSpecific in these utilities
 // TODO: Find a better home for these
 
@@ -99,7 +116,13 @@ struct ObjectInfoGetter : public ObjectInfoGetterBase
     template <typename Map, typename MappedTypePtr = ConstCorrectMappedTypePtr<Map>>
     MappedTypePtr operator()(format::HandleId id, Map* map)
     {
-        return GetObjectInfoImpl<Map, MappedTypePtr>(id, map);
+        auto* object_info = GetObjectInfoImpl<Map, MappedTypePtr>(id, map);
+
+        if constexpr (has_handle_future_v<T>)
+        {
+            sync_handle(const_cast<T*>(object_info));
+        }
+        return object_info;
     }
 };
 
diff --git a/framework/decode/vulkan_replay_consumer_base.cpp b/framework/decode/vulkan_replay_consumer_base.cpp
index a45a62fa64..a2c1169bfb 100644
--- a/framework/decode/vulkan_replay_consumer_base.cpp
+++ b/framework/decode/vulkan_replay_consumer_base.cpp
@@ -249,10 +249,10 @@ VulkanReplayConsumerBase::VulkanReplayConsumerBase(std::shared_ptr<application::
 
 VulkanReplayConsumerBase::~VulkanReplayConsumerBase()
 {
-    for (const std::pair<format::HandleId, std::pair<const VulkanDeviceInfo*, VkPipelineCache>>& elt :
-         tracked_pipeline_caches_)
+    for (const auto& [handle_id, cache_pair] : tracked_pipeline_caches_)
     {
-        SavePipelineCache(elt.first, elt.second.first, elt.second.second);
+        const auto& [device_info, pipeline_cache] = cache_pair;
+        SavePipelineCache(handle_id, device_info, pipeline_cache);
     }
 
     // Idle all devices before destroying other resources.
@@ -261,6 +261,10 @@ VulkanReplayConsumerBase::~VulkanReplayConsumerBase()
     // free replacer internal vulkan-resources
     _device_address_replacers.clear();
 
+    // process queued async tasks
+    background_queue_.join_all();
+    main_thread_queue_.poll();
+
     // Cleanup screenshot resources before destroying device.
     object_info_table_->VisitVkDeviceInfo([this](const VulkanDeviceInfo* info) {
         assert(info != nullptr);
diff --git a/framework/decode/vulkan_replay_consumer_base.h b/framework/decode/vulkan_replay_consumer_base.h
index 7a09b61a37..68a1f07752 100644
--- a/framework/decode/vulkan_replay_consumer_base.h
+++ b/framework/decode/vulkan_replay_consumer_base.h
@@ -406,26 +406,6 @@ class VulkanReplayConsumerBase : public VulkanConsumer
         handle_mapping::AddHandleArray(parent_id, ids, ids_len, handles, handles_len, object_info_table_, AddFunc);
     }
 
-    template <typename T>
-    void AddHandlesAsync(format::HandleId        parent_id,
-                         const format::HandleId* ids,
-                         size_t                  ids_len,
-                         void (CommonObjectInfoTable::*AddFunc)(T&&),
-                         std::function<handle_create_result_t<typename T::HandleType>()> create_function)
-    {
-        if (create_function)
-        {
-            std::shared_future<handle_create_result_t<typename T::HandleType>> result_future =
-                background_queue_.post(std::move(create_function));
-
-            // poll in case there are no worker-threads
-            background_queue_.poll();
-
-            handle_mapping::AddHandleArrayAsync(
-                parent_id, ids, ids_len, object_info_table_, AddFunc, std::move(result_future));
-        }
-    }
-
     template <typename T>
     void AddHandlesAsync(format::HandleId        parent_id,
                          const format::HandleId* ids,


Reviews:
user: login: MarkY-LunarG
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-11T13:10:05Z
user: login: MarkY-LunarG
body: What is the test case for this as well?  I know you don't have one, but it would be nice to at least list what application showed this problem and how you tested.
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-03-11T13:12:20Z
user: login: fabian-lunarg
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-11T13:28:45Z

Review comments:
diff_hunk: @@ -50,6 +50,23 @@ struct has_handle_future<T, decltype((void)T::future, 0)>
 template <typename T>
 inline constexpr bool has_handle_future_v = has_handle_future<T>::value;
 
+/**
+ * @brief   sync_handle will block and wait for pending handle-creation operations, then populate the object's handle.
+ *
+ * @tparam  T       object-type
+ * @param   object  a provided pointer to an asynchronous object-wrapper
+ */
+template <typename T>
+static inline void sync_handle(T* object)
+{
+    static_assert(has_handle_future_v<T>);
+    if (object != nullptr && object->handle == VK_NULL_HANDLE && object->future.valid())
path: framework/decode/vulkan_object_info_table_base.h
user: login: MarkY-LunarG
body: So, I don't know this code, but how does it perform the synchronize?  Is it all in the "object->future.valid()" call?
created_at: 2025-03-11T13:10:05Z
author_association: CONTRIBUTOR
diff_hunk: @@ -50,6 +50,23 @@ struct has_handle_future<T, decltype((void)T::future, 0)>
 template <typename T>
 inline constexpr bool has_handle_future_v = has_handle_future<T>::value;
 
+/**
+ * @brief   sync_handle will block and wait for pending handle-creation operations, then populate the object's handle.
+ *
+ * @tparam  T       object-type
+ * @param   object  a provided pointer to an asynchronous object-wrapper
+ */
+template <typename T>
+static inline void sync_handle(T* object)
+{
+    static_assert(has_handle_future_v<T>);
+    if (object != nullptr && object->handle == VK_NULL_HANDLE && object->future.valid())
path: framework/decode/vulkan_object_info_table_base.h
user: login: fabian-lunarg
body: the blocking call is: `object->future.get()` (future is a `std::shared_future`)
created_at: 2025-03-11T13:28:45Z
author_association: CONTRIBUTOR

PR comments:
user: login: ci-tester-lunarg
created_at: 2025-03-05T12:20:52Z
body: CI gfxreconstruct build queued with queue ID 387022.
author_association: COLLABORATOR
user: login: ci-tester-lunarg
created_at: 2025-03-05T12:44:58Z
body: CI gfxreconstruct build queued with queue ID 387028.
author_association: COLLABORATOR
user: login: ci-tester-lunarg
created_at: 2025-03-05T13:34:05Z
body: CI gfxreconstruct build # 6257 running.
author_association: COLLABORATOR
user: login: ci-tester-lunarg
created_at: 2025-03-05T14:13:09Z
body: CI gfxreconstruct build # 6257 failed.
author_association: COLLABORATOR
user: login: ci-tester-lunarg
created_at: 2025-03-05T15:08:55Z
body: CI gfxreconstruct build queued with queue ID 387184.
author_association: COLLABORATOR
user: login: ci-tester-lunarg
created_at: 2025-03-05T16:49:55Z
body: CI gfxreconstruct build # 6262 running.
author_association: COLLABORATOR
user: login: ci-tester-lunarg
created_at: 2025-03-05T17:29:15Z
body: CI gfxreconstruct build # 6262 passed.
author_association: COLLABORATOR
user: login: ci-tester-lunarg
created_at: 2025-03-11T07:34:11Z
body: CI gfxreconstruct build queued with queue ID 392237.
author_association: COLLABORATOR
user: login: ci-tester-lunarg
created_at: 2025-03-11T07:35:01Z
body: CI gfxreconstruct build # 6297 running.
author_association: COLLABORATOR
user: login: ci-tester-lunarg
created_at: 2025-03-11T08:31:21Z
body: CI gfxreconstruct build # 6297 passed.
author_association: COLLABORATOR
user: login: fabian-lunarg
created_at: 2025-03-11T13:26:30Z
body: > What is the test case for this as well? I know you don't have one, but it would be nice to at least list what application showed this problem and how you tested.

we do have an internal test-case (combining `--pipeline-compile-jobs -1 -m rebind`) in our CI,
at least for leaked handles during cleanup.

to reproduce the reported issue with address-replacement this sample can be used:
https://github.com/KhronosGroup/Vulkan-Samples/tree/main/samples/extensions/ray_tracing_reflection
again using `--pipeline-compile-jobs -1 -m rebind`

lastly, I'm often also testing with my `pbr_viewer` from here https://github.com/crocdialer/vierkant_projects
(there are nightlies for linux&windows). 
author_association: CONTRIBUTOR
user: login: MarkY-LunarG
created_at: 2025-03-11T13:34:13Z
body: > > What is the test case for this as well? I know you don't have one, but it would be nice to at least list what application showed this problem and how you tested.
> 
> we do have an internal test-case (combining `--pipeline-compile-jobs -1 -m rebind`) in our CI, at least for leaked handles during cleanup.
> 
> to reproduce the reported issue with address-replacement this sample can be used: https://github.com/KhronosGroup/Vulkan-Samples/tree/main/samples/extensions/ray_tracing_reflection again using `--pipeline-compile-jobs -1 -m rebind`
> 
> lastly, I'm often also testing with my `pbr_viewer` from here https://github.com/crocdialer/vierkant_projects (there are nightlies for linux&windows).

Cool.  Can you modify the last commit with some of that info?  That's all I think is necessary.  Otherwise, LGTM.
author_association: CONTRIBUTOR
user: login: ci-tester-lunarg
created_at: 2025-03-11T13:42:45Z
body: CI gfxreconstruct build queued with queue ID 392493.
author_association: COLLABORATOR
user: login: ci-tester-lunarg
created_at: 2025-03-11T14:27:31Z
body: CI gfxreconstruct build # 6299 running.
author_association: COLLABORATOR
user: login: ci-tester-lunarg
created_at: 2025-03-11T15:34:46Z
body: CI gfxreconstruct build # 6299 passed.
author_association: COLLABORATOR