Repository: intel/llvm
Pull request #17546: [clang-linker-wrapper] Dump any device code format
Author: login: bader
Created: 2025-03-20T01:42:34Z | Merged: 2025-03-26T00:42:10Z

Despite the option name sycl-dump-device-code it enables dumping of only SPIR-V files emitted by the translator tool. This change dumps device code right before embedding it into the fat object. The tool dumps offloading image of any format.


Diff:
diff --git a/clang/test/Driver/linker-wrapper-sycl.cpp b/clang/test/Driver/linker-wrapper-sycl.cpp
index d32f33149c3b3..6ca069d82d5f5 100644
--- a/clang/test/Driver/linker-wrapper-sycl.cpp
+++ b/clang/test/Driver/linker-wrapper-sycl.cpp
@@ -47,11 +47,6 @@
 // RUN: clang-linker-wrapper -sycl-device-libraries=%t.devicelib.o -sycl-post-link-options="SYCL_POST_LINK_OPTIONS" -llvm-spirv-options="LLVM_SPIRV_OPTIONS" "--host-triple=x86_64-unknown-linux-gnu" "--triple=spir64" "--linker-path=/usr/bin/ld" -shared "--" HOST_LINKER_FLAGS "-dynamic-linker" HOST_DYN_LIB "-o" "a.out" HOST_LIB_PATH HOST_STAT_LIB %t.o --dry-run 2>&1 | FileCheck -check-prefix=CHK-SHARED %s
 // CHK-SHARED: "{{.*}}clang"{{.*}} -fPIC
 
-// RUN: rm %T/linker_wrapper_dump || true
-// RUN: clang-linker-wrapper -sycl-dump-device-code=%T/linker_wrapper_dump -sycl-device-libraries=%t.devicelib.o "--host-triple=x86_64-unknown-linux-gnu" "--triple=spir64" "--linker-path=/usr/bin/ld" -shared "--" HOST_LINKER_FLAGS "-dynamic-linker" HOST_DYN_LIB "-o" "a.out" HOST_LIB_PATH HOST_STAT_LIB %t.o --dry-run
-// RUN: ls %T/linker_wrapper_dump | FileCheck -check-prefix=CHK-SYCL-DUMP-DEVICE %s
-// CHK-SYCL-DUMP-DEVICE: {{.*}}.spv
-
 /// Check for list of commands for standalone clang-linker-wrapper run for sycl (AOT for Intel GPU)
 // -------
 // Generate .o file as linker wrapper input.
diff --git a/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp b/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp
index fadc7863968ca..adb942a6e6a31 100644
--- a/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp
+++ b/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp
@@ -151,7 +151,7 @@ static std::optional<llvm::module_split::IRSplitMode> SYCLModuleSplitMode;
 
 static bool UseSYCLPostLinkTool;
 
-static SmallString<128> SPIRVDumpDir;
+static SmallString<128> OffloadImageDumpDir;
 
 using OffloadingImage = OffloadBinary::OffloadingImage;
 
@@ -923,29 +923,6 @@ static Expected<StringRef> runLLVMToSPIRVTranslation(StringRef File,
   if (Error Err = executeCommands(*LLVMToSPIRVPath, CmdArgs))
     return std::move(Err);
 
-  if (!SPIRVDumpDir.empty()) {
-    std::error_code EC =
-        llvm::sys::fs::create_directory(SPIRVDumpDir, /*IgnoreExisting*/ true);
-    if (EC)
-      return createStringError(
-          EC,
-          formatv("failed to create dump directory. path: {0}, error_code: {1}",
-                  SPIRVDumpDir, EC.value()));
-
-    StringRef Sep = llvm::sys::path::get_separator();
-    StringRef Path = *TempFileOrErr;
-    StringRef Filename = Path.rsplit(Sep).second;
-    SmallString<128> CopyPath = SPIRVDumpDir;
-    CopyPath.append(Filename);
-    EC = llvm::sys::fs::copy_file(Path, CopyPath);
-    if (EC)
-      return createStringError(
-          EC,
-          formatv(
-              "failed to copy file. original: {0}, copy: {1}, error_code: {2}",
-              Path, CopyPath, EC.value()));
-  }
-
   return *TempFileOrErr;
 }
 
@@ -1103,6 +1080,18 @@ wrapSYCLBinariesFromFile(std::vector<module_split::SplitModule> &SplitModules,
     RegularTarget = "spir64";
 
   for (auto &SI : SplitModules) {
+    if (!OffloadImageDumpDir.empty()) {
+      StringRef CopyFrom = SI.ModuleFilePath;
+      SmallString<128> CopyTo = OffloadImageDumpDir;
+      StringRef Filename = sys::path::filename(CopyFrom);
+      CopyTo.append(Filename);
+      std::error_code EC = sys::fs::copy_file(CopyFrom, CopyTo);
+      if (EC)
+        return createStringError(EC, formatv("failed to copy file. From: "
+                                             "{0} to: {1}, error_code: {2}",
+                                             CopyFrom, CopyTo, EC.value()));
+    }
+
     auto MBOrDesc = MemoryBuffer::getFile(SI.ModuleFilePath);
     if (!MBOrDesc)
       return createFileError(SI.ModuleFilePath, MBOrDesc.getError());
@@ -2618,13 +2607,11 @@ int main(int Argc, char **Argv) {
 
   if (Args.hasArg(OPT_sycl_dump_device_code_EQ)) {
     Arg *A = Args.getLastArg(OPT_sycl_dump_device_code_EQ);
-    SmallString<128> Dir(A->getValue());
-    if (Dir.empty())
-      llvm::sys::path::native(Dir = "./");
+    OffloadImageDumpDir = A->getValue();
+    if (OffloadImageDumpDir.empty())
+      sys::path::native(OffloadImageDumpDir = "./");
     else
-      Dir.append(llvm::sys::path::get_separator());
-
-    SPIRVDumpDir = Dir;
+      OffloadImageDumpDir.append(sys::path::get_separator());
   }
 
   {
diff --git a/clang/tools/clang-linker-wrapper/LinkerWrapperOpts.td b/clang/tools/clang-linker-wrapper/LinkerWrapperOpts.td
index 1885fb430b2bd..d7a1aa02e8978 100644
--- a/clang/tools/clang-linker-wrapper/LinkerWrapperOpts.td
+++ b/clang/tools/clang-linker-wrapper/LinkerWrapperOpts.td
@@ -242,7 +242,7 @@ Flags<[WrapperOnlyOption]>,  HelpText<"Embed LLVM IR for runtime kernel fusion">
 
 def sycl_dump_device_code_EQ : Joined<["--", "-"], "sycl-dump-device-code=">,
   Flags<[WrapperOnlyOption]>,
-  HelpText<"Path to the folder where the tool dumps SPIR-V device code. Other formats aren't dumped.">;
+  HelpText<"Directory to dump offloading images to.">;
 
 // Options to enable/disable device dynamic linking.
 def sycl_allow_device_image_dependencies : Flag<["--", "-"], "sycl-allow-device-image-dependencies">,


Reviews:
user: login: maksimsab
state: CHANGES_REQUESTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-20T12:40:49Z
user: login: bader
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-20T15:05:49Z
user: login: bader
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-20T15:07:00Z
user: login: srividya-sundaram
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-03-24T16:55:33Z
user: login: asudarsa
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-24T19:38:30Z
user: login: asudarsa
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-24T19:42:50Z
user: login: asudarsa
body: LGTM. Suggested a nit and a possible followup. I am trying to think if we can add some testing for this.
We can add in a separate PR as well.

Thanks
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-03-24T19:52:44Z
user: login: bader
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-24T23:05:16Z
user: login: bader
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-24T23:07:52Z
user: login: maksimsab
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-25T15:43:13Z
user: login: bader
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-25T16:17:15Z
user: login: maksimsab
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-25T16:23:42Z
user: login: bader
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-25T16:35:10Z
user: login: maksimsab
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-03-25T16:48:51Z

Review comments:
diff_hunk: @@ -2624,7 +2613,15 @@ int main(int Argc, char **Argv) {
     else
       Dir.append(llvm::sys::path::get_separator());
 
-    SPIRVDumpDir = Dir;
+    OffloadImageDumpDir = Dir;
+
+    std::error_code EC = llvm::sys::fs::create_directory(
path: clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp
user: login: maksimsab
body: In case of `dry-run` it is uncommon to leave traces.
created_at: 2025-03-20T12:31:53Z
author_association: CONTRIBUTOR
diff_hunk: @@ -47,11 +47,6 @@
 // RUN: clang-linker-wrapper -sycl-device-libraries=%t.devicelib.o -sycl-post-link-options="SYCL_POST_LINK_OPTIONS" -llvm-spirv-options="LLVM_SPIRV_OPTIONS" "--host-triple=x86_64-unknown-linux-gnu" "--triple=spir64" "--linker-path=/usr/bin/ld" -shared "--" HOST_LINKER_FLAGS "-dynamic-linker" HOST_DYN_LIB "-o" "a.out" HOST_LIB_PATH HOST_STAT_LIB %t.o --dry-run 2>&1 | FileCheck -check-prefix=CHK-SHARED %s
 // CHK-SHARED: "{{.*}}clang"{{.*}} -fPIC
 
-// RUN: rm %T/linker_wrapper_dump || true
-// RUN: clang-linker-wrapper -sycl-dump-device-code=%T/linker_wrapper_dump -sycl-device-libraries=%t.devicelib.o "--host-triple=x86_64-unknown-linux-gnu" "--triple=spir64" "--linker-path=/usr/bin/ld" -shared "--" HOST_LINKER_FLAGS "-dynamic-linker" HOST_DYN_LIB "-o" "a.out" HOST_LIB_PATH HOST_STAT_LIB %t.o --dry-run
path: clang/test/Driver/linker-wrapper-sycl.cpp
user: login: maksimsab
body: Shouldn't we leave that?
created_at: 2025-03-20T12:40:38Z
author_association: CONTRIBUTOR
diff_hunk: @@ -2624,7 +2613,15 @@ int main(int Argc, char **Argv) {
     else
       Dir.append(llvm::sys::path::get_separator());
 
-    SPIRVDumpDir = Dir;
+    OffloadImageDumpDir = Dir;
+
+    std::error_code EC = llvm::sys::fs::create_directory(
path: clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp
user: login: bader
body: Fixed.
I'd like to note that community code doesn't leave traces, whereas SYCL addition leave traces all over the place.
created_at: 2025-03-20T15:05:49Z
author_association: CONTRIBUTOR
diff_hunk: @@ -47,11 +47,6 @@
 // RUN: clang-linker-wrapper -sycl-device-libraries=%t.devicelib.o -sycl-post-link-options="SYCL_POST_LINK_OPTIONS" -llvm-spirv-options="LLVM_SPIRV_OPTIONS" "--host-triple=x86_64-unknown-linux-gnu" "--triple=spir64" "--linker-path=/usr/bin/ld" -shared "--" HOST_LINKER_FLAGS "-dynamic-linker" HOST_DYN_LIB "-o" "a.out" HOST_LIB_PATH HOST_STAT_LIB %t.o --dry-run 2>&1 | FileCheck -check-prefix=CHK-SHARED %s
 // CHK-SHARED: "{{.*}}clang"{{.*}} -fPIC
 
-// RUN: rm %T/linker_wrapper_dump || true
-// RUN: clang-linker-wrapper -sycl-dump-device-code=%T/linker_wrapper_dump -sycl-device-libraries=%t.devicelib.o "--host-triple=x86_64-unknown-linux-gnu" "--triple=spir64" "--linker-path=/usr/bin/ld" -shared "--" HOST_LINKER_FLAGS "-dynamic-linker" HOST_DYN_LIB "-o" "a.out" HOST_LIB_PATH HOST_STAT_LIB %t.o --dry-run
path: clang/test/Driver/linker-wrapper-sycl.cpp
user: login: bader
body: No. This test is utterly broken. Let me know if you have suggestions on writing a working test.
created_at: 2025-03-20T15:07:00Z
author_association: CONTRIBUTOR
diff_hunk: @@ -242,7 +242,7 @@ Flags<[WrapperOnlyOption]>,  HelpText<"Embed LLVM IR for runtime kernel fusion">
 
 def sycl_dump_device_code_EQ : Joined<["--", "-"], "sycl-dump-device-code=">,
   Flags<[WrapperOnlyOption]>,
-  HelpText<"Path to the folder where the tool dumps SPIR-V device code. Other formats aren't dumped.">;
+  HelpText<"Directory to dump offloading images to.">;
path: clang/tools/clang-linker-wrapper/LinkerWrapperOpts.td
user: login: asudarsa
body: This option mirrors the user facing option 'sycl-dump-device-code' which was introduced for the old offloading model here: https://github.com/intel/llvm/pull/11132. In that PR, I see that we use this only for dumping SPIR-V code. If the idea is to extend this option to cover all possible device outputs, may be we should do this for the old offloading model as well?

Thanks
created_at: 2025-03-24T19:38:29Z
author_association: CONTRIBUTOR
diff_hunk: @@ -2620,11 +2609,22 @@ int main(int Argc, char **Argv) {
     Arg *A = Args.getLastArg(OPT_sycl_dump_device_code_EQ);
     SmallString<128> Dir(A->getValue());
     if (Dir.empty())
-      llvm::sys::path::native(Dir = "./");
+      sys::path::native(Dir = "./");
     else
-      Dir.append(llvm::sys::path::get_separator());
-
-    SPIRVDumpDir = Dir;
+      Dir.append(sys::path::get_separator());
+
+    OffloadImageDumpDir = Dir;
+
+    if (!DryRun) {
path: clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp
user: login: asudarsa
body: May be we can avoid the use of dry-run with -sycl-dump-device-code and emit an error if user does that?

Thanks
created_at: 2025-03-24T19:42:50Z
author_association: CONTRIBUTOR
diff_hunk: @@ -242,7 +242,7 @@ Flags<[WrapperOnlyOption]>,  HelpText<"Embed LLVM IR for runtime kernel fusion">
 
 def sycl_dump_device_code_EQ : Joined<["--", "-"], "sycl-dump-device-code=">,
   Flags<[WrapperOnlyOption]>,
-  HelpText<"Path to the folder where the tool dumps SPIR-V device code. Other formats aren't dumped.">;
+  HelpText<"Directory to dump offloading images to.">;
path: clang/tools/clang-linker-wrapper/LinkerWrapperOpts.td
user: login: bader
body: I'm not sure if there is any interest in supporting additional formats for "old offloading model". 
The old offloading model is supposed to be replaced with the new offloading model soon. Right?

FYI: non-SPIR-V targets seems to be happy with -fsycl-device-only output. See https://github.com/intel/llvm/pull/17390.

If we have customers for this use case, please, file a feature request to handle it separately.
created_at: 2025-03-24T23:05:16Z
author_association: CONTRIBUTOR
diff_hunk: @@ -2620,11 +2609,22 @@ int main(int Argc, char **Argv) {
     Arg *A = Args.getLastArg(OPT_sycl_dump_device_code_EQ);
     SmallString<128> Dir(A->getValue());
     if (Dir.empty())
-      llvm::sys::path::native(Dir = "./");
+      sys::path::native(Dir = "./");
     else
-      Dir.append(llvm::sys::path::get_separator());
-
-    SPIRVDumpDir = Dir;
+      Dir.append(sys::path::get_separator());
+
+    OffloadImageDumpDir = Dir;
+
+    if (!DryRun) {
path: clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp
user: login: bader
body: Why would we want to restrict `clang-linker-wrapper` this way?
created_at: 2025-03-24T23:07:52Z
author_association: CONTRIBUTOR
diff_hunk: @@ -2620,11 +2609,22 @@ int main(int Argc, char **Argv) {
     Arg *A = Args.getLastArg(OPT_sycl_dump_device_code_EQ);
     SmallString<128> Dir(A->getValue());
     if (Dir.empty())
-      llvm::sys::path::native(Dir = "./");
+      sys::path::native(Dir = "./");
     else
-      Dir.append(llvm::sys::path::get_separator());
-
-    SPIRVDumpDir = Dir;
+      Dir.append(sys::path::get_separator());
+
+    OffloadImageDumpDir = Dir;
+
+    if (!DryRun) {
path: clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp
user: login: maksimsab
body: I think, we could just construct `OffloadImageDumpDir` here without actually creating this directory since `copy_file` creates this directory and the `wrapSYCLBinariesFromFile` handles the `dry-run` mode.
created_at: 2025-03-25T15:43:13Z
author_association: CONTRIBUTOR
diff_hunk: @@ -2620,11 +2609,22 @@ int main(int Argc, char **Argv) {
     Arg *A = Args.getLastArg(OPT_sycl_dump_device_code_EQ);
     SmallString<128> Dir(A->getValue());
     if (Dir.empty())
-      llvm::sys::path::native(Dir = "./");
+      sys::path::native(Dir = "./");
     else
-      Dir.append(llvm::sys::path::get_separator());
-
-    SPIRVDumpDir = Dir;
+      Dir.append(sys::path::get_separator());
+
+    OffloadImageDumpDir = Dir;
+
+    if (!DryRun) {
path: clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp
user: login: bader
body: Done.
created_at: 2025-03-25T16:17:15Z
author_association: CONTRIBUTOR
diff_hunk: @@ -47,11 +47,6 @@
 // RUN: clang-linker-wrapper -sycl-device-libraries=%t.devicelib.o -sycl-post-link-options="SYCL_POST_LINK_OPTIONS" -llvm-spirv-options="LLVM_SPIRV_OPTIONS" "--host-triple=x86_64-unknown-linux-gnu" "--triple=spir64" "--linker-path=/usr/bin/ld" -shared "--" HOST_LINKER_FLAGS "-dynamic-linker" HOST_DYN_LIB "-o" "a.out" HOST_LIB_PATH HOST_STAT_LIB %t.o --dry-run 2>&1 | FileCheck -check-prefix=CHK-SHARED %s
 // CHK-SHARED: "{{.*}}clang"{{.*}} -fPIC
 
-// RUN: rm %T/linker_wrapper_dump || true
-// RUN: clang-linker-wrapper -sycl-dump-device-code=%T/linker_wrapper_dump -sycl-device-libraries=%t.devicelib.o "--host-triple=x86_64-unknown-linux-gnu" "--triple=spir64" "--linker-path=/usr/bin/ld" -shared "--" HOST_LINKER_FLAGS "-dynamic-linker" HOST_DYN_LIB "-o" "a.out" HOST_LIB_PATH HOST_STAT_LIB %t.o --dry-run
path: clang/test/Driver/linker-wrapper-sycl.cpp
user: login: maksimsab
body: I see. The following piece of code should be reworked so that it uses an empty `MemoryBuffer` as you suggested in an internal tracker.
https://github.com/intel/llvm/blob/5c5954eb81ce04a84a5d3757632630758ef9578d/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp#L1074-L1086

Then this test could be returned.
created_at: 2025-03-25T16:23:42Z
author_association: CONTRIBUTOR
diff_hunk: @@ -47,11 +47,6 @@
 // RUN: clang-linker-wrapper -sycl-device-libraries=%t.devicelib.o -sycl-post-link-options="SYCL_POST_LINK_OPTIONS" -llvm-spirv-options="LLVM_SPIRV_OPTIONS" "--host-triple=x86_64-unknown-linux-gnu" "--triple=spir64" "--linker-path=/usr/bin/ld" -shared "--" HOST_LINKER_FLAGS "-dynamic-linker" HOST_DYN_LIB "-o" "a.out" HOST_LIB_PATH HOST_STAT_LIB %t.o --dry-run 2>&1 | FileCheck -check-prefix=CHK-SHARED %s
 // CHK-SHARED: "{{.*}}clang"{{.*}} -fPIC
 
-// RUN: rm %T/linker_wrapper_dump || true
-// RUN: clang-linker-wrapper -sycl-dump-device-code=%T/linker_wrapper_dump -sycl-device-libraries=%t.devicelib.o "--host-triple=x86_64-unknown-linux-gnu" "--triple=spir64" "--linker-path=/usr/bin/ld" -shared "--" HOST_LINKER_FLAGS "-dynamic-linker" HOST_DYN_LIB "-o" "a.out" HOST_LIB_PATH HOST_STAT_LIB %t.o --dry-run
path: clang/test/Driver/linker-wrapper-sycl.cpp
user: login: bader
body: It will also require significant test rework as it checks that files are created in the file system in dry-run mode.
I suggest you rework the dry-run mode for all SYCL methods and all corresponding tests in separate PR.
Does it make sense to you?
created_at: 2025-03-25T16:35:09Z
author_association: CONTRIBUTOR

PR comments:
user: login: bader
created_at: 2025-03-21T17:03:04Z
body: @maksimsab, ping.
author_association: CONTRIBUTOR
user: login: bader
created_at: 2025-03-24T16:52:37Z
body: @maksimsab, ping^2.
author_association: CONTRIBUTOR
user: login: bader
created_at: 2025-03-25T16:18:48Z
body: @maksimsab, please, approve if there are no other change requests.
author_association: CONTRIBUTOR
user: login: maksimsab
created_at: 2025-03-25T16:41:35Z
body: I am trying to resolve the issue with the test locally. Need some time.
author_association: CONTRIBUTOR
user: login: bader
created_at: 2025-03-25T16:46:08Z
body: > I am trying to resolve the issue with the test locally. Need some time.

I think before fixing the test, we need to align the way dry-run mode is handled in the code added for SYCL with the upstream approach. I suggest we remove the test in this PR and fix testing of SYCL mode as a separate change.
Does it work for you?
author_association: CONTRIBUTOR
user: login: maksimsab
created_at: 2025-03-25T16:48:43Z
body: OK.

author_association: CONTRIBUTOR
user: login: bader
created_at: 2025-03-26T00:42:06Z
body: All failures are caused by lack of machines to run the tests.
author_association: CONTRIBUTOR