Repository: microsoft/terminal (language: cpp)

Issue:
Better Warnings for Custom Pixel Shader Compilation Failures
# Description of the new feature/enhancement
Right now, when D3DCompileFromFile fails, a warning is displayed that says "Unable to compile the specified pixel shader." To make it easier to debug custom shader files, it would be helpful to see the exact error message from the compiler.
Current Warning:
![original](https://github.com/microsoft/terminal/assets/104504393/bb3c65dc-d241-4918-84a6-5169eed2ad81)

# Proposed technical implementation details
**Error Type** | **Expected Outcome**
Compile Time Error | Warning window with the exact error message from compiler
File not Found | Warning window with file not found + location of file as seen in settings.json
Path not Found | Similar behaviour

I will be creating a pull request shortly.


Discussion / hints:

Fix patch:
diff --git a/src/cascadia/TerminalControl/Resources/en-US/Resources.resw b/src/cascadia/TerminalControl/Resources/en-US/Resources.resw
index 54c6a4643b5..d618171e58b 100644
--- a/src/cascadia/TerminalControl/Resources/en-US/Resources.resw
+++ b/src/cascadia/TerminalControl/Resources/en-US/Resources.resw
@@ -207,7 +207,8 @@ Please either install the missing font or choose another one.</value>
     <comment>{0} is a file name</comment>
   </data>
   <data name="PixelShaderCompileFailed" xml:space="preserve">
-    <value>Unable to compile the specified pixel shader.</value>
+    <value>Pixel shader failed to compile: {0}</value>
+    <comment>{0} is the error message generated by the compiler</comment>
   </data>
   <data name="UnexpectedRendererError" xml:space="preserve">
     <value>Renderer encountered an unexpected error: {0}</value>
@@ -223,7 +224,7 @@ Please either install the missing font or choose another one.</value>
   </data>
   <data name="RendererErrorOther" xml:space="preserve">
     <value>Renderer encountered an unexpected error: {0:#010x} {1}</value>
-    <comment>{Locked="{0:#010x}","{1}"} {0:#010x} is a placeholder for a Windows error code (e.g. 0x88985002). {1} is the corresponding message.</comment>
+    <comment>{Locked="{0:#010x}","{1}"} {0:#010x} is a placeholder for a Windows error code (e.g. 0x88985002). {1} is the corresponding message. {2} is the filename.</comment>
   </data>
   <data name="TermControlReadOnly" xml:space="preserve">
     <value>Read-only mode is enabled.</value>
@@ -320,4 +321,4 @@ Please either install the missing font or choose another one.</value>
     <value>Suggested input: {0}</value>
     <comment>{Locked="{0}"} {0} will be replaced with a string of input that is suggested for the user to input</comment>
   </data>
-</root>
+</root>
\ No newline at end of file
diff --git a/src/cascadia/TerminalControl/TermControl.cpp b/src/cascadia/TerminalControl/TermControl.cpp
index 1fffe66dc50..bc045f36381 100644
--- a/src/cascadia/TerminalControl/TermControl.cpp
+++ b/src/cascadia/TerminalControl/TermControl.cpp
@@ -1162,7 +1162,7 @@ namespace winrt::Microsoft::Terminal::Control::implementation
             message = winrt::hstring{ fmt::format(std::wstring_view{ RS_(L"PixelShaderNotFound") }, parameter) };
             break;
         case D2DERR_SHADER_COMPILE_FAILED:
-            message = winrt::hstring{ fmt::format(std::wstring_view{ RS_(L"PixelShaderCompileFailed") }) };
+            message = winrt::hstring{ fmt::format(std::wstring_view{ RS_(L"PixelShaderCompileFailed") }, parameter) };
             break;
         case DWRITE_E_NOFONT:
             message = winrt::hstring{ fmt::format(std::wstring_view{ RS_(L"RendererErrorFontNotFound") }, parameter) };
@@ -1175,7 +1175,14 @@ namespace winrt::Microsoft::Terminal::Control::implementation
             wchar_t buf[512];
             const auto len = FormatMessageW(FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, nullptr, hr, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), &buf[0], ARRAYSIZE(buf), nullptr);
             const std::wstring_view msg{ &buf[0], len };
-            message = winrt::hstring{ fmt::format(std::wstring_view{ RS_(L"RendererErrorOther") }, hr, msg) };
+            std::wstring resourceString = RS_(L"RendererErrorOther").c_str();
+            //conditional message construction
+            std::wstring partialMessage = fmt::format(std::wstring_view{ resourceString }, hr, msg);
+            if (!parameter.empty())
+            {
+                fmt::format_to(std::back_inserter(partialMessage), LR"( "{0}")", parameter);
+            }
+            message = winrt::hstring{ partialMessage };
             break;
         }
         }
diff --git a/src/renderer/atlas/BackendD3D.cpp b/src/renderer/atlas/BackendD3D.cpp
index b266d3ba092..231ede7695e 100644
--- a/src/renderer/atlas/BackendD3D.cpp
+++ b/src/renderer/atlas/BackendD3D.cpp
@@ -15,6 +15,7 @@
 #include "dwrite.h"
 #include "wic.h"
 #include "../../types/inc/ColorFix.hpp"
+#include "../../types/inc/convert.hpp"
 
 #if ATLAS_DEBUG_SHOW_DIRTY || ATLAS_DEBUG_COLORIZE_GLYPH_ATLAS
 #include "colorbrewer.h"
@@ -451,15 +452,21 @@ void BackendD3D::_recreateCustomShader(const RenderingPayload& p)
         {
             if (error)
             {
-                LOG_HR_MSG(hr, "%.*hs", static_cast<int>(error->GetBufferSize()), static_cast<char*>(error->GetBufferPointer()));
+                if (p.warningCallback)
+                {
+                    //to handle compile time errors
+                    const std::string_view errMsgStrView{ static_cast<const char*>(error->GetBufferPointer()), error->GetBufferSize() };
+                    const auto errMsgWstring = ConvertToW(CP_ACP, errMsgStrView);
+                    p.warningCallback(D2DERR_SHADER_COMPILE_FAILED, errMsgWstring);
+                }
             }
             else
             {
-                LOG_HR(hr);
-            }
-            if (p.warningCallback)
-            {
-                p.warningCallback(D2DERR_SHADER_COMPILE_FAILED, p.s->misc->customPixelShaderPath);
+                if (p.warningCallback)
+                {
+                    //to handle errors such as file not found, path not found, access denied
+                    p.warningCallback(hr, p.s->misc->customPixelShaderPath);
+                }
             }
         }
 


Test patch: