Repository: microsoft/terminal (language: cpp)

Issue:
Shaders load but don't work
### Windows Terminal version

1.13.10336.0 Preview

### Windows build number

10.0.19042.1586

### Other Software

_No response_

### Steps to reproduce

I have downloaded https://github.com/microsoft/terminal/blob/main/samples/PixelShaders/Rasterbars.hlsl as **test.hlsl** file and placed it in **C:\TMP\** directory. Then I changed my **Windows Terminal** configuration like below:

```json
{
    "$help": "https://aka.ms/terminal-documentation",
    "$schema": "https://aka.ms/terminal-profiles-schema",
    "actions": 
    [
        {
            "command": "toggleShaderEffects",
            "keys": "shift+f10"
        },
        {
            "command": "paste",
            "keys": "ctrl+v"
        },
        {
            "command": 
            {
                "action": "copy",
                "singleLine": false
            },
            "keys": "ctrl+c"
        },
        {
            "command": "toggleFocusMode",
            "keys": "shift+f11"
        },
        {
            "command": "find",
            "keys": "ctrl+shift+f"
        },
        {
            "command": 
            {
                "action": "splitPane",
                "split": "auto",
                "splitMode": "duplicate"
            },
            "keys": "alt+shift+d"
        }
    ],
    "alwaysShowNotificationIcon": true,
    "defaultProfile": "{0caa0dad-35be-5f56-a8ff-afceeeaa6101}",
    "profiles": 
    {
        "defaults": 
        {
            "experimental.pixelShaderPath": "C:\\TMP\\test.hlsl",
            "experimental.useAtlasEngine": false,
            "font": 
            {
                "face": "FiraCode NF"
            }
        },
        "list": 
        [
            {
                "commandline": "%SystemRoot%\\System32\\WindowsPowerShell\\v1.0\\powershell.exe",
                "guid": "{61c54bbd-c2c6-5271-96e7-009a87ff44bf}",
                "hidden": false,
                "name": "Windows PowerShell"
            },
            {
                "commandline": "%SystemRoot%\\System32\\cmd.exe",
                "guid": "{0caa0dad-35be-5f56-a8ff-afceeeaa6101}",
                "hidden": false,
                "name": "Command Prompt"
            },
            {
                "guid": "{b453ae62-4e3d-5e58-b989-0a998ec441b8}",
                "hidden": false,
                "name": "Azure Cloud Shell",
                "source": "Windows.Terminal.Azure"
            },
            {
                "guid": "{2ece5bfe-50ed-5f3a-ab87-5cd4baafed2b}",
                "hidden": false,
                "name": "Git Bash",
                "source": "Git"
            },
            {
                "guid": "{58ad8b0c-3ef8-5f4d-bc6f-13e4c00f2530}",
                "hidden": false,
                "name": "Debian",
                "source": "Windows.Terminal.Wsl"
            }
        ]
    },
    "schemes": 
    [
        {
            "background": "#0C0C0C",
            "black": "#0C0C0C",
            "blue": "#0037DA",
            "brightBlack": "#767676",
            "brightBlue": "#3B78FF",
            "brightCyan": "#61D6D6",
            "brightGreen": "#16C60C",
            "brightPurple": "#B4009E",
            "brightRed": "#E74856",
            "brightWhite": "#F2F2F2",
            "brightYellow": "#F9F1A5",
            "cursorColor": "#FFFFFF",
            "cyan": "#3A96DD",
            "foreground": "#CCCCCC",
            "green": "#13A10E",
            "name": "Campbell",
            "purple": "#881798",
            "red": "#C50F1F",
            "selectionBackground": "#FFFFFF",
            "white": "#CCCCCC",
            "yellow": "#C19C00"
        },
        {
            "background": "#012456",
            "black": "#0C0C0C",
            "blue": "#0037DA",
            "brightBlack": "#767676",
            "brightBlue": "#3B78FF",
            "brightCyan": "#61D6D6",
            "brightGreen": "#16C60C",
            "brightPurple": "#B4009E",
            "brightRed": "#E74856",
            "brightWhite": "#F2F2F2",
            "brightYellow": "#F9F1A5",
            "cursorColor": "#FFFFFF",
            "cyan": "#3A96DD",
            "foreground": "#CCCCCC",
            "green": "#13A10E",
            "name": "Campbell Powershell",
            "purple": "#881798",
            "red": "#C50F1F",
            "selectionBackground": "#FFFFFF",
            "white": "#CCCCCC",
            "yellow": "#C19C00"
        },
        {
            "background": "#282C34",
            "black": "#282C34",
            "blue": "#61AFEF",
            "brightBlack": "#5A6374",
            "brightBlue": "#61AFEF",
            "brightCyan": "#56B6C2",
            "brightGreen": "#98C379",
            "brightPurple": "#C678DD",
            "brightRed": "#E06C75",
            "brightWhite": "#DCDFE4",
            "brightYellow": "#E5C07B",
            "cursorColor": "#FFFFFF",
            "cyan": "#56B6C2",
            "foreground": "#DCDFE4",
            "green": "#98C379",
            "name": "One Half Dark",
            "purple": "#C678DD",
            "red": "#E06C75",
            "selectionBackground": "#FFFFFF",
            "white": "#DCDFE4",
            "yellow": "#E5C07B"
        },
        {
            "background": "#FAFAFA",
            "black": "#383A42",
            "blue": "#0184BC",
            "brightBlack": "#4F525D",
            "brightBlue": "#61AFEF",
            "brightCyan": "#56B5C1",
            "brightGreen": "#98C379",
            "brightPurple": "#C577DD",
            "brightRed": "#DF6C75",
            "brightWhite": "#FFFFFF",
            "brightYellow": "#E4C07A",
            "cursorColor": "#4F525D",
            "cyan": "#0997B3",
            "foreground": "#383A42",
            "green": "#50A14F",
            "name": "One Half Light",
            "purple": "#A626A4",
            "red": "#E45649",
            "selectionBackground": "#FFFFFF",
            "white": "#FAFAFA",
            "yellow": "#C18301"
        },
        {
            "background": "#002B36",
            "black": "#002B36",
            "blue": "#268BD2",
            "brightBlack": "#073642",
            "brightBlue": "#839496",
            "brightCyan": "#93A1A1",
            "brightGreen": "#586E75",
            "brightPurple": "#6C71C4",
            "brightRed": "#CB4B16",
            "brightWhite": "#FDF6E3",
            "brightYellow": "#657B83",
            "cursorColor": "#FFFFFF",
            "cyan": "#2AA198",
            "foreground": "#839496",
            "green": "#859900",
            "name": "Solarized Dark",
            "purple": "#D33682",
            "red": "#DC322F",
            "selectionBackground": "#FFFFFF",
            "white": "#EEE8D5",
            "yellow": "#B58900"
        },
        {
            "background": "#FDF6E3",
            "black": "#002B36",
            "blue": "#268BD2",
            "brightBlack": "#073642",
            "brightBlue": "#839496",
            "brightCyan": "#93A1A1",
            "brightGreen": "#586E75",
            "brightPurple": "#6C71C4",
            "brightRed": "#CB4B16",
            "brightWhite": "#FDF6E3",
            "brightYellow": "#657B83",
            "cursorColor": "#002B36",
            "cyan": "#2AA198",
            "foreground": "#657B83",
            "green": "#859900",
            "name": "Solarized Light",
            "purple": "#D33682",
            "red": "#DC322F",
            "selectionBackground": "#FFFFFF",
            "white": "#EEE8D5",
            "yellow": "#B58900"
        },
        {
            "background": "#000000",
            "black": "#000000",
            "blue": "#3465A4",
            "brightBlack": "#555753",
            "brightBlue": "#729FCF",
            "brightCyan": "#34E2E2",
            "brightGreen": "#8AE234",
            "brightPurple": "#AD7FA8",
            "brightRed": "#EF2929",
            "brightWhite": "#EEEEEC",
            "brightYellow": "#FCE94F",
            "cursorColor": "#FFFFFF",
            "cyan": "#06989A",
            "foreground": "#D3D7CF",
            "green": "#4E9A06",
            "name": "Tango Dark",
            "purple": "#75507B",
            "red": "#CC0000",
            "selectionBackground": "#FFFFFF",
            "white": "#D3D7CF",
            "yellow": "#C4A000"
        },
        {
            "background": "#FFFFFF",
            "black": "#000000",
            "blue": "#3465A4",
            "brightBlack": "#555753",
            "brightBlue": "#729FCF",
            "brightCyan": "#34E2E2",
            "brightGreen": "#8AE234",
            "brightPurple": "#AD7FA8",
            "brightRed": "#EF2929",
            "brightWhite": "#EEEEEC",
            "brightYellow": "#FCE94F",
            "cursorColor": "#000000",
            "cyan": "#06989A",
            "foreground": "#555753",
            "green": "#4E9A06",
            "name": "Tango Light",
            "purple": "#75507B",
            "red": "#CC0000",
            "selectionBackground": "#FFFFFF",
            "white": "#D3D7CF",
            "yellow": "#C4A000"
        },
        {
            "background": "#000000",
            "black": "#000000",
            "blue": "#000080",
            "brightBlack": "#808080",
            "brightBlue": "#0000FF",
            "brightCyan": "#00FFFF",
            "brightGreen": "#00FF00",
            "brightPurple": "#FF00FF",
            "brightRed": "#FF0000",
            "brightWhite": "#FFFFFF",
            "brightYellow": "#FFFF00",
            "cursorColor": "#FFFFFF",
            "cyan": "#008080",
            "foreground": "#C0C0C0",
            "green": "#008000",
            "name": "Vintage",
            "purple": "#800080",
            "red": "#800000",
            "selectionBackground": "#FFFFFF",
            "white": "#C0C0C0",
            "yellow": "#808000"
        }
    ]
}
```

### Expected Behavior

![RasterBars](https://github.com/microsoft/terminal/blob/main/samples/PixelShaders/Screenshots/TerminalRasterbars.PNG?raw=true)

### Actual Behavior

![Screenshot (2)](https://user-images.githubusercontent.com/57063715/157651945-d25f9857-d0dd-4b85-a687-1628affe1439.png)


Discussion / hints:
I have also seen https://github.com/microsoft/terminal/issues/9354 and https://github.com/microsoft/terminal/issues/9058 but those say that bug was fixed in **Windows Terminal Preview v1.8.1032.0** while I have a later version.
Ah, shaders don't work with the atlas engine quite yet. This is tracked as a bullet point in #9999 but I'll leave this open to track it fully. Thanks!
@zadjii-msft I will try without **Atlas engine**.
Edit: No effect. Still nothing

@UltraStudioLTD Did you set your `antialiasingMode` to `cleartype`? If so: That doesn't seem to work at the moment.
Otherwise, could you update or post your settings.json?
@lhecker It didn't work. I updated **setting.json** file posted to match currently used.
@UltraStudioLTD You have `"antialiasingMode": "cleartype",` in your settings. Remove that line and it will work. 🙂
@lhecker It didn't work.
By the way here is **C:\TMP\test.hlsl** file

```hlsl
// A minimal pixel shader that shows some raster bars

// The terminal graphics as a texture
Texture2D shaderTexture;
SamplerState samplerState;

// Terminal settings such as the resolution of the texture
cbuffer PixelShaderSettings {
  // The number of seconds since the pixel shader was enabled
  float  Time;
  // UI Scale
  float  Scale;
  // Resolution of the shaderTexture
  float2 Resolution;
  // Background color as rgba
  float4 Background;
};

// A pixel shader is a program that given a texture coordinate (tex) produces a color.
// tex is an x,y tuple that ranges from 0,0 (top left) to 1,1 (bottom right).
// Just ignore the pos parameter.
float4 main(float4 pos : SV_POSITION, float2 tex : TEXCOORD) : SV_TARGET
{
    // Read the color value at the current texture coordinate (tex)
    //  float4 is tuple of 4 floats, rgba
    float4 color = shaderTexture.Sample(samplerState, tex);

    // Read the color value at some offset, will be used as shadow
    float4 ocolor = shaderTexture.Sample(samplerState, tex+2.0*Scale*float2(-1.0, -1.0)/Resolution.y);

    // Thickness of raster
    const float thickness = 0.1;

    float ny = floor(tex.y/thickness);
    float my = tex.y%thickness;
    const float pi = 3.141592654;


    // ny is used to compute the rasterbar base color
    float cola = ny*2.0*pi;
    float3 col = 0.75+0.25*float3(sin(cola*0.111), sin(cola*0.222), sin(cola*0.333));

    // my is used to compute the rasterbar brightness
    //  smoothstep is a great little function: https://en.wikipedia.org/wiki/Smoothstep
    float brightness = 1.0-smoothstep(0.0, thickness*0.5, abs(my - 0.5*thickness));

    float3 rasterColor = col*brightness;

    // lerp(x, y, a) is another very useful function: https://en.wikipedia.org/wiki/Linear_interpolation
    float3 final = rasterColor;
    // Create the drop shadow of the terminal graphics
    //  .w is the alpha channel, 0 is fully transparent and 1 is fully opaque
    final = lerp(final, float(0.0), ocolor.w);
    // Draw the terminal graphics
    final = lerp(final, color.xyz, color.w);

    // Return the final color, set alpha to 1 (ie opaque)
    return float4(final, 1.0);
}
```
Are you 100% sure? 😅

![image](https://user-images.githubusercontent.com/2256941/157720263-2539b2f3-395f-4d7a-93c5-52f4f852d4ba.png)

You did reload the tab after disabling AtlasEngine / the experimental text renderer right?
I even reopened it

What's your **settings,json**?
Oh and I am using Preview version by the way
> What's your settings,json?

I'm using your settings.json and the same Windows Terminal version (1.13.10336.0).
I solved issue by switching to **Software Rendering** instead of **Hardware Rendering**
> I solved issue by switching to Software Rendering instead of Hardware Rendering

This is interesting actually. Can you tell us more about the hardware you use? What's your CPU and GPU? Do you use Windows Terminal via a Virtual Machine, Remote Desktop, or some other mechanism?
This shouldn't happen and we should try and fix your issue. 🙂

If you have the time and would like to investigate (really just in case), could you enable hardware rendering again and download [DebugView](https://docs.microsoft.com/en-us/sysinternals/downloads/debugview) and see if any logs come up when you start Windows Terminal ("WT")? If you go into your start menu and search for `dxcpl.exe` you can find a tool that can enable DirectX debugging for WT. Here's how to use it:

![image](https://user-images.githubusercontent.com/2256941/157915837-ff7027d5-483a-4b0f-a8fc-a21b65c8187c.png)

If you're done debugging, don't forget to "Clear" the "Executable List". You need to click "Apply" (or "OK") in the primary window in order for the change to take effect.

All of that is just in case you have the time and interest in investigating this of course. 🙂
@lhecker I will upload details later when I will have time. For now I will enjoy CRT shader that I always wanted (since **cool-retro-term**)
Edit: **CRT** shader is super slow due **Software Rendering** 😭. Only option is **Retro**
@lhecker 
OK, here are the details:

- My **CPU** is [**Intel® Core™2 Duo CPU E6750 @ 2.66GHz**](https://www.intel.com/content/www/us/en/products/sku/30784/intel-core2-duo-processor-e6750-4m-cache-2-66-ghz-1333-mhz-fsb/specifications.html)
- My **GPU** is [**NVIDIA GeForce 8600 GT**](https://www.nvidia.com/en-us/drivers/geforce-8600-8500-tech-specs) or [**Alternative Technical Specifications (UnOfficial)**](https://www.techpowerup.com/gpu-specs/geforce-8600-gt.c198)
- I don't use Windows Terminal via any remote or virtual means.

[WTPreviewHardwareRendering.LOG](https://github.com/microsoft/terminal/files/8234892/WTPreviewHardwareRendering.LOG)
And as I mentioned **HLSL** rendering works only with **Software Rendering** with both **Preview** and **Stable** versions
Found it:
```
00000232	1.32877946	[8336] D3D11 ERROR: ID3D11Device::CreateVertexShader: Shader must be vs_4_0. Shader version provided: vs_5_0 [ STATE_CREATION ERROR #167: CREATEVERTEXSHADER_INVALIDSHADERTYPE]	
```

Your GPU only supports feature level 10.0 for DirectX, which only apparently supports HLSL 4.0.

Fix patch:
diff --git a/src/renderer/dx/DxRenderer.cpp b/src/renderer/dx/DxRenderer.cpp
index 18b607fb70c..265160cc983 100644
--- a/src/renderer/dx/DxRenderer.cpp
+++ b/src/renderer/dx/DxRenderer.cpp
@@ -187,11 +187,7 @@ DxEngine::~DxEngine()
 // - entry - Entry function of shader
 // Return Value:
 // - Compiled binary. Errors are thrown and logged.
-inline Microsoft::WRL::ComPtr<ID3DBlob>
-_CompileShader(
-    std::string source,
-    std::string target,
-    std::string entry = "main")
+static Microsoft::WRL::ComPtr<ID3DBlob> _CompileShader(const std::string_view& source, const char* target)
 {
 #if !TIL_FEATURE_DXENGINESHADERSUPPORT_ENABLED
     THROW_HR(E_UNEXPECTED);
@@ -201,24 +197,24 @@ _CompileShader(
     Microsoft::WRL::ComPtr<ID3DBlob> error{};
 
     const HRESULT hr = D3DCompile(
-        source.c_str(),
+        source.data(),
         source.size(),
         nullptr,
         nullptr,
         nullptr,
-        entry.c_str(),
-        target.c_str(),
-        0,
+        "main",
+        target,
+        D3DCOMPILE_PACK_MATRIX_COLUMN_MAJOR | D3DCOMPILE_OPTIMIZATION_LEVEL3,
         0,
         &code,
         &error);
 
     if (FAILED(hr))
     {
-        LOG_HR_MSG(hr, "D3DCompile failed with %x.", static_cast<int>(hr));
+        LOG_HR_MSG(hr, "D3DCompile failed with %08x", hr);
         if (error)
         {
-            LOG_HR_MSG(hr, "D3DCompile error\n%*S", static_cast<int>(error->GetBufferSize()), static_cast<PWCHAR>(error->GetBufferPointer()));
+            LOG_HR_MSG(hr, "D3DCompile error\n%S", static_cast<const char*>(error->GetBufferPointer()));
         }
 
         THROW_HR(hr);
@@ -351,15 +347,33 @@ HRESULT DxEngine::_SetupTerminalEffects()
     vp.TopLeftY = 0;
     _d3dDeviceContext->RSSetViewports(1, &vp);
 
+    const char* shaderTargetVS = nullptr;
+    const char* shaderTargetPS = nullptr;
+    switch (_d3dDevice->GetFeatureLevel())
+    {
+    case D3D_FEATURE_LEVEL_10_0:
+        shaderTargetVS = "vs_4_0";
+        shaderTargetPS = "ps_4_0";
+        break;
+    case D3D_FEATURE_LEVEL_10_1:
+        shaderTargetVS = "vs_4_1";
+        shaderTargetPS = "ps_4_1";
+        break;
+    default:
+        shaderTargetVS = "vs_5_0";
+        shaderTargetPS = "ps_5_0";
+        break;
+    }
+
     // Prepare shaders.
-    auto vertexBlob = _CompileShader(screenVertexShaderString, "vs_5_0");
+    auto vertexBlob = _CompileShader(&screenVertexShaderString[0], shaderTargetVS);
     Microsoft::WRL::ComPtr<ID3DBlob> pixelBlob;
     // As the pixel shader source is user provided it's possible there's a problem with it
     //  so load it inside a try catch, on any error log and fallback on the error pixel shader
     //  If even the error pixel shader fails to load rely on standard exception handling
     try
     {
-        pixelBlob = _CompileShader(pixelShaderSource, "ps_5_0");
+        pixelBlob = _CompileShader(pixelShaderSource, shaderTargetPS);
     }
     catch (...)
     {
@@ -542,11 +556,12 @@ try
                               // D3D11_CREATE_DEVICE_DEBUG |
                               D3D11_CREATE_DEVICE_SINGLETHREADED;
 
-    const std::array<D3D_FEATURE_LEVEL, 5> FeatureLevels{ D3D_FEATURE_LEVEL_11_1,
-                                                          D3D_FEATURE_LEVEL_11_0,
-                                                          D3D_FEATURE_LEVEL_10_1,
-                                                          D3D_FEATURE_LEVEL_10_0,
-                                                          D3D_FEATURE_LEVEL_9_1 };
+    static constexpr std::array FeatureLevels{
+        D3D_FEATURE_LEVEL_11_1,
+        D3D_FEATURE_LEVEL_11_0,
+        D3D_FEATURE_LEVEL_10_1,
+        D3D_FEATURE_LEVEL_10_0,
+    };
 
     // Trying hardware first for maximum performance, then trying WARP (software) renderer second
     // in case we're running inside a downlevel VM where hardware passthrough isn't enabled like


Test patch: