Repository: microsoft/terminal (language: cpp)

Issue:
Atlas engine should support user-customizable pixel shaders & the retro terminal effect



Discussion / hints:

Fix patch:
diff --git a/samples/PixelShaders/Retro.hlsl b/samples/PixelShaders/Retro.hlsl
index cfefe04e7da..0073f2ca87c 100644
--- a/samples/PixelShaders/Retro.hlsl
+++ b/samples/PixelShaders/Retro.hlsl
@@ -2,47 +2,46 @@
 Texture2D shaderTexture;
 SamplerState samplerState;
 
-cbuffer PixelShaderSettings {
-  float  Time;
-  float  Scale;
-  float2 Resolution;
-  float4 Background;
+cbuffer PixelShaderSettings
+{
+    float time;
+    float scale;
+    float2 resolution;
+    float4 background;
 };
 
-#define SCANLINE_FACTOR 0.5
-#define SCALED_SCANLINE_PERIOD Scale
-#define SCALED_GAUSSIAN_SIGMA (2.0*Scale)
+#define SCANLINE_FACTOR 0.5f
+#define SCALED_SCANLINE_PERIOD scale
+#define SCALED_GAUSSIAN_SIGMA (2.0f * scale)
 
 static const float M_PI = 3.14159265f;
 
 float Gaussian2D(float x, float y, float sigma)
 {
-    return 1/(sigma*sqrt(2*M_PI)) * exp(-0.5*(x*x + y*y)/sigma/sigma);
+    return 1 / (sigma * sqrt(2 * M_PI)) * exp(-0.5 * (x * x + y * y) / sigma / sigma);
 }
 
 float4 Blur(Texture2D input, float2 tex_coord, float sigma)
 {
-    uint width, height;
+    float width, height;
     shaderTexture.GetDimensions(width, height);
 
-    float texelWidth = 1.0f/width;
-    float texelHeight = 1.0f/height;
+    float texelWidth = 1.0f / width;
+    float texelHeight = 1.0f / height;
 
     float4 color = { 0, 0, 0, 0 };
 
-    int sampleCount = 13;
+    float sampleCount = 13;
 
-    for (int x = 0; x < sampleCount; x++)
+    for (float x = 0; x < sampleCount; x++)
     {
         float2 samplePos = { 0, 0 };
+        samplePos.x = tex_coord.x + (x - sampleCount / 2.0f) * texelWidth;
 
-        samplePos.x = tex_coord.x + (x - sampleCount/2) * texelWidth;
-        for (int y = 0; y < sampleCount; y++)
+        for (float y = 0; y < sampleCount; y++)
         {
-            samplePos.y = tex_coord.y + (y - sampleCount/2) * texelHeight;
-            if (samplePos.x <= 0 || samplePos.y <= 0 || samplePos.x >= width || samplePos.y >= height) continue;
-
-            color += input.Sample(samplerState, samplePos) * Gaussian2D((x - sampleCount/2), (y - sampleCount/2), sigma);
+            samplePos.y = tex_coord.y + (y - sampleCount / 2.0f) * texelHeight;
+            color += input.Sample(samplerState, samplePos) * Gaussian2D(x - sampleCount / 2.0f, y - sampleCount / 2.0f, sigma);
         }
     }
 
@@ -51,7 +50,7 @@ float4 Blur(Texture2D input, float2 tex_coord, float sigma)
 
 float SquareWave(float y)
 {
-    return 1 - (floor(y / SCALED_SCANLINE_PERIOD) % 2) * SCANLINE_FACTOR;
+    return 1.0f - (floor(y / SCALED_SCANLINE_PERIOD) % 2.0f) * SCANLINE_FACTOR;
 }
 
 float4 Scanline(float4 color, float4 pos)
@@ -60,9 +59,9 @@ float4 Scanline(float4 color, float4 pos)
 
     // TODO:GH#3929 make this configurable.
     // Remove the && false to draw scanlines everywhere.
-    if (length(color.rgb) < 0.2 && false)
+    if (length(color.rgb) < 0.2f && false)
     {
-        return color + wave*0.1;
+        return color + wave * 0.1f;
     }
     else
     {
@@ -70,14 +69,14 @@ float4 Scanline(float4 color, float4 pos)
     }
 }
 
+// clang-format off
 float4 main(float4 pos : SV_POSITION, float2 tex : TEXCOORD) : SV_TARGET
+// clang-format on
 {
-    Texture2D input = shaderTexture;
-
     // TODO:GH#3930 Make these configurable in some way.
-    float4 color = input.Sample(samplerState, tex);
-    color += Blur(input, tex, SCALED_GAUSSIAN_SIGMA)*0.3;
+    float4 color = shaderTexture.Sample(samplerState, tex);
+    color += Blur(shaderTexture, tex, SCALED_GAUSSIAN_SIGMA) * 0.3f;
     color = Scanline(color, pos);
 
     return color;
-}
\ No newline at end of file
+}
diff --git a/src/renderer/atlas/AtlasEngine.api.cpp b/src/renderer/atlas/AtlasEngine.api.cpp
index 4a9bbe390ae..647d57bb7ea 100644
--- a/src/renderer/atlas/AtlasEngine.api.cpp
+++ b/src/renderer/atlas/AtlasEngine.api.cpp
@@ -293,7 +293,7 @@ HRESULT AtlasEngine::Enable() noexcept
 
 [[nodiscard]] bool AtlasEngine::GetRetroTerminalEffect() const noexcept
 {
-    return false;
+    return _api.useRetroTerminalEffect;
 }
 
 [[nodiscard]] float AtlasEngine::GetScaling() const noexcept
@@ -332,7 +332,7 @@ void AtlasEngine::SetAntialiasingMode(const D2D1_TEXT_ANTIALIAS_MODE antialiasin
     if (_api.antialiasingMode != mode)
     {
         _api.antialiasingMode = mode;
-        _resolveAntialiasingMode();
+        _resolveTransparencySettings();
         WI_SetFlag(_api.invalidations, ApiInvalidations::Font);
     }
 }
@@ -344,11 +344,10 @@ void AtlasEngine::SetCallback(std::function<void()> pfn) noexcept
 
 void AtlasEngine::EnableTransparentBackground(const bool isTransparent) noexcept
 {
-    const auto mixin = !isTransparent ? 0xff000000 : 0x00000000;
-    if (_api.backgroundOpaqueMixin != mixin)
+    if (_api.enableTransparentBackground != isTransparent)
     {
-        _api.backgroundOpaqueMixin = mixin;
-        _resolveAntialiasingMode();
+        _api.enableTransparentBackground = isTransparent;
+        _resolveTransparencySettings();
         WI_SetFlag(_api.invalidations, ApiInvalidations::SwapChain);
     }
 }
@@ -369,10 +368,22 @@ void AtlasEngine::SetForceFullRepaintRendering(bool enable) noexcept
 
 void AtlasEngine::SetPixelShaderPath(std::wstring_view value) noexcept
 {
+    if (_api.customPixelShaderPath != value)
+    {
+        _api.customPixelShaderPath = value;
+        _resolveTransparencySettings();
+        WI_SetFlag(_api.invalidations, ApiInvalidations::Device);
+    }
 }
 
 void AtlasEngine::SetRetroTerminalEffect(bool enable) noexcept
 {
+    if (_api.useRetroTerminalEffect != enable)
+    {
+        _api.useRetroTerminalEffect = enable;
+        _resolveTransparencySettings();
+        WI_SetFlag(_api.invalidations, ApiInvalidations::Device);
+    }
 }
 
 void AtlasEngine::SetSelectionBackground(const COLORREF color, const float alpha) noexcept
@@ -451,13 +462,15 @@ void AtlasEngine::UpdateHyperlinkHoveredId(const uint16_t hoveredId) noexcept
 
 #pragma endregion
 
-void AtlasEngine::_resolveAntialiasingMode() noexcept
+void AtlasEngine::_resolveTransparencySettings() noexcept
 {
     // If the user asks for ClearType, but also for a transparent background
     // (which our ClearType shader doesn't simultaneously support)
     // then we need to sneakily force the renderer to grayscale AA.
-    const auto forceGrayscaleAA = _api.antialiasingMode == D2D1_TEXT_ANTIALIAS_MODE_CLEARTYPE && !_api.backgroundOpaqueMixin;
-    _api.realizedAntialiasingMode = forceGrayscaleAA ? D2D1_TEXT_ANTIALIAS_MODE_GRAYSCALE : _api.antialiasingMode;
+    _api.realizedAntialiasingMode = _api.enableTransparentBackground && _api.antialiasingMode == D2D1_TEXT_ANTIALIAS_MODE_CLEARTYPE ? D2D1_TEXT_ANTIALIAS_MODE_GRAYSCALE : _api.antialiasingMode;
+    // An opaque background allows us to use true "independent" flips. See AtlasEngine::_createSwapChain().
+    // We can't enable them if custom shaders are specified, because it's unknown, whether they support opaque inputs.
+    _api.backgroundOpaqueMixin = _api.enableTransparentBackground || !_api.customPixelShaderPath.empty() || _api.useRetroTerminalEffect ? 0x00000000 : 0xff000000;
 }
 
 void AtlasEngine::_updateFont(const wchar_t* faceName, const FontInfoDesired& fontInfoDesired, FontInfo& fontInfo, const std::unordered_map<std::wstring_view, uint32_t>& features, const std::unordered_map<std::wstring_view, float>& axes)
diff --git a/src/renderer/atlas/AtlasEngine.cpp b/src/renderer/atlas/AtlasEngine.cpp
index 92555aec0d6..e89e9d7fbe2 100644
--- a/src/renderer/atlas/AtlasEngine.cpp
+++ b/src/renderer/atlas/AtlasEngine.cpp
@@ -4,6 +4,8 @@
 #include "pch.h"
 #include "AtlasEngine.h"
 
+#include <custom_shader_ps.h>
+#include <custom_shader_vs.h>
 #include <shader_ps.h>
 #include <shader_vs.h>
 
@@ -301,25 +303,6 @@ try
 }
 CATCH_RETURN()
 
-[[nodiscard]] bool AtlasEngine::RequiresContinuousRedraw() noexcept
-{
-    return debugGeneralPerformance;
-}
-
-void AtlasEngine::WaitUntilCanRender() noexcept
-{
-    // IDXGISwapChain2::GetFrameLatencyWaitableObject returns an auto-reset event.
-    // Once we've waited on the event, waiting on it again will block until the timeout elapses.
-    // _r.waitForPresentation guards against this.
-    if (!debugGeneralPerformance && std::exchange(_r.waitForPresentation, false))
-    {
-        WaitForSingleObjectEx(_r.frameLatencyWaitableObject.get(), 100, true);
-#ifndef NDEBUG
-        _r.frameLatencyWaitableObjectUsed = true;
-#endif
-    }
-}
-
 [[nodiscard]] HRESULT AtlasEngine::PrepareForTeardown(_Out_ bool* const pForcePaint) noexcept
 {
     RETURN_HR_IF_NULL(E_INVALIDARG, pForcePaint);
@@ -607,6 +590,7 @@ void AtlasEngine::_createResources()
             D3D_FEATURE_LEVEL_11_1,
             D3D_FEATURE_LEVEL_11_0,
             D3D_FEATURE_LEVEL_10_1,
+            D3D_FEATURE_LEVEL_10_0,
         };
 
         auto hr = S_OK;
@@ -668,6 +652,98 @@ void AtlasEngine::_createResources()
 
         THROW_IF_FAILED(_r.device->CreateVertexShader(&shader_vs[0], sizeof(shader_vs), nullptr, _r.vertexShader.put()));
         THROW_IF_FAILED(_r.device->CreatePixelShader(&shader_ps[0], sizeof(shader_ps), nullptr, _r.pixelShader.put()));
+
+        if (!_api.customPixelShaderPath.empty())
+        {
+            const char* target = nullptr;
+            switch (_r.device->GetFeatureLevel())
+            {
+            case D3D_FEATURE_LEVEL_10_0:
+                target = "ps_4_0";
+                break;
+            case D3D_FEATURE_LEVEL_10_1:
+                target = "ps_4_1";
+                break;
+            default:
+                target = "ps_5_0";
+                break;
+            }
+
+            static constexpr auto flags = D3DCOMPILE_PACK_MATRIX_COLUMN_MAJOR | D3DCOMPILE_ENABLE_STRICTNESS | D3DCOMPILE_WARNINGS_ARE_ERRORS
+#ifdef NDEBUG
+                                          | D3DCOMPILE_OPTIMIZATION_LEVEL3;
+#else
+                                          | D3DCOMPILE_DEBUG | D3DCOMPILE_SKIP_OPTIMIZATION;
+#endif
+
+            wil::com_ptr<ID3DBlob> error;
+            wil::com_ptr<ID3DBlob> blob;
+            const auto hr = D3DCompileFromFile(
+                /* pFileName   */ _api.customPixelShaderPath.c_str(),
+                /* pDefines    */ nullptr,
+                /* pInclude    */ D3D_COMPILE_STANDARD_FILE_INCLUDE,
+                /* pEntrypoint */ "main",
+                /* pTarget     */ target,
+                /* Flags1      */ flags,
+                /* Flags2      */ 0,
+                /* ppCode      */ blob.addressof(),
+                /* ppErrorMsgs */ error.addressof());
+
+            if (SUCCEEDED(hr))
+            {
+                THROW_IF_FAILED(_r.device->CreatePixelShader(blob->GetBufferPointer(), blob->GetBufferSize(), nullptr, _r.customPixelShader.put()));
+            }
+            else
+            {
+                if (error)
+                {
+                    LOG_HR_MSG(hr, "%*hs", error->GetBufferSize(), error->GetBufferPointer());
+                }
+                else
+                {
+                    LOG_HR(hr);
+                }
+
+                if (_api.warningCallback)
+                {
+                    _api.warningCallback(D2DERR_SHADER_COMPILE_FAILED);
+                }
+            }
+
+            _r.requiresContinuousRedraw = true;
+        }
+        else if (_api.useRetroTerminalEffect)
+        {
+            THROW_IF_FAILED(_r.device->CreatePixelShader(&custom_shader_ps[0], sizeof(custom_shader_ps), nullptr, _r.customPixelShader.put()));
+        }
+
+        if (_r.customPixelShader)
+        {
+            THROW_IF_FAILED(_r.device->CreateVertexShader(&custom_shader_vs[0], sizeof(custom_shader_vs), nullptr, _r.customVertexShader.put()));
+
+            {
+                D3D11_BUFFER_DESC desc{};
+                desc.ByteWidth = sizeof(CustomConstBuffer);
+                desc.Usage = D3D11_USAGE_DYNAMIC;
+                desc.BindFlags = D3D11_BIND_CONSTANT_BUFFER;
+                desc.CPUAccessFlags = D3D11_CPU_ACCESS_WRITE;
+                THROW_IF_FAILED(_r.device->CreateBuffer(&desc, nullptr, _r.customShaderConstantBuffer.put()));
+            }
+
+            {
+                D3D11_SAMPLER_DESC desc{};
+                desc.Filter = D3D11_FILTER_MIN_MAG_MIP_LINEAR;
+                desc.AddressU = D3D11_TEXTURE_ADDRESS_BORDER;
+                desc.AddressV = D3D11_TEXTURE_ADDRESS_BORDER;
+                desc.AddressW = D3D11_TEXTURE_ADDRESS_BORDER;
+                desc.MaxAnisotropy = 1;
+                desc.ComparisonFunc = D3D11_COMPARISON_ALWAYS;
+                desc.MaxLOD = D3D11_FLOAT32_MAX;
+                THROW_IF_FAILED(_r.device->CreateSamplerState(&desc, _r.customShaderSamplerState.put()));
+            }
+
+            _r.customShaderStartTime = std::chrono::steady_clock::now();
+        }
     }
 
     WI_ClearFlag(_api.invalidations, ApiInvalidations::Device);
@@ -711,10 +787,9 @@ void AtlasEngine::_createSwapChain()
         desc.BufferCount = 2;
         desc.Scaling = DXGI_SCALING_NONE;
         desc.SwapEffect = _sr.isWindows10OrGreater && !_r.d2dMode ? DXGI_SWAP_EFFECT_FLIP_DISCARD : DXGI_SWAP_EFFECT_FLIP_SEQUENTIAL;
-        // * HWND swap chains can't do alpha.
-        // * If our background is opaque we can enable "independent" flips by setting DXGI_SWAP_EFFECT_FLIP_DISCARD and DXGI_ALPHA_MODE_IGNORE.
-        //   As our swap chain won't have to compose with DWM anymore it reduces the display latency dramatically.
-        desc.AlphaMode = _api.hwnd || _api.backgroundOpaqueMixin ? DXGI_ALPHA_MODE_IGNORE : DXGI_ALPHA_MODE_PREMULTIPLIED;
+        // If our background is opaque we can enable "independent" flips by setting DXGI_SWAP_EFFECT_FLIP_DISCARD and DXGI_ALPHA_MODE_IGNORE.
+        // As our swap chain won't have to compose with DWM anymore it reduces the display latency dramatically.
+        desc.AlphaMode = _api.backgroundOpaqueMixin ? DXGI_ALPHA_MODE_IGNORE : DXGI_ALPHA_MODE_PREMULTIPLIED;
         desc.Flags = debugGeneralPerformance ? 0 : DXGI_SWAP_CHAIN_FLAG_FRAME_LATENCY_WAITABLE_OBJECT;
 
         wil::com_ptr<IDXGIFactory2> dxgiFactory;
@@ -830,6 +905,20 @@ void AtlasEngine::_recreateSizeDependentResources()
             THROW_IF_FAILED(_r.swapChain->GetBuffer(0, __uuidof(ID3D11Texture2D), buffer.put_void()));
             THROW_IF_FAILED(_r.device->CreateRenderTargetView(buffer.get(), nullptr, _r.renderTargetView.put()));
         }
+        if (_r.customPixelShader)
+        {
+            D3D11_TEXTURE2D_DESC desc{};
+            desc.Width = _api.sizeInPixel.x;
+            desc.Height = _api.sizeInPixel.y;
+            desc.MipLevels = 1;
+            desc.ArraySize = 1;
+            desc.Format = DXGI_FORMAT_B8G8R8A8_UNORM;
+            desc.SampleDesc = { 1, 0 };
+            desc.BindFlags = D3D11_BIND_SHADER_RESOURCE | D3D11_BIND_RENDER_TARGET;
+            THROW_IF_FAILED(_r.device->CreateTexture2D(&desc, nullptr, _r.customOffscreenTexture.addressof()));
+            THROW_IF_FAILED(_r.device->CreateShaderResourceView(_r.customOffscreenTexture.get(), nullptr, _r.customOffscreenTextureView.addressof()));
+            THROW_IF_FAILED(_r.device->CreateRenderTargetView(_r.customOffscreenTexture.get(), nullptr, _r.customOffscreenTextureTargetView.addressof()));
+        }
 
         // Tell D3D which parts of the render target will be visible.
         // Everything outside of the viewport will be black.
diff --git a/src/renderer/atlas/AtlasEngine.h b/src/renderer/atlas/AtlasEngine.h
index 347dab8083d..a9632ac89cf 100644
--- a/src/renderer/atlas/AtlasEngine.h
+++ b/src/renderer/atlas/AtlasEngine.h
@@ -113,6 +113,16 @@ namespace Microsoft::Console::Render
             ATLAS_POD_OPS(vec2)
         };
 
+        template<typename T>
+        struct vec3
+        {
+            T x{};
+            T y{};
+            T z{};
+
+            ATLAS_POD_OPS(vec3)
+        };
+
         template<typename T>
         struct vec4
         {
@@ -155,6 +165,7 @@ namespace Microsoft::Console::Render
 
         using f32 = float;
         using f32x2 = vec2<f32>;
+        using f32x3 = vec3<f32>;
         using f32x4 = vec4<f32>;
 
         struct TextAnalyzerResult
@@ -857,6 +868,16 @@ namespace Microsoft::Console::Render
 #pragma warning(suppress : 4324) // 'ConstBuffer': structure was padded due to alignment specifier
         };
 
+        struct alignas(16) CustomConstBuffer
+        {
+            // WARNING: Same rules as for ConstBuffer above apply.
+            alignas(sizeof(f32)) f32 time = 0;
+            alignas(sizeof(f32)) f32 scale = 0;
+            alignas(sizeof(f32x2)) f32x2 resolution;
+            alignas(sizeof(f32x4)) f32x4 background;
+#pragma warning(suppress : 4324) // 'CustomConstBuffer': structure was padded due to alignment specifier
+        };
+
         // Handled in BeginPaint()
         enum class ApiInvalidations : u8
         {
@@ -904,11 +925,12 @@ namespace Microsoft::Console::Render
         bool _emplaceGlyph(IDWriteFontFace* fontFace, size_t bufferPos1, size_t bufferPos2);
 
         // AtlasEngine.api.cpp
-        void _resolveAntialiasingMode() noexcept;
+        void _resolveTransparencySettings() noexcept;
         void _updateFont(const wchar_t* faceName, const FontInfoDesired& fontInfoDesired, FontInfo& fontInfo, const std::unordered_map<std::wstring_view, uint32_t>& features, const std::unordered_map<std::wstring_view, float>& axes);
         void _resolveFontMetrics(const wchar_t* faceName, const FontInfoDesired& fontInfoDesired, FontInfo& fontInfo, FontMetrics* fontMetrics = nullptr) const;
 
         // AtlasEngine.r.cpp
+        void _renderWithCustomShader() const;
         void _setShaderResources() const;
         void _updateConstantBuffer() const noexcept;
         void _adjustAtlasSize();
@@ -974,6 +996,14 @@ namespace Microsoft::Console::Render
             wil::com_ptr<ID3D11Buffer> constantBuffer;
             wil::com_ptr<ID3D11Buffer> cellBuffer;
             wil::com_ptr<ID3D11ShaderResourceView> cellView;
+            wil::com_ptr<ID3D11Texture2D> customOffscreenTexture;
+            wil::com_ptr<ID3D11ShaderResourceView> customOffscreenTextureView;
+            wil::com_ptr<ID3D11RenderTargetView> customOffscreenTextureTargetView;
+            wil::com_ptr<ID3D11VertexShader> customVertexShader;
+            wil::com_ptr<ID3D11PixelShader> customPixelShader;
+            wil::com_ptr<ID3D11Buffer> customShaderConstantBuffer;
+            wil::com_ptr<ID3D11SamplerState> customShaderSamplerState;
+            std::chrono::steady_clock::time_point customShaderStartTime;
 
             // D2D resources
             wil::com_ptr<ID3D11Texture2D> atlasBuffer;
@@ -1013,6 +1043,7 @@ namespace Microsoft::Console::Render
             i16 scrollOffset = 0;
             bool d2dMode = false;
             bool waitForPresentation = false;
+            bool requiresContinuousRedraw = false;
 
 #ifndef NDEBUG
             // See documentation for IDXGISwapChain2::GetFrameLatencyWaitableObject method:
@@ -1045,7 +1076,7 @@ namespace Microsoft::Console::Render
             u16x2 sizeInPixel; // changes are flagged as ApiInvalidations::Size
 
             // UpdateDrawingBrushes()
-            u32 backgroundOpaqueMixin = 0xff000000; // changes are flagged as ApiInvalidations::Device
+            u32 backgroundOpaqueMixin = 0xff000000; // changes are flagged as ApiInvalidations::SwapChain
             u32x2 currentColor;
             AtlasKeyAttributes attributes{};
             u16x2 lastPaintBufferLineCoord;
@@ -1069,7 +1100,11 @@ namespace Microsoft::Console::Render
             HWND hwnd = nullptr;
             u16 dpi = USER_DEFAULT_SCREEN_DPI; // changes are flagged as ApiInvalidations::Font|Size
             u8 antialiasingMode = D2D1_TEXT_ANTIALIAS_MODE_CLEARTYPE; // changes are flagged as ApiInvalidations::Font
-            u8 realizedAntialiasingMode = D2D1_TEXT_ANTIALIAS_MODE_CLEARTYPE; // caches antialiasingMode, depends on antialiasingMode and backgroundOpaqueMixin, see _resolveAntialiasingMode
+            u8 realizedAntialiasingMode = D2D1_TEXT_ANTIALIAS_MODE_CLEARTYPE; // caches antialiasingMode, depends on antialiasingMode and backgroundOpaqueMixin, see _resolveTransparencySettings
+            bool enableTransparentBackground = false;
+
+            std::wstring customPixelShaderPath; // changes are flagged as ApiInvalidations::Device
+            bool useRetroTerminalEffect = false; // changes are flagged as ApiInvalidations::Device
 
             ApiInvalidations invalidations = ApiInvalidations::Device;
         } _api;
diff --git a/src/renderer/atlas/AtlasEngine.r.cpp b/src/renderer/atlas/AtlasEngine.r.cpp
index 5bc00a0225e..bdfaa4a9a51 100644
--- a/src/renderer/atlas/AtlasEngine.r.cpp
+++ b/src/renderer/atlas/AtlasEngine.r.cpp
@@ -44,7 +44,8 @@ constexpr bool isInInversionList(const std::array<wchar_t, N>& ranges, wchar_t n
     return (idx & 1) != 0;
 }
 
-constexpr D2D1_COLOR_F colorFromU32(uint32_t rgba)
+template<typename T = D2D1_COLOR_F>
+constexpr T colorFromU32(uint32_t rgba)
 {
     const auto r = static_cast<float>((rgba >> 0) & 0xff) / 255.0f;
     const auto g = static_cast<float>((rgba >> 8) & 0xff) / 255.0f;
@@ -92,10 +93,15 @@ try
             _r.deviceContext->Unmap(_r.cellBuffer.get(), 0);
         }
 
-        // After Present calls, the back buffer needs to explicitly be
-        // re-bound to the D3D11 immediate context before it can be used again.
-        _r.deviceContext->OMSetRenderTargets(1, _r.renderTargetView.addressof(), nullptr);
-        _r.deviceContext->Draw(3, 0);
+        if (_r.customPixelShader) [[unlikely]]
+        {
+            _renderWithCustomShader();
+        }
+        else
+        {
+            _r.deviceContext->OMSetRenderTargets(1, _r.renderTargetView.addressof(), nullptr);
+            _r.deviceContext->Draw(3, 0);
+        }
 
         // > IDXGISwapChain::Present: Partial Presentation (using a dirty rects or scroll) is not supported
         // > for SwapChains created with DXGI_SWAP_EFFECT_DISCARD or DXGI_SWAP_EFFECT_FLIP_DISCARD.
@@ -118,23 +124,103 @@ catch (const wil::ResultException& exception)
 }
 CATCH_RETURN()
 
+[[nodiscard]] bool AtlasEngine::RequiresContinuousRedraw() noexcept
+{
+    return debugGeneralPerformance || _r.requiresContinuousRedraw;
+}
+
+void AtlasEngine::WaitUntilCanRender() noexcept
+{
+    // IDXGISwapChain2::GetFrameLatencyWaitableObject returns an auto-reset event.
+    // Once we've waited on the event, waiting on it again will block until the timeout elapses.
+    // _r.waitForPresentation guards against this.
+    if (!debugGeneralPerformance && std::exchange(_r.waitForPresentation, false))
+    {
+        WaitForSingleObjectEx(_r.frameLatencyWaitableObject.get(), 100, true);
+#ifndef NDEBUG
+        _r.frameLatencyWaitableObjectUsed = true;
+#endif
+    }
+}
+
 #pragma endregion
 
-void AtlasEngine::_setShaderResources() const
+void AtlasEngine::_renderWithCustomShader() const
 {
-    _r.deviceContext->VSSetShader(_r.vertexShader.get(), nullptr, 0);
-    _r.deviceContext->PSSetShader(_r.pixelShader.get(), nullptr, 0);
+    // Render with our main shader just like Present().
+    {
+        // OM: Output Merger
+        _r.deviceContext->OMSetRenderTargets(1, _r.customOffscreenTextureTargetView.addressof(), nullptr);
+        _r.deviceContext->Draw(3, 0);
+    }
+
+    // Update the custom shader's constant buffer.
+    {
+        CustomConstBuffer data;
+        data.time = std::chrono::duration<float>(std::chrono::steady_clock::now() - _r.customShaderStartTime).count();
+        data.scale = _r.pixelPerDIP;
+        data.resolution.x = static_cast<float>(_r.cellCount.x * _r.fontMetrics.cellSize.x);
+        data.resolution.y = static_cast<float>(_r.cellCount.y * _r.fontMetrics.cellSize.y);
+        data.background = colorFromU32<f32x4>(_r.backgroundColor);
+
+#pragma warning(suppress : 26494) // Variable 'mapped' is uninitialized. Always initialize an object (type.5).
+        D3D11_MAPPED_SUBRESOURCE mapped;
+        THROW_IF_FAILED(_r.deviceContext->Map(_r.customShaderConstantBuffer.get(), 0, D3D11_MAP_WRITE_DISCARD, 0, &mapped));
+        assert(mapped.RowPitch >= sizeof(data));
+        memcpy(mapped.pData, &data, sizeof(data));
+        _r.deviceContext->Unmap(_r.customShaderConstantBuffer.get(), 0);
+    }
+
+    // Render with the custom shader.
+    {
+        // OM: Output Merger
+        // customOffscreenTextureView was just rendered to via customOffscreenTextureTargetView and is
+        // set as the output target. Before we can use it as an input we have to remove it as an output.
+        _r.deviceContext->OMSetRenderTargets(1, _r.renderTargetView.addressof(), nullptr);
+
+        // VS: Vertex Shader
+        _r.deviceContext->VSSetShader(_r.customVertexShader.get(), nullptr, 0);
+
+        // PS: Pixel Shader
+        _r.deviceContext->PSSetShader(_r.customPixelShader.get(), nullptr, 0);
+        _r.deviceContext->PSSetConstantBuffers(0, 1, _r.customShaderConstantBuffer.addressof());
+        _r.deviceContext->PSSetShaderResources(0, 1, _r.customOffscreenTextureView.addressof());
+        _r.deviceContext->PSSetSamplers(0, 1, _r.customShaderSamplerState.addressof());
+
+        _r.deviceContext->Draw(4, 0);
+    }
 
+    // For the next frame we need to restore our context state.
+    {
+        // VS: Vertex Shader
+        _r.deviceContext->VSSetShader(_r.vertexShader.get(), nullptr, 0);
+
+        // PS: Pixel Shader
+        _r.deviceContext->PSSetShader(_r.pixelShader.get(), nullptr, 0);
+        _r.deviceContext->PSSetConstantBuffers(0, 1, _r.constantBuffer.addressof());
+        const std::array resources{ _r.cellView.get(), _r.atlasView.get() };
+        _r.deviceContext->PSSetShaderResources(0, gsl::narrow_cast<UINT>(resources.size()), resources.data());
+        _r.deviceContext->PSSetSamplers(0, 0, nullptr);
+    }
+}
+
+void AtlasEngine::_setShaderResources() const
+{
+    // IA: Input Assembler
     // Our vertex shader uses a trick from Bill Bilodeau published in
     // "Vertex Shader Tricks" at GDC14 to draw a fullscreen triangle
     // without vertex/index buffers. This prepares our context for this.
     _r.deviceContext->IASetVertexBuffers(0, 0, nullptr, nullptr, nullptr);
     _r.deviceContext->IASetIndexBuffer(nullptr, DXGI_FORMAT_UNKNOWN, 0);
     _r.deviceContext->IASetInputLayout(nullptr);
-    _r.deviceContext->IASetPrimitiveTopology(D3D11_PRIMITIVE_TOPOLOGY_TRIANGLELIST);
+    _r.deviceContext->IASetPrimitiveTopology(D3D11_PRIMITIVE_TOPOLOGY_TRIANGLESTRIP);
 
-    _r.deviceContext->PSSetConstantBuffers(0, 1, _r.constantBuffer.addressof());
+    // VS: Vertex Shader
+    _r.deviceContext->VSSetShader(_r.vertexShader.get(), nullptr, 0);
 
+    // PS: Pixel Shader
+    _r.deviceContext->PSSetShader(_r.pixelShader.get(), nullptr, 0);
+    _r.deviceContext->PSSetConstantBuffers(0, 1, _r.constantBuffer.addressof());
     const std::array resources{ _r.cellView.get(), _r.atlasView.get() };
     _r.deviceContext->PSSetShaderResources(0, gsl::narrow_cast<UINT>(resources.size()), resources.data());
 }
diff --git a/src/renderer/atlas/atlas.vcxproj b/src/renderer/atlas/atlas.vcxproj
index 355a9175240..e9907d15490 100644
--- a/src/renderer/atlas/atlas.vcxproj
+++ b/src/renderer/atlas/atlas.vcxproj
@@ -27,12 +27,34 @@
     <ClInclude Include="AtlasEngine.h" />
   </ItemGroup>
   <ItemGroup>
+    <FxCompile Include="custom_shader_ps.hlsl">
+      <ShaderType>Pixel</ShaderType>
+      <ShaderModel>4.0</ShaderModel>
+      <AllResourcesBound>true</AllResourcesBound>
+      <VariableName>custom_shader_ps</VariableName>
+      <ObjectFileOutput />
+      <HeaderFileOutput>$(OutDir)$(ProjectName)\%(Filename).h</HeaderFileOutput>
+      <TreatWarningAsError>true</TreatWarningAsError>
+      <AdditionalOptions>/Zpc %(AdditionalOptions)</AdditionalOptions>
+      <AdditionalOptions Condition="'$(Configuration)'=='Release'">/O3 /Qstrip_debug /Qstrip_reflect %(AdditionalOptions)</AdditionalOptions>
+    </FxCompile>
+    <FxCompile Include="custom_shader_vs.hlsl">
+      <ShaderType>Vertex</ShaderType>
+      <ShaderModel>4.0</ShaderModel>
+      <AllResourcesBound>true</AllResourcesBound>
+      <VariableName>custom_shader_vs</VariableName>
+      <ObjectFileOutput />
+      <HeaderFileOutput>$(OutDir)$(ProjectName)\%(Filename).h</HeaderFileOutput>
+      <TreatWarningAsError>true</TreatWarningAsError>
+      <AdditionalOptions>/Zpc %(AdditionalOptions)</AdditionalOptions>
+      <AdditionalOptions Condition="'$(Configuration)'=='Release'">/O3 /Qstrip_debug /Qstrip_reflect %(AdditionalOptions)</AdditionalOptions>
+    </FxCompile>
     <FxCompile Include="dwrite.hlsl">
       <ExcludedFromBuild>true</ExcludedFromBuild>
     </FxCompile>
     <FxCompile Include="shader_ps.hlsl">
       <ShaderType>Pixel</ShaderType>
-      <ShaderModel>4.1</ShaderModel>
+      <ShaderModel>4.0</ShaderModel>
       <AllResourcesBound>true</AllResourcesBound>
       <VariableName>shader_ps</VariableName>
       <ObjectFileOutput />
@@ -43,7 +65,7 @@
     </FxCompile>
     <FxCompile Include="shader_vs.hlsl">
       <ShaderType>Vertex</ShaderType>
-      <ShaderModel>4.1</ShaderModel>
+      <ShaderModel>4.0</ShaderModel>
       <AllResourcesBound>true</AllResourcesBound>
       <VariableName>shader_vs</VariableName>
       <ObjectFileOutput />
diff --git a/src/renderer/atlas/custom_shader_ps.hlsl b/src/renderer/atlas/custom_shader_ps.hlsl
new file mode 100644
index 00000000000..0073f2ca87c
--- /dev/null
+++ b/src/renderer/atlas/custom_shader_ps.hlsl
@@ -0,0 +1,82 @@
+// The original retro pixel shader
+Texture2D shaderTexture;
+SamplerState samplerState;
+
+cbuffer PixelShaderSettings
+{
+    float time;
+    float scale;
+    float2 resolution;
+    float4 background;
+};
+
+#define SCANLINE_FACTOR 0.5f
+#define SCALED_SCANLINE_PERIOD scale
+#define SCALED_GAUSSIAN_SIGMA (2.0f * scale)
+
+static const float M_PI = 3.14159265f;
+
+float Gaussian2D(float x, float y, float sigma)
+{
+    return 1 / (sigma * sqrt(2 * M_PI)) * exp(-0.5 * (x * x + y * y) / sigma / sigma);
+}
+
+float4 Blur(Texture2D input, float2 tex_coord, float sigma)
+{
+    float width, height;
+    shaderTexture.GetDimensions(width, height);
+
+    float texelWidth = 1.0f / width;
+    float texelHeight = 1.0f / height;
+
+    float4 color = { 0, 0, 0, 0 };
+
+    float sampleCount = 13;
+
+    for (float x = 0; x < sampleCount; x++)
+    {
+        float2 samplePos = { 0, 0 };
+        samplePos.x = tex_coord.x + (x - sampleCount / 2.0f) * texelWidth;
+
+        for (float y = 0; y < sampleCount; y++)
+        {
+            samplePos.y = tex_coord.y + (y - sampleCount / 2.0f) * texelHeight;
+            color += input.Sample(samplerState, samplePos) * Gaussian2D(x - sampleCount / 2.0f, y - sampleCount / 2.0f, sigma);
+        }
+    }
+
+    return color;
+}
+
+float SquareWave(float y)
+{
+    return 1.0f - (floor(y / SCALED_SCANLINE_PERIOD) % 2.0f) * SCANLINE_FACTOR;
+}
+
+float4 Scanline(float4 color, float4 pos)
+{
+    float wave = SquareWave(pos.y);
+
+    // TODO:GH#3929 make this configurable.
+    // Remove the && false to draw scanlines everywhere.
+    if (length(color.rgb) < 0.2f && false)
+    {
+        return color + wave * 0.1f;
+    }
+    else
+    {
+        return color * wave;
+    }
+}
+
+// clang-format off
+float4 main(float4 pos : SV_POSITION, float2 tex : TEXCOORD) : SV_TARGET
+// clang-format on
+{
+    // TODO:GH#3930 Make these configurable in some way.
+    float4 color = shaderTexture.Sample(samplerState, tex);
+    color += Blur(shaderTexture, tex, SCALED_GAUSSIAN_SIGMA) * 0.3f;
+    color = Scanline(color, pos);
+
+    return color;
+}
diff --git a/src/renderer/atlas/custom_shader_vs.hlsl b/src/renderer/atlas/custom_shader_vs.hlsl
new file mode 100644
index 00000000000..5bb9fbff70b
--- /dev/null
+++ b/src/renderer/atlas/custom_shader_vs.hlsl
@@ -0,0 +1,17 @@
+struct VS_OUTPUT
+{
+    float4 pos : SV_POSITION;
+    float2 tex : TEXCOORD;
+};
+
+// clang-format off
+VS_OUTPUT main(uint id : SV_VERTEXID)
+// clang-format on
+{
+    VS_OUTPUT output;
+    // The following two lines are taken from https://gamedev.stackexchange.com/a/77670
+    // written by János Turánszki, licensed under CC BY-SA 3.0.
+    output.tex = float2(id % 2, id % 4 / 2);
+    output.pos = float4((output.tex.x - 0.5f) * 2.0f, -(output.tex.y - 0.5f) * 2.0f, 0, 1);
+    return output;
+}
diff --git a/src/renderer/dx/DxRenderer.cpp b/src/renderer/dx/DxRenderer.cpp
index b0a349ff26e..9621dbb8bb7 100644
--- a/src/renderer/dx/DxRenderer.cpp
+++ b/src/renderer/dx/DxRenderer.cpp
@@ -446,9 +446,9 @@ HRESULT DxEngine::_SetupTerminalEffects()
     // Sampler state is needed to use texture as input to shader.
     D3D11_SAMPLER_DESC samplerDesc{};
     samplerDesc.Filter = D3D11_FILTER_MIN_MAG_MIP_LINEAR;
-    samplerDesc.AddressU = D3D11_TEXTURE_ADDRESS_CLAMP;
-    samplerDesc.AddressV = D3D11_TEXTURE_ADDRESS_CLAMP;
-    samplerDesc.AddressW = D3D11_TEXTURE_ADDRESS_CLAMP;
+    samplerDesc.AddressU = D3D11_TEXTURE_ADDRESS_BORDER;
+    samplerDesc.AddressV = D3D11_TEXTURE_ADDRESS_BORDER;
+    samplerDesc.AddressW = D3D11_TEXTURE_ADDRESS_BORDER;
     samplerDesc.MipLODBias = 0.0f;
     samplerDesc.MaxAnisotropy = 1;
     samplerDesc.ComparisonFunc = D3D11_COMPARISON_ALWAYS;


Test patch: