Implement support for masked stencil clears on Vulkan (#5589)

* Implement support for masked stencil clears on Vulkan

* PR feedback

src/Ryujinx.Graphics.Vulkan/FramebufferParams.cs

@@ -148,6 +148,16 @@ namespace Ryujinx.Graphics.Vulkan
             return _attachments[index];
         }
 
+        public Auto<DisposableImageView> GetDepthStencilAttachment()
+        {
+            if (!HasDepthStencil)
+            {
+                return null;
+            }
+
+            return _attachments[AttachmentsCount - 1];
+        }
+
         public ComponentType GetAttachmentComponentType(int index)
         {
             if (_colors != null && (uint)index < _colors.Length)

src/Ryujinx.Graphics.Vulkan/HelperShader.cs

@@ -38,6 +38,7 @@ namespace Ryujinx.Graphics.Vulkan
         private readonly IProgram _programColorClearF;
         private readonly IProgram _programColorClearSI;
         private readonly IProgram _programColorClearUI;
+        private readonly IProgram _programDepthStencilClear;
         private readonly IProgram _programStrideChange;
         private readonly IProgram _programConvertD32S8ToD24S8;
         private readonly IProgram _programConvertIndexBuffer;
@@ -105,6 +106,12 @@ namespace Ryujinx.Graphics.Vulkan
                 new ShaderSource(ReadSpirv("ColorClearUIFragment.spv"), ShaderStage.Fragment, TargetLanguage.Spirv),
             }, colorClearResourceLayout);
 
+            _programDepthStencilClear = gd.CreateProgramWithMinimalLayout(new[]
+            {
+                new ShaderSource(ReadSpirv("ColorClearVertex.spv"), ShaderStage.Vertex, TargetLanguage.Spirv),
+                new ShaderSource(ReadSpirv("DepthStencilClearFragment.spv"), ShaderStage.Fragment, TargetLanguage.Spirv),
+            }, colorClearResourceLayout);
+
             var strideChangeResourceLayout = new ResourceLayoutBuilder()
                 .Add(ResourceStages.Compute, ResourceType.UniformBuffer, 0)
                 .Add(ResourceStages.Compute, ResourceType.StorageBuffer, 1)
@@ -446,10 +453,6 @@ namespace Ryujinx.Graphics.Vulkan
                 0f,
                 1f);
 
-            Span<Rectangle<int>> scissors = stackalloc Rectangle<int>[1];
-
-            scissors[0] = new Rectangle<int>(0, 0, dstWidth, dstHeight);
-
             if (dstIsDepthOrStencil)
             {
                 _pipeline.SetProgram(src.Info.Target.IsMultisample() ? _programDepthBlitMs : _programDepthBlit);
@@ -470,7 +473,7 @@ namespace Ryujinx.Graphics.Vulkan
 
             _pipeline.SetRenderTarget(dst, (uint)dstWidth, (uint)dstHeight, (uint)dstSamples, dstIsDepthOrStencil, dstFormat);
             _pipeline.SetRenderTargetColorMasks(new uint[] { 0xf });
-            _pipeline.SetScissors(scissors);
+            _pipeline.SetScissors(stackalloc Rectangle<int>[] { new Rectangle<int>(0, 0, dstWidth, dstHeight) });
 
             if (clearAlpha)
             {
@@ -547,12 +550,8 @@ namespace Ryujinx.Graphics.Vulkan
                 0f,
                 1f);
 
-            Span<Rectangle<int>> scissors = stackalloc Rectangle<int>[1];
-
-            scissors[0] = new Rectangle<int>(0, 0, dstWidth, dstHeight);
-
             _pipeline.SetRenderTarget(dst, (uint)dstWidth, (uint)dstHeight, (uint)dstSamples, true, dstFormat);
-            _pipeline.SetScissors(scissors);
+            _pipeline.SetScissors(stackalloc Rectangle<int>[] { new Rectangle<int>(0, 0, dstWidth, dstHeight) });
             _pipeline.SetViewports(viewports);
             _pipeline.SetPrimitiveTopology(PrimitiveTopology.TriangleStrip);
 
@@ -639,7 +638,11 @@ namespace Ryujinx.Graphics.Vulkan
             }
         }
 
-        private static StencilTestDescriptor CreateStencilTestDescriptor(bool enabled)
+        private static StencilTestDescriptor CreateStencilTestDescriptor(
+            bool enabled,
+            int refValue = 0,
+            int compareMask = 0xff,
+            int writeMask = 0xff)
         {
             return new StencilTestDescriptor(
                 enabled,
@@ -647,16 +650,16 @@ namespace Ryujinx.Graphics.Vulkan
                 StencilOp.Replace,
                 StencilOp.Replace,
                 StencilOp.Replace,
-                0,
-                0xff,
-                0xff,
+                refValue,
+                compareMask,
+                writeMask,
                 CompareOp.Always,
                 StencilOp.Replace,
                 StencilOp.Replace,
                 StencilOp.Replace,
-                0,
-                0xff,
-                0xff);
+                refValue,
+                compareMask,
+                writeMask);
         }
 
         public void Clear(
@@ -695,10 +698,6 @@ namespace Ryujinx.Graphics.Vulkan
                 0f,
                 1f);
 
-            Span<Rectangle<int>> scissors = stackalloc Rectangle<int>[1];
-
-            scissors[0] = scissor;
-
             IProgram program;
 
             if (type == ComponentType.SignedInteger)
@@ -718,7 +717,7 @@ namespace Ryujinx.Graphics.Vulkan
             _pipeline.SetRenderTarget(dst, (uint)dstWidth, (uint)dstHeight, false, dstFormat);
             _pipeline.SetRenderTargetColorMasks(new[] { componentMask });
             _pipeline.SetViewports(viewports);
-            _pipeline.SetScissors(scissors);
+            _pipeline.SetScissors(stackalloc Rectangle<int>[] { scissor });
             _pipeline.SetPrimitiveTopology(PrimitiveTopology.TriangleStrip);
             _pipeline.Draw(4, 1, 0, 0);
             _pipeline.Finish();
@@ -726,6 +725,56 @@ namespace Ryujinx.Graphics.Vulkan
             gd.BufferManager.Delete(bufferHandle);
         }
 
+        public void Clear(
+            VulkanRenderer gd,
+            Auto<DisposableImageView> dst,
+            float depthValue,
+            bool depthMask,
+            int stencilValue,
+            int stencilMask,
+            int dstWidth,
+            int dstHeight,
+            VkFormat dstFormat,
+            Rectangle<int> scissor)
+        {
+            const int ClearColorBufferSize = 16;
+
+            gd.FlushAllCommands();
+
+            using var cbs = gd.CommandBufferPool.Rent();
+
+            _pipeline.SetCommandBuffer(cbs);
+
+            var bufferHandle = gd.BufferManager.CreateWithHandle(gd, ClearColorBufferSize);
+
+            gd.BufferManager.SetData<float>(bufferHandle, 0, stackalloc float[] { depthValue });
+
+            _pipeline.SetUniformBuffers(stackalloc[] { new BufferAssignment(1, new BufferRange(bufferHandle, 0, ClearColorBufferSize)) });
+
+            Span<Viewport> viewports = stackalloc Viewport[1];
+
+            viewports[0] = new Viewport(
+                new Rectangle<float>(0, 0, dstWidth, dstHeight),
+                ViewportSwizzle.PositiveX,
+                ViewportSwizzle.PositiveY,
+                ViewportSwizzle.PositiveZ,
+                ViewportSwizzle.PositiveW,
+                0f,
+                1f);
+
+            _pipeline.SetProgram(_programDepthStencilClear);
+            _pipeline.SetRenderTarget(dst, (uint)dstWidth, (uint)dstHeight, true, dstFormat);
+            _pipeline.SetViewports(viewports);
+            _pipeline.SetScissors(stackalloc Rectangle<int>[] { scissor });
+            _pipeline.SetPrimitiveTopology(PrimitiveTopology.TriangleStrip);
+            _pipeline.SetDepthTest(new DepthTestDescriptor(true, depthMask, CompareOp.Always));
+            _pipeline.SetStencilTest(CreateStencilTestDescriptor(stencilMask != 0, stencilValue, 0xff, stencilMask));
+            _pipeline.Draw(4, 1, 0, 0);
+            _pipeline.Finish();
+
+            gd.BufferManager.Delete(bufferHandle);
+        }
+
         public void DrawTexture(
             VulkanRenderer gd,
             PipelineBase pipeline,
@@ -778,8 +827,6 @@ namespace Ryujinx.Graphics.Vulkan
                 0f,
                 1f);
 
-            Span<Rectangle<int>> scissors = stackalloc Rectangle<int>[1];
-
             pipeline.SetProgram(_programColorBlit);
             pipeline.SetViewports(viewports);
             pipeline.SetPrimitiveTopology(PrimitiveTopology.TriangleStrip);
@@ -1119,11 +1166,7 @@ namespace Ryujinx.Graphics.Vulkan
                     0f,
                     1f);
 
-                Span<Rectangle<int>> scissors = stackalloc Rectangle<int>[1];
-
-                scissors[0] = new Rectangle<int>(0, 0, dst.Width, dst.Height);
-
-                _pipeline.SetScissors(scissors);
+                _pipeline.SetScissors(stackalloc Rectangle<int>[] { new Rectangle<int>(0, 0, dst.Width, dst.Height) });
                 _pipeline.SetViewports(viewports);
                 _pipeline.SetPrimitiveTopology(PrimitiveTopology.TriangleStrip);
 
@@ -1251,12 +1294,8 @@ namespace Ryujinx.Graphics.Vulkan
                 0f,
                 1f);
 
-            Span<Rectangle<int>> scissors = stackalloc Rectangle<int>[1];
-
-            scissors[0] = new Rectangle<int>(0, 0, dst.Width, dst.Height);
-
             _pipeline.SetRenderTargetColorMasks(new uint[] { 0xf });
-            _pipeline.SetScissors(scissors);
+            _pipeline.SetScissors(stackalloc Rectangle<int>[] { new Rectangle<int>(0, 0, dst.Width, dst.Height) });
             _pipeline.SetViewports(viewports);
             _pipeline.SetPrimitiveTopology(PrimitiveTopology.TriangleStrip);
 
@@ -1731,6 +1770,7 @@ namespace Ryujinx.Graphics.Vulkan
                 _programColorClearF.Dispose();
                 _programColorClearSI.Dispose();
                 _programColorClearUI.Dispose();
+                _programDepthStencilClear.Dispose();
                 _programStrideChange.Dispose();
                 _programConvertIndexBuffer.Dispose();
                 _programConvertIndirectData.Dispose();

src/Ryujinx.Graphics.Vulkan/PipelineBase.cs

@@ -243,10 +243,8 @@ namespace Ryujinx.Graphics.Vulkan
             Gd.Api.CmdClearAttachments(CommandBuffer, 1, &attachment, 1, &clearRect);
         }
 
-        public unsafe void ClearRenderTargetDepthStencil(int layer, int layerCount, float depthValue, bool depthMask, int stencilValue, int stencilMask)
+        public unsafe void ClearRenderTargetDepthStencil(int layer, int layerCount, float depthValue, bool depthMask, int stencilValue, bool stencilMask)
         {
-            // TODO: Use stencilMask (fully).
-
             if (FramebufferParams == null || !FramebufferParams.HasDepthStencil)
             {
                 return;
@@ -255,7 +253,7 @@ namespace Ryujinx.Graphics.Vulkan
             var clearValue = new ClearValue(null, new ClearDepthStencilValue(depthValue, (uint)stencilValue));
             var flags = depthMask ? ImageAspectFlags.DepthBit : 0;
 
-            if (stencilMask != 0)
+            if (stencilMask)
             {
                 flags |= ImageAspectFlags.StencilBit;
             }

src/Ryujinx.Graphics.Vulkan/PipelineFull.cs

@@ -81,6 +81,42 @@ namespace Ryujinx.Graphics.Vulkan
             }
         }
 
+        public void ClearRenderTargetDepthStencil(int layer, int layerCount, float depthValue, bool depthMask, int stencilValue, int stencilMask)
+        {
+            if (FramebufferParams == null)
+            {
+                return;
+            }
+
+            if (stencilMask != 0 && stencilMask != 0xff)
+            {
+                // We can't use CmdClearAttachments if not clearing all (mask is all ones, 0xFF) or none (mask is 0) of the stencil bits,
+                // because on Vulkan, the pipeline state does not affect clears.
+                var dstTexture = FramebufferParams.GetDepthStencilAttachment();
+                if (dstTexture == null)
+                {
+                    return;
+                }
+
+                // TODO: Clear only the specified layer.
+                Gd.HelperShader.Clear(
+                    Gd,
+                    dstTexture,
+                    depthValue,
+                    depthMask,
+                    stencilValue,
+                    stencilMask,
+                    (int)FramebufferParams.Width,
+                    (int)FramebufferParams.Height,
+                    FramebufferParams.AttachmentFormats[FramebufferParams.AttachmentsCount - 1],
+                    ClearScissor);
+            }
+            else
+            {
+                ClearRenderTargetDepthStencil(layer, layerCount, depthValue, depthMask, stencilValue, stencilMask != 0);
+            }
+        }
+
         public void EndHostConditionalRendering()
         {
             if (Gd.Capabilities.SupportsConditionalRendering)

src/Ryujinx.Graphics.Vulkan/Ryujinx.Graphics.Vulkan.csproj

@@ -42,6 +42,7 @@
     <EmbeddedResource Include="Shaders\SpirvBinaries\DepthBlitMsFragment.spv" />
     <EmbeddedResource Include="Shaders\SpirvBinaries\DepthDrawToMsFragment.spv" />
     <EmbeddedResource Include="Shaders\SpirvBinaries\DepthDrawToNonMsFragment.spv" />
+    <EmbeddedResource Include="Shaders\SpirvBinaries\DepthStencilClearFragment.spv" />
     <EmbeddedResource Include="Shaders\SpirvBinaries\StencilBlitFragment.spv" />
     <EmbeddedResource Include="Shaders\SpirvBinaries\StencilBlitMsFragment.spv" />
     <EmbeddedResource Include="Shaders\SpirvBinaries\StencilDrawToMsFragment.spv" />

src/Ryujinx.Graphics.Vulkan/Shaders/DepthStencilClearFragmentShaderSource.frag

@@ -0,0 +1,8 @@
+#version 450 core
+
+layout (location = 0) in vec4 clear_colour;
+
+void main()
+{
+    gl_FragDepth = clear_colour.x;
+}