Vulkan: Zero blend state when disabled or write mask is 0 (#3719)

* Zero blend state when disabled or write mask is 0

Any difference in the blend state when blend is disabled is meaningless, but Ryujinx would compare different disabled blends and compile them as separate pipelines. This change ensures that all pipelines where blend state is meaningless record it as such, which avoids compiling a bunch of pipelines that are essentially identical.

The NVIDIA driver is pretty forgiving when it comes to silly pipeline misses like this, but other drivers don't offer the same level of kindness.

This should reduce stuttering on those drivers, and might improve overall performance very slightly due to less pipeline variants being in the hash table.

* Fix blend possibly being wrong when an attachment is unmasked

Ryujinx.Graphics.Vulkan/PipelineBase.cs

@@ -68,6 +68,8 @@ namespace Ryujinx.Graphics.Vulkan
         private bool _tfEnabled;
         private bool _tfActive;
 
+        private PipelineColorBlendAttachmentState[] _storedBlend;
+
         public ulong DrawCount { get; private set; }
 
         public unsafe PipelineBase(VulkanRenderer gd, Device device)
@@ -104,6 +106,8 @@ namespace Ryujinx.Graphics.Vulkan
             _newState.Initialize();
             _newState.LineWidth = 1f;
             _newState.SamplesCount = 1;
+
+            _storedBlend = new PipelineColorBlendAttachmentState[8];
         }
 
         public void Initialize()
@@ -498,13 +502,28 @@ namespace Ryujinx.Graphics.Vulkan
         {
             ref var vkBlend = ref _newState.Internal.ColorBlendAttachmentState[index];
 
-            vkBlend.BlendEnable = blend.Enable;
-            vkBlend.SrcColorBlendFactor = blend.ColorSrcFactor.Convert();
-            vkBlend.DstColorBlendFactor = blend.ColorDstFactor.Convert();
-            vkBlend.ColorBlendOp = blend.ColorOp.Convert();
-            vkBlend.SrcAlphaBlendFactor = blend.AlphaSrcFactor.Convert();
-            vkBlend.DstAlphaBlendFactor = blend.AlphaDstFactor.Convert();
-            vkBlend.AlphaBlendOp = blend.AlphaOp.Convert();
+            if (blend.Enable)
+            {
+                vkBlend.BlendEnable = blend.Enable;
+                vkBlend.SrcColorBlendFactor = blend.ColorSrcFactor.Convert();
+                vkBlend.DstColorBlendFactor = blend.ColorDstFactor.Convert();
+                vkBlend.ColorBlendOp = blend.ColorOp.Convert();
+                vkBlend.SrcAlphaBlendFactor = blend.AlphaSrcFactor.Convert();
+                vkBlend.DstAlphaBlendFactor = blend.AlphaDstFactor.Convert();
+                vkBlend.AlphaBlendOp = blend.AlphaOp.Convert();
+            }
+            else
+            {
+                vkBlend = new PipelineColorBlendAttachmentState(
+                    colorWriteMask: vkBlend.ColorWriteMask);
+            }
+
+            if (vkBlend.ColorWriteMask == 0)
+            {
+                _storedBlend[index] = vkBlend;
+
+                vkBlend = new PipelineColorBlendAttachmentState();
+            }
 
             _newState.BlendConstantR = blend.BlendConstant.Red;
             _newState.BlendConstantG = blend.BlendConstant.Green;
@@ -669,8 +688,25 @@ namespace Ryujinx.Graphics.Vulkan
             for (int i = 0; i < count; i++)
             {
                 ref var vkBlend = ref _newState.Internal.ColorBlendAttachmentState[i];
+                var newMask = (ColorComponentFlags)componentMask[i];
+
+                // When color write mask is 0, remove all blend state to help the pipeline cache.
+                // Restore it when the mask becomes non-zero.
+                if (vkBlend.ColorWriteMask != newMask)
+                {
+                    if (newMask == 0)
+                    {
+                        _storedBlend[i] = vkBlend;
+
+                        vkBlend = new PipelineColorBlendAttachmentState();
+                    }
+                    else if (vkBlend.ColorWriteMask == 0)
+                    {
+                        vkBlend = _storedBlend[i];
+                    }
+                }
 
-                vkBlend.ColorWriteMask = (ColorComponentFlags)componentMask[i];
+                vkBlend.ColorWriteMask = newMask;
 
                 if (componentMask[i] != 0)
                 {

Ryujinx.Graphics.Vulkan/PipelineConverter.cs

@@ -257,15 +257,23 @@ namespace Ryujinx.Graphics.Vulkan
             {
                 var blend = state.BlendDescriptors[i];
 
-                pipeline.Internal.ColorBlendAttachmentState[i] = new PipelineColorBlendAttachmentState(
-                    blend.Enable,
-                    blend.ColorSrcFactor.Convert(),
-                    blend.ColorDstFactor.Convert(),
-                    blend.ColorOp.Convert(),
-                    blend.AlphaSrcFactor.Convert(),
-                    blend.AlphaDstFactor.Convert(),
-                    blend.AlphaOp.Convert(),
-                    (ColorComponentFlags)state.ColorWriteMask[i]);
+                if (blend.Enable && state.ColorWriteMask[i] != 0)
+                {
+                    pipeline.Internal.ColorBlendAttachmentState[i] = new PipelineColorBlendAttachmentState(
+                        blend.Enable,
+                        blend.ColorSrcFactor.Convert(),
+                        blend.ColorDstFactor.Convert(),
+                        blend.ColorOp.Convert(),
+                        blend.AlphaSrcFactor.Convert(),
+                        blend.AlphaDstFactor.Convert(),
+                        blend.AlphaOp.Convert(),
+                        (ColorComponentFlags)state.ColorWriteMask[i]);
+                }
+                else
+                {
+                    pipeline.Internal.ColorBlendAttachmentState[i] = new PipelineColorBlendAttachmentState(
+                        colorWriteMask: (ColorComponentFlags)state.ColorWriteMask[i]);
+                }
             }
 
             int maxAttachmentIndex = 0;