Make sure blend is disabled if render target has integer format (#5122)

* Make sure blend is disabled if render target has integer format

* Change approach to avoid permanently mutating state

src/Ryujinx.Graphics.Vulkan/FramebufferParams.cs

@@ -21,6 +21,7 @@ namespace Ryujinx.Graphics.Vulkan
         public uint[] AttachmentSamples { get; }
         public VkFormat[] AttachmentFormats { get; }
         public int[] AttachmentIndices { get; }
+        public uint AttachmentIntegerFormatMask { get; }
 
         public int AttachmentsCount { get; }
         public int MaxColorAttachmentIndex => AttachmentIndices.Length > 0 ? AttachmentIndices[AttachmentIndices.Length - 1] : -1;
@@ -74,6 +75,7 @@ namespace Ryujinx.Graphics.Vulkan
 
             int index = 0;
             int bindIndex = 0;
+            uint attachmentIntegerFormatMask = 0;
 
             foreach (ITexture color in colors)
             {
@@ -89,6 +91,11 @@ namespace Ryujinx.Graphics.Vulkan
                     AttachmentFormats[index] = texture.VkFormat;
                     AttachmentIndices[index] = bindIndex;
 
+                    if (texture.Info.Format.IsInteger())
+                    {
+                        attachmentIntegerFormatMask |= 1u << bindIndex;
+                    }
+
                     width = Math.Min(width, (uint)texture.Width);
                     height = Math.Min(height, (uint)texture.Height);
                     layers = Math.Min(layers, (uint)texture.Layers);
@@ -102,6 +109,8 @@ namespace Ryujinx.Graphics.Vulkan
                 bindIndex++;
             }
 
+            AttachmentIntegerFormatMask = attachmentIntegerFormatMask;
+
             if (depthStencil is TextureView dsTexture && dsTexture.Valid)
             {
                 _attachments[count - 1] = dsTexture.GetImageViewForAttachment();

src/Ryujinx.Graphics.Vulkan/PipelineBase.cs

@@ -1487,6 +1487,7 @@ namespace Ryujinx.Graphics.Vulkan
         {
             var dstAttachmentFormats = _newState.Internal.AttachmentFormats.AsSpan();
             FramebufferParams.AttachmentFormats.CopyTo(dstAttachmentFormats);
+            _newState.Internal.AttachmentIntegerFormatMask = FramebufferParams.AttachmentIntegerFormatMask;
 
             for (int i = FramebufferParams.AttachmentFormats.Length; i < dstAttachmentFormats.Length; i++)
             {

src/Ryujinx.Graphics.Vulkan/PipelineConverter.cs

@@ -294,6 +294,7 @@ namespace Ryujinx.Graphics.Vulkan
 
             int attachmentCount = 0;
             int maxColorAttachmentIndex = -1;
+            uint attachmentIntegerFormatMask = 0;
 
             for (int i = 0; i < Constants.MaxRenderTargets; i++)
             {
@@ -301,6 +302,11 @@ namespace Ryujinx.Graphics.Vulkan
                 {
                     pipeline.Internal.AttachmentFormats[attachmentCount++] = gd.FormatCapabilities.ConvertToVkFormat(state.AttachmentFormats[i]);
                     maxColorAttachmentIndex = i;
+
+                    if (state.AttachmentFormats[i].IsInteger())
+                    {
+                        attachmentIntegerFormatMask |= 1u << i;
+                    }
                 }
             }
 
@@ -311,6 +317,7 @@ namespace Ryujinx.Graphics.Vulkan
 
             pipeline.ColorBlendAttachmentStateCount = (uint)(maxColorAttachmentIndex + 1);
             pipeline.VertexAttributeDescriptionsCount = (uint)Math.Min(Constants.MaxVertexAttributes, state.VertexAttribCount);
+            pipeline.Internal.AttachmentIntegerFormatMask = attachmentIntegerFormatMask;
 
             return pipeline;
         }

src/Ryujinx.Graphics.Vulkan/PipelineState.cs

@@ -1,6 +1,7 @@
 using Ryujinx.Common.Memory;
 using Silk.NET.Vulkan;
 using System;
+using System.Numerics;
 
 namespace Ryujinx.Graphics.Vulkan
 {
@@ -542,6 +543,27 @@ namespace Ryujinx.Graphics.Vulkan
                     MaxDepthBounds = MaxDepthBounds
                 };
 
+                uint blendEnables = 0;
+
+                if (gd.IsMoltenVk && Internal.AttachmentIntegerFormatMask != 0)
+                {
+                    // Blend can't be enabled for integer formats, so let's make sure it is disabled.
+                    uint attachmentIntegerFormatMask = Internal.AttachmentIntegerFormatMask;
+
+                    while (attachmentIntegerFormatMask != 0)
+                    {
+                        int i = BitOperations.TrailingZeroCount(attachmentIntegerFormatMask);
+
+                        if (Internal.ColorBlendAttachmentState[i].BlendEnable)
+                        {
+                            blendEnables |= 1u << i;
+                        }
+
+                        Internal.ColorBlendAttachmentState[i].BlendEnable = false;
+                        attachmentIntegerFormatMask &= ~(1u << i);
+                    }
+                }
+
                 var colorBlendState = new PipelineColorBlendStateCreateInfo()
                 {
                     SType = StructureType.PipelineColorBlendStateCreateInfo,
@@ -619,6 +641,15 @@ namespace Ryujinx.Graphics.Vulkan
                 };
 
                 gd.Api.CreateGraphicsPipelines(device, cache, 1, &pipelineCreateInfo, null, &pipelineHandle).ThrowOnError();
+
+                // Restore previous blend enable values if we changed it.
+                while (blendEnables != 0)
+                {
+                    int i = BitOperations.TrailingZeroCount(blendEnables);
+
+                    Internal.ColorBlendAttachmentState[i].BlendEnable = true;
+                    blendEnables &= ~(1u << i);
+                }
             }
 
             pipeline = new Auto<DisposablePipeline>(new DisposablePipeline(gd.Api, device, pipelineHandle));

src/Ryujinx.Graphics.Vulkan/PipelineUid.cs

@@ -35,6 +35,7 @@ namespace Ryujinx.Graphics.Vulkan
         public Array16<Rect2D> Scissors;
         public Array8<PipelineColorBlendAttachmentState> ColorBlendAttachmentState;
         public Array9<Format> AttachmentFormats;
+        public uint AttachmentIntegerFormatMask;
 
         public override bool Equals(object obj)
         {