Fix incorrect vertex attribute format change (#5112)

* Fix incorrect vertex attribute format change

* Only change vertex format if the host supports the new format

src/Ryujinx.Graphics.Vulkan/FormatCapabilities.cs

@@ -65,6 +65,13 @@ namespace Ryujinx.Graphics.Vulkan
             return (formatFeatureFlags & flags) == flags;
         }
 
+        public bool BufferFormatSupports(FormatFeatureFlags flags, VkFormat format)
+        {
+            _api.GetPhysicalDeviceFormatProperties(_physicalDevice, format, out var fp);
+
+            return (fp.BufferFeatures & flags) == flags;
+        }
+
         public bool OptimalFormatSupports(FormatFeatureFlags flags, GAL.Format format)
         {
             var formatFeatureFlags = _optimalTable[(int)format];

src/Ryujinx.Graphics.Vulkan/PipelineBase.cs

@@ -551,7 +551,6 @@ namespace Ryujinx.Graphics.Vulkan
                         (uint)maxDrawCount,
                         (uint)stride);
                 }
-
             }
             else
             {

src/Ryujinx.Graphics.Vulkan/PipelineState.cs

@@ -414,7 +414,7 @@ namespace Ryujinx.Graphics.Vulkan
 
             if (isMoltenVk)
             {
-                UpdateVertexAttributeDescriptions();
+                UpdateVertexAttributeDescriptions(gd);
             }
 
             fixed (VertexInputAttributeDescription* pVertexAttributeDescriptions = &Internal.VertexAttributeDescriptions[0])
@@ -641,7 +641,7 @@ namespace Ryujinx.Graphics.Vulkan
             }
         }
 
-        private void UpdateVertexAttributeDescriptions()
+        private void UpdateVertexAttributeDescriptions(VulkanRenderer gd)
         {
             // Vertex attributes exceeding the stride are invalid.
             // In metal, they cause glitches with the vertex shader fetching incorrect values.
@@ -651,30 +651,52 @@ namespace Ryujinx.Graphics.Vulkan
             for (int index = 0; index < VertexAttributeDescriptionsCount; index++)
             {
                 var attribute = Internal.VertexAttributeDescriptions[index];
-                ref var vb = ref Internal.VertexBindingDescriptions[(int)attribute.Binding];
+                int vbIndex = GetVertexBufferIndex(attribute.Binding);
 
-                Format format = attribute.Format;
-
-                while (vb.Stride != 0 && attribute.Offset + FormatTable.GetAttributeFormatSize(format) > vb.Stride)
+                if (vbIndex >= 0)
                 {
-                    Format newFormat = FormatTable.DropLastComponent(format);
+                    ref var vb = ref Internal.VertexBindingDescriptions[vbIndex];
+
+                    Format format = attribute.Format;
 
-                    if (newFormat == format)
+                    while (vb.Stride != 0 && attribute.Offset + FormatTable.GetAttributeFormatSize(format) > vb.Stride)
                     {
-                        // That case means we failed to find a format that fits within the stride,
-                        // so just restore the original format and give up.
-                        format = attribute.Format;
-                        break;
+                        Format newFormat = FormatTable.DropLastComponent(format);
+
+                        if (newFormat == format)
+                        {
+                            // That case means we failed to find a format that fits within the stride,
+                            // so just restore the original format and give up.
+                            format = attribute.Format;
+                            break;
+                        }
+
+                        format = newFormat;
                     }
 
-                    format = newFormat;
+                    if (attribute.Format != format && gd.FormatCapabilities.BufferFormatSupports(FormatFeatureFlags.VertexBufferBit, format))
+                    {
+                        attribute.Format = format;
+                    }
                 }
 
-                attribute.Format = format;
                 _vertexAttributeDescriptions2[index] = attribute;
             }
         }
 
+        private int GetVertexBufferIndex(uint binding)
+        {
+            for (int index = 0; index < VertexBindingDescriptionsCount; index++)
+            {
+                if (Internal.VertexBindingDescriptions[index].Binding == binding)
+                {
+                    return index;
+                }
+            }
+
+            return -1;
+        }
+
         public void Dispose()
         {
             Stages.Dispose();