Clamp vertex buffer size to mapped size if too high (#6272)

* Clamp vertex buffer size to mapped size if too high

* Update comment

src/Ryujinx.Graphics.Gpu/Engine/Threed/StateUpdater.cs

@@ -26,6 +26,9 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed
         public const int PrimitiveRestartStateIndex = 12;
         public const int RenderTargetStateIndex = 27;
 
+        // Vertex buffers larger than this size will be clamped to the mapped size.
+        private const ulong VertexBufferSizeToMappedSizeThreshold = 256 * 1024 * 1024; // 256 MB
+
         private readonly GpuContext _context;
         private readonly GpuChannel _channel;
         private readonly DeviceStateWithShadow<ThreedClassState> _state;
@@ -1144,6 +1147,14 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed
 
                         size = Math.Min(size, maxVertexBufferSize);
                     }
+                    else if (size > VertexBufferSizeToMappedSizeThreshold)
+                    {
+                        // Make sure we have a sane vertex buffer size, since in some cases applications
+                        // might set the "end address" of the vertex buffer to the end of the GPU address space,
+                        // which would result in a several GBs large buffer.
+
+                        size = _channel.MemoryManager.GetMappedSize(address, size);
+                    }
                 }
                 else
                 {