Inherit buffer tracking handles rather than recreating on resize (#2330)

This greatly speeds up games that constantly resize buffers, and removes stuttering on games that resize large buffers occasionally:

- Large improvement on Super Mario 3D All-Stars (#1663 needed for best performance)
- Improvement to Hyrule Warriors: AoC, and UE4 games. These games can still stutter due to texture creation/loading.
- Small improvement to other games, potential 1-frame stutters avoided.

`ForceSynchronizeMemory`, which was added with POWER, is no longer needed. Some tests have been added for the MultiRegionHandle.

Ryujinx.Cpu/IVirtualMemoryManagerTracked.cs

@@ -1,6 +1,8 @@
 using Ryujinx.Cpu.Tracking;
 using Ryujinx.Memory;
+using Ryujinx.Memory.Tracking;
 using System;
+using System.Collections.Generic;
 
 namespace Ryujinx.Cpu
 {
@@ -26,9 +28,10 @@ namespace Ryujinx.Cpu
         /// </summary>
         /// <param name="address">CPU virtual address of the region</param>
         /// <param name="size">Size of the region</param>
+        /// <param name="handles">Handles to inherit state from or reuse. When none are present, provide null</param>
         /// <param name="granularity">Desired granularity of write tracking</param>
         /// <returns>The memory tracking handle</returns>
-        CpuMultiRegionHandle BeginGranularTracking(ulong address, ulong size, ulong granularity);
+        CpuMultiRegionHandle BeginGranularTracking(ulong address, ulong size, IEnumerable<IRegionHandle> handles, ulong granularity);
 
         /// <summary>
         /// Obtains a smart memory tracking handle for the given virtual region, with a specified granularity. This should be disposed when finished with.

Ryujinx.Cpu/MemoryManager.cs

@@ -550,9 +550,9 @@ namespace Ryujinx.Cpu
         }
 
         /// <inheritdoc/>
-        public CpuMultiRegionHandle BeginGranularTracking(ulong address, ulong size, ulong granularity)
+        public CpuMultiRegionHandle BeginGranularTracking(ulong address, ulong size, IEnumerable<IRegionHandle> handles, ulong granularity)
         {
-            return new CpuMultiRegionHandle(Tracking.BeginGranularTracking(address, size, granularity));
+            return new CpuMultiRegionHandle(Tracking.BeginGranularTracking(address, size, handles, granularity));
         }
 
         /// <inheritdoc/>

Ryujinx.Cpu/MemoryManagerHostMapped.cs

@@ -586,9 +586,9 @@ namespace Ryujinx.Cpu
         }
 
         /// <inheritdoc/>
-        public CpuMultiRegionHandle BeginGranularTracking(ulong address, ulong size, ulong granularity)
+        public CpuMultiRegionHandle BeginGranularTracking(ulong address, ulong size, IEnumerable<IRegionHandle> handles, ulong granularity)
         {
-            return new CpuMultiRegionHandle(Tracking.BeginGranularTracking(address, size, granularity));
+            return new CpuMultiRegionHandle(Tracking.BeginGranularTracking(address, size, handles, granularity));
         }
 
         /// <inheritdoc/>

Ryujinx.Cpu/Tracking/CpuMultiRegionHandle.cs

@@ -1,5 +1,6 @@
 using Ryujinx.Memory.Tracking;
 using System;
+using System.Collections.Generic;
 
 namespace Ryujinx.Cpu.Tracking
 {
@@ -16,6 +17,7 @@ namespace Ryujinx.Cpu.Tracking
 
         public void Dispose() => _impl.Dispose();
         public void ForceDirty(ulong address, ulong size) => _impl.ForceDirty(address, size);
+        public IEnumerable<IRegionHandle> GetHandles() => _impl.GetHandles();
         public void QueryModified(Action<ulong, ulong> modifiedAction) => _impl.QueryModified(modifiedAction);
         public void QueryModified(ulong address, ulong size, Action<ulong, ulong> modifiedAction) => _impl.QueryModified(address, size, modifiedAction);
         public void QueryModified(ulong address, ulong size, Action<ulong, ulong> modifiedAction, int sequenceNumber) => _impl.QueryModified(address, size, modifiedAction, sequenceNumber);

Ryujinx.Cpu/Tracking/CpuRegionHandle.cs

@@ -21,6 +21,7 @@ namespace Ryujinx.Cpu.Tracking
         public void Dispose() => _impl.Dispose();
         public bool DirtyOrVolatile() => _impl.DirtyOrVolatile();
         public void ForceDirty() => _impl.ForceDirty();
+        public IRegionHandle GetHandle() => _impl;
         public void RegisterAction(RegionSignal action) => _impl.RegisterAction(action);
         public void RegisterDirtyEvent(Action action) => _impl.RegisterDirtyEvent(action);
         public void Reprotect(bool asDirty = false) => _impl.Reprotect(asDirty);

Ryujinx.Graphics.Gpu/Memory/Buffer.cs

@@ -3,6 +3,8 @@ using Ryujinx.Graphics.GAL;
 using Ryujinx.Memory.Range;
 using Ryujinx.Memory.Tracking;
 using System;
+using System.Collections.Generic;
+using System.Linq;
 
 namespace Ryujinx.Graphics.Gpu.Memory
 {
@@ -68,7 +70,8 @@ namespace Ryujinx.Graphics.Gpu.Memory
         /// <param name="context">GPU context that the buffer belongs to</param>
         /// <param name="address">Start address of the buffer</param>
         /// <param name="size">Size of the buffer in bytes</param>
-        public Buffer(GpuContext context, ulong address, ulong size)
+        /// <param name="baseBuffers">Buffers which this buffer contains, and will inherit tracking handles from</param>
+        public Buffer(GpuContext context, ulong address, ulong size, IEnumerable<Buffer> baseBuffers = null)
         {
             _context = context;
             Address  = address;
@@ -78,13 +81,45 @@ namespace Ryujinx.Graphics.Gpu.Memory
 
             _useGranular = size > GranularBufferThreshold;
 
+            IEnumerable<IRegionHandle> baseHandles = null;
+
+            if (baseBuffers != null)
+            {
+                baseHandles = baseBuffers.SelectMany(buffer =>
+                {
+                    if (buffer._useGranular)
+                    {
+                        return buffer._memoryTrackingGranular.GetHandles();
+                    }
+                    else
+                    {
+                        return Enumerable.Repeat(buffer._memoryTracking.GetHandle(), 1);
+                    }
+                });
+            }
+
             if (_useGranular)
             {
-                _memoryTrackingGranular = context.PhysicalMemory.BeginGranularTracking(address, size);
+                _memoryTrackingGranular = context.PhysicalMemory.BeginGranularTracking(address, size, baseHandles);
             }
             else
             {
                 _memoryTracking = context.PhysicalMemory.BeginTracking(address, size);
+
+                if (baseHandles != null)
+                {
+                    _memoryTracking.Reprotect(false);
+
+                    foreach (IRegionHandle handle in baseHandles)
+                    {
+                        if (handle.Dirty)
+                        {
+                            _memoryTracking.Reprotect(true);
+                        }
+
+                        handle.Dispose();
+                    }
+                }
             }
 
             _externalFlushDelegate = new RegionSignal(ExternalFlush);
@@ -180,39 +215,6 @@ namespace Ryujinx.Graphics.Gpu.Memory
             }
         }
 
-        /// <summary>
-        /// Performs guest to host memory synchronization of the buffer data, regardless of sequence number.
-        /// </summary>
-        /// <remarks>
-        /// This causes the buffer data to be overwritten if a write was detected from the CPU,
-        /// since the last call to this method.
-        /// </remarks>
-        /// <param name="address">Start address of the range to synchronize</param>
-        /// <param name="size">Size in bytes of the range to synchronize</param>
-        public void ForceSynchronizeMemory(ulong address, ulong size)
-        {
-            if (_useGranular)
-            {
-                _memoryTrackingGranular.QueryModified(address, size, _modifiedDelegate);
-            }
-            else
-            {
-                if (_memoryTracking.DirtyOrVolatile())
-                {
-                    _memoryTracking.Reprotect();
-
-                    if (_modifiedRanges != null)
-                    {
-                        _modifiedRanges.ExcludeModifiedRegions(Address, Size, _loadDelegate);
-                    }
-                    else
-                    {
-                        _context.Renderer.SetBufferData(Handle, 0, _context.PhysicalMemory.GetSpan(Address, (int)Size));
-                    }
-                }
-            }
-        }
-
         /// <summary>
         /// Ensure that the modified range list exists.
         /// </summary>
@@ -461,18 +463,26 @@ namespace Ryujinx.Graphics.Gpu.Memory
         }
 
         /// <summary>
-        /// Disposes the host buffer.
+        /// Disposes the host buffer's data, not its tracking handles.
         /// </summary>
-        public void Dispose()
+        public void DisposeData()
         {
             _modifiedRanges?.Clear();
 
-            _memoryTrackingGranular?.Dispose();
-            _memoryTracking?.Dispose();
-
             _context.Renderer.DeleteBuffer(Handle);
 
             UnmappedSequence++;
         }
+
+        /// <summary>
+        /// Disposes the host buffer.
+        /// </summary>
+        public void Dispose()
+        {
+            _memoryTrackingGranular?.Dispose();
+            _memoryTracking?.Dispose();
+
+            DisposeData();
+        }
     }
 }

Ryujinx.Graphics.Gpu/Memory/BufferManager.cs

@@ -533,8 +533,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
                         }
                     }
 
-                    Buffer newBuffer = new Buffer(_context, address, endAddress - address);
-                    newBuffer.SynchronizeMemory(address, endAddress - address);
+                    Buffer newBuffer = new Buffer(_context, address, endAddress - address, _bufferOverlaps.Take(overlapsCount));
 
                     lock (_buffers)
                     {
@@ -547,14 +546,14 @@ namespace Ryujinx.Graphics.Gpu.Memory
 
                         int dstOffset = (int)(buffer.Address - newBuffer.Address);
 
-                        buffer.ForceSynchronizeMemory(buffer.Address, buffer.Size);
-
                         buffer.CopyTo(newBuffer, dstOffset);
                         newBuffer.InheritModifiedRanges(buffer);
 
-                        buffer.Dispose();
+                        buffer.DisposeData();
                     }
 
+                    newBuffer.SynchronizeMemory(address, endAddress - address);
+
                     // Existing buffers were modified, we need to rebind everything.
                     _rebind = true;
                 }

Ryujinx.Graphics.Gpu/Memory/PhysicalMemory.cs

@@ -2,7 +2,9 @@ using Ryujinx.Cpu;
 using Ryujinx.Cpu.Tracking;
 using Ryujinx.Memory;
 using Ryujinx.Memory.Range;
+using Ryujinx.Memory.Tracking;
 using System;
+using System.Collections.Generic;
 using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
 
@@ -200,11 +202,12 @@ namespace Ryujinx.Graphics.Gpu.Memory
         /// </summary>
         /// <param name="address">CPU virtual address of the region</param>
         /// <param name="size">Size of the region</param>
+        /// <param name="handles">Handles to inherit state from or reuse</param>
         /// <param name="granularity">Desired granularity of write tracking</param>
         /// <returns>The memory tracking handle</returns>
-        public CpuMultiRegionHandle BeginGranularTracking(ulong address, ulong size, ulong granularity = 4096)
+        public CpuMultiRegionHandle BeginGranularTracking(ulong address, ulong size, IEnumerable<IRegionHandle> handles = null, ulong granularity = 4096)
         {
-            return _cpuMemory.BeginGranularTracking(address, size, granularity);
+            return _cpuMemory.BeginGranularTracking(address, size, handles, granularity);
         }
 
         /// <summary>

Ryujinx.Memory.Tests/MultiRegionTrackingTests.cs

@@ -35,7 +35,7 @@ namespace Ryujinx.Memory.Tests
         {
             return smart ?
                 _tracking.BeginSmartGranularTracking(address, size, granularity) :
-                (IMultiRegionHandle)_tracking.BeginGranularTracking(address, size, granularity);
+                (IMultiRegionHandle)_tracking.BeginGranularTracking(address, size, null, granularity);
         }
 
         private void RandomOrder(Random random, List<int> indices, Action<int> action)
@@ -279,5 +279,125 @@ namespace Ryujinx.Memory.Tests
 
             Assert.AreEqual(0, _tracking.GetRegionCount());
         }
+
+        [Test]
+        public void InheritHandles()
+        {
+            // Test merging the following into a granular region handle:
+            // - 3x gap (creates new granular handles)
+            // - 3x from multiregion: not dirty, dirty and with action
+            // - 2x gap
+            // - 3x single page: not dirty, dirty and with action
+            // - 3x two page: not dirty, dirty and with action (handle is not reused, but its state is copied to the granular handles)
+            // - 1x gap
+            // For a total of 18 pages.
+
+            bool[] actionsTriggered = new bool[3];
+
+            MultiRegionHandle granular = _tracking.BeginGranularTracking(PageSize * 3, PageSize * 3, null, PageSize);
+            PreparePages(granular, 3, PageSize * 3);
+
+            // Write to the second handle in the multiregion.
+            _tracking.VirtualMemoryEvent(PageSize * 4, PageSize, true);
+
+            // Add an action to the third handle in the multiregion.
+            granular.RegisterAction(PageSize * 5, PageSize, (_, _) => { actionsTriggered[0] = true; });
+
+            RegionHandle[] singlePages = new RegionHandle[3];
+
+            for (int i = 0; i < 3; i++)
+            {
+                singlePages[i] = _tracking.BeginTracking(PageSize * (8 + (ulong)i), PageSize);
+                singlePages[i].Reprotect();
+            }
+
+            // Write to the second handle.
+            _tracking.VirtualMemoryEvent(PageSize * 9, PageSize, true);
+
+            // Add an action to the third handle.
+            singlePages[2].RegisterAction((_, _) => { actionsTriggered[1] = true; });
+
+            RegionHandle[] doublePages = new RegionHandle[3];
+
+            for (int i = 0; i < 3; i++)
+            {
+                doublePages[i] = _tracking.BeginTracking(PageSize * (11 + (ulong)i * 2), PageSize * 2);
+                doublePages[i].Reprotect();
+            }
+
+            // Write to the second handle.
+            _tracking.VirtualMemoryEvent(PageSize * 13, PageSize * 2, true);
+
+            // Add an action to the third handle.
+            doublePages[2].RegisterAction((_, _) => { actionsTriggered[2] = true; });
+
+            // Finally, create a granular handle that inherits all these handles.
+
+            IEnumerable<IRegionHandle>[] handleGroups = new IEnumerable<IRegionHandle>[]
+            {
+                granular.GetHandles(),
+                singlePages,
+                doublePages
+            };
+
+            MultiRegionHandle combined = _tracking.BeginGranularTracking(0, PageSize * 18, handleGroups.SelectMany((handles) => handles), PageSize);
+
+            bool[] expectedDirty = new bool[]
+            {
+                true, true, true, // Gap.
+                false, true, false, // Multi-region.
+                true, true, // Gap.
+                false, true, false, // Individual handles.
+                false, false, true, true, false, false, // Double size handles.
+                true // Gap.
+            };
+
+            for (int i = 0; i < 18; i++)
+            {
+                bool modified = false;
+                combined.QueryModified(PageSize * (ulong)i, PageSize, (_, _) => { modified = true; });
+
+                Assert.AreEqual(expectedDirty[i], modified);
+            }
+
+            Assert.AreEqual(new bool[3], actionsTriggered);
+
+            _tracking.VirtualMemoryEvent(PageSize * 5, PageSize, false);
+            Assert.IsTrue(actionsTriggered[0]);
+
+            _tracking.VirtualMemoryEvent(PageSize * 10, PageSize, false);
+            Assert.IsTrue(actionsTriggered[1]);
+
+            _tracking.VirtualMemoryEvent(PageSize * 15, PageSize, false);
+            Assert.IsTrue(actionsTriggered[2]);
+
+            // The double page handles should be disposed, as they were split into granular handles.
+            foreach (RegionHandle doublePage in doublePages)
+            {
+                // These should have been disposed.
+                bool throws = false;
+
+                try
+                {
+                    doublePage.Dispose();
+                }
+                catch (ObjectDisposedException)
+                {
+                    throws = true;
+                }
+
+                Assert.IsTrue(throws);
+            }
+
+            IEnumerable<IRegionHandle> combinedHandles = combined.GetHandles();
+
+            Assert.AreEqual(handleGroups[0].ElementAt(0), combinedHandles.ElementAt(3));
+            Assert.AreEqual(handleGroups[0].ElementAt(1), combinedHandles.ElementAt(4));
+            Assert.AreEqual(handleGroups[0].ElementAt(2), combinedHandles.ElementAt(5));
+
+            Assert.AreEqual(singlePages[0], combinedHandles.ElementAt(8));
+            Assert.AreEqual(singlePages[1], combinedHandles.ElementAt(9));
+            Assert.AreEqual(singlePages[2], combinedHandles.ElementAt(10));
+        }
     }
 }

Ryujinx.Memory/Tracking/MemoryTracking.cs

@@ -134,13 +134,14 @@ namespace Ryujinx.Memory.Tracking
         /// </summary>
         /// <param name="address">CPU virtual address of the region</param>
         /// <param name="size">Size of the region</param>
+        /// <param name="handles">Handles to inherit state from or reuse. When none are present, provide null</param>
         /// <param name="granularity">Desired granularity of write tracking</param>
         /// <returns>The memory tracking handle</returns>
-        public MultiRegionHandle BeginGranularTracking(ulong address, ulong size, ulong granularity)
+        public MultiRegionHandle BeginGranularTracking(ulong address, ulong size, IEnumerable<IRegionHandle> handles, ulong granularity)
         {
             (address, size) = PageAlign(address, size);
 
-            return new MultiRegionHandle(this, address, size, granularity);
+            return new MultiRegionHandle(this, address, size, handles, granularity);
         }
 
         /// <summary>

Ryujinx.Memory/Tracking/MultiRegionHandle.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Generic;
 
 namespace Ryujinx.Memory.Tracking
 {
@@ -18,16 +19,68 @@ namespace Ryujinx.Memory.Tracking
 
         public bool Dirty { get; private set; } = true;
 
-        internal MultiRegionHandle(MemoryTracking tracking, ulong address, ulong size, ulong granularity)
+        internal MultiRegionHandle(MemoryTracking tracking, ulong address, ulong size, IEnumerable<IRegionHandle> handles, ulong granularity)
         {
             _handles = new RegionHandle[size / granularity];
             Granularity = granularity;
 
-            for (int i = 0; i < _handles.Length; i++)
+            int i = 0;
+
+            if (handles != null)
+            {
+                // Inherit from the handles we were given. Any gaps must be filled with new handles,
+                // and old handles larger than our granularity must copy their state onto new granular handles and dispose.
+                // It is assumed that the provided handles do not overlap, in order, are on page boundaries,
+                // and don't extend past the requested range.
+
+                foreach (RegionHandle handle in handles)
+                {
+                    int startIndex = (int)((handle.Address - address) / granularity);
+
+                    // Fill any gap left before this handle.
+                    while (i < startIndex)
+                    {
+                        RegionHandle fillHandle = tracking.BeginTracking(address + (ulong)i * granularity, granularity);
+                        fillHandle.Parent = this;
+                        _handles[i++] = fillHandle;
+                    }
+
+                    if (handle.Size == granularity)
+                    {
+                        handle.Parent = this;
+                        _handles[i++] = handle;
+                    }
+                    else
+                    {
+                        int endIndex = (int)((handle.EndAddress - address) / granularity);
+
+                        while (i < endIndex)
+                        {
+                            RegionHandle splitHandle = tracking.BeginTracking(address + (ulong)i * granularity, granularity);
+                            splitHandle.Parent = this;
+
+                            splitHandle.Reprotect(handle.Dirty);
+
+                            RegionSignal signal = handle.PreAction;
+                            if (signal != null)
+                            {
+                                splitHandle.RegisterAction(signal);
+                            }
+
+                            _handles[i++] = splitHandle;
+                        }
+
+                        handle.Dispose();
+                    }
+                }
+            }
+
+            // Fill any remaining space with new handles.
+            while (i < _handles.Length)
             {
                 RegionHandle handle = tracking.BeginTracking(address + (ulong)i * granularity, granularity);
                 handle.Parent = this;
-                _handles[i] = handle;
+                _handles[i++] = handle;
             }
 
             Address = address;
@@ -48,6 +101,11 @@ namespace Ryujinx.Memory.Tracking
             }
         }
 
+        public IEnumerable<RegionHandle> GetHandles()
+        {
+            return _handles;
+        }
+
         public void SignalWrite()
         {
             Dirty = true;