Allow _volatile to be set from MultiRegionHandle checks again (#3830)

* Allow _volatile to be set from MultiRegionHandle checks again

Tracking handles have a `_volatile` flag which indicates that the resource being tracked is modified every time it is used under a new sequence number. This is used to reduce the time spent reprotecting memory for tracking writes to commonly modified buffers, like constant buffers.

This optimisation works by detecting if a buffer is modified every time a check happens. If a buffer is checked but it is not dirty, then that data is likely not modified every sequence number, and should use memory protection for write tracking. If the opposite is the case all the time, it is faster to just assume it's dirty as we'd just be wasting time protecting the memory.

The new MultiRegionBitmap could not notify handles that they had been checked as part of the fast bitmap lookup, so bindings larger than 4096 bytes wouldn't trigger it at all. This meant that they would be subject to a ton of reprotection if they were modified often.

This does mean there are two separate sources for a _volatile set: VolatileOrDirty + _checkCount, and the bitmap check. These shouldn't interfere with each other, though.

This fixes performance regressions from #3775 in Pokemon Sword, and hopefully Yu-Gi-Oh! RUSH DUEL: Dawn of the Battle Royale. May affect other games.

* Fix stupid mistake

Ryujinx.Memory/Tracking/MultiRegionHandle.cs

@@ -25,6 +25,7 @@ namespace Ryujinx.Memory.Tracking
 
         private int _sequenceNumber;
         private BitMap _sequenceNumberBitmap;
+        private BitMap _dirtyCheckedBitmap;
         private int _uncheckedHandles;
 
         public bool Dirty { get; private set; } = true;
@@ -36,6 +37,7 @@ namespace Ryujinx.Memory.Tracking
 
             _dirtyBitmap = new ConcurrentBitmap(_handles.Length, true);
             _sequenceNumberBitmap = new BitMap(_handles.Length);
+            _dirtyCheckedBitmap = new BitMap(_handles.Length);
 
             int i = 0;
 
@@ -246,16 +248,18 @@ namespace Ryujinx.Memory.Tracking
         }
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        private void ParseDirtyBits(long dirtyBits, long mask, int index, long[] seqMasks, ref int baseBit, ref int prevHandle, ref ulong rgStart, ref ulong rgSize, Action<ulong, ulong> modifiedAction)
+        private void ParseDirtyBits(long dirtyBits, long mask, int index, long[] seqMasks, long[] checkMasks, ref int baseBit, ref int prevHandle, ref ulong rgStart, ref ulong rgSize, Action<ulong, ulong> modifiedAction)
         {
             long seqMask = mask & ~seqMasks[index];
+            long checkMask = (~dirtyBits) & seqMask;
             dirtyBits &= seqMask;
 
             while (dirtyBits != 0)
             {
                 int bit = BitOperations.TrailingZeroCount(dirtyBits);
+                long bitValue = 1L << bit;
 
-                dirtyBits &= ~(1L << bit);
+                dirtyBits &= ~bitValue;
 
                 int handleIndex = baseBit + bit;
 
@@ -273,11 +277,14 @@ namespace Ryujinx.Memory.Tracking
                 }
 
                 rgSize += handle.Size;
-                handle.Reprotect();
+                handle.Reprotect(false, (checkMasks[index] & bitValue) == 0);
+
+                checkMasks[index] &= ~bitValue;
 
                 prevHandle = handleIndex;
             }
 
+            checkMasks[index] |= checkMask;
             seqMasks[index] |= mask;
             _uncheckedHandles -= BitOperations.PopCount((ulong)seqMask);
 
@@ -328,6 +335,7 @@ namespace Ryujinx.Memory.Tracking
             ulong rgSize = 0;
 
             long[] seqMasks = _sequenceNumberBitmap.Masks;
+            long[] checkedMasks = _dirtyCheckedBitmap.Masks;
             long[] masks = _dirtyBitmap.Masks;
 
             int startIndex = startHandle >> ConcurrentBitmap.IntShift;
@@ -345,20 +353,20 @@ namespace Ryujinx.Memory.Tracking
 
             if (startIndex == endIndex)
             {
-                ParseDirtyBits(startValue, startMask & endMask, startIndex, seqMasks, ref baseBit, ref prevHandle, ref rgStart, ref rgSize, modifiedAction);
+                ParseDirtyBits(startValue, startMask & endMask, startIndex, seqMasks, checkedMasks, ref baseBit, ref prevHandle, ref rgStart, ref rgSize, modifiedAction);
             }
             else
             {
-                ParseDirtyBits(startValue, startMask, startIndex, seqMasks, ref baseBit, ref prevHandle, ref rgStart, ref rgSize, modifiedAction);
+                ParseDirtyBits(startValue, startMask, startIndex, seqMasks, checkedMasks, ref baseBit, ref prevHandle, ref rgStart, ref rgSize, modifiedAction);
 
                 for (int i = startIndex + 1; i < endIndex; i++)
                 {
-                    ParseDirtyBits(Volatile.Read(ref masks[i]), -1L, i, seqMasks, ref baseBit, ref prevHandle, ref rgStart, ref rgSize, modifiedAction);
+                    ParseDirtyBits(Volatile.Read(ref masks[i]), -1L, i, seqMasks, checkedMasks, ref baseBit, ref prevHandle, ref rgStart, ref rgSize, modifiedAction);
                 }
 
                 long endValue = Volatile.Read(ref masks[endIndex]);
 
-                ParseDirtyBits(endValue, endMask, endIndex, seqMasks, ref baseBit, ref prevHandle, ref rgStart, ref rgSize, modifiedAction);
+                ParseDirtyBits(endValue, endMask, endIndex, seqMasks, checkedMasks, ref baseBit, ref prevHandle, ref rgStart, ref rgSize, modifiedAction);
             }
 
             if (rgSize != 0)

Ryujinx.Memory/Tracking/RegionHandle.cs

@@ -263,15 +263,15 @@ namespace Ryujinx.Memory.Tracking
         /// </summary>
         public void ForceDirty()
         {
-            _checkCount++;
-
             Dirty = true;
         }
 
         /// <summary>
         /// Consume the dirty flag for this handle, and reprotect so it can be set on the next write.
         /// </summary>
-        public void Reprotect(bool asDirty = false)
+        /// <param name="asDirty">True if the handle should be reprotected as dirty, rather than have it cleared</param>
+        /// <param name="consecutiveCheck">True if this reprotect is the result of consecutive dirty checks</param>
+        public void Reprotect(bool asDirty, bool consecutiveCheck = false)
         {
             if (_volatile) return;
 
@@ -296,7 +296,7 @@ namespace Ryujinx.Memory.Tracking
             }
             else if (!asDirty)
             {
-                if (_checkCount > 0 && _checkCount < CheckCountForInfrequent)
+                if (consecutiveCheck || (_checkCount > 0 && _checkCount < CheckCountForInfrequent))
                 {
                     if (++_volatileCount >= VolatileThreshold && _preAction == null)
                     {
@@ -313,6 +313,15 @@ namespace Ryujinx.Memory.Tracking
             }
         }
 
+        /// <summary>
+        /// Consume the dirty flag for this handle, and reprotect so it can be set on the next write.
+        /// </summary>
+        /// <param name="asDirty">True if the handle should be reprotected as dirty, rather than have it cleared</param>
+        public void Reprotect(bool asDirty = false)
+        {
+            Reprotect(asDirty, false);
+        }
+
         /// <summary>
         /// Register an action to perform when the tracked region is read or written.
         /// The action is automatically removed after it runs.