Remove pool cache entries for incompatible overlapping textures (#2568)

This greatly reduces memory usage in games that aggressively reuse memory without removing dead textures from the pool, such as the Xenoblade games, UE3 games, and to a lesser extent, UE4/unity games.

This change stops memory usage from ballooning in xenoblade and some other games. It will also reduce texture view/dependency complexity in some games - for example in MK8D it will reduce the number of surface copies between lighting cubemaps generated for actors.

There shouldn't be any performance impact from doing this, though the deletion and creation of textures could be improved by improving the OpenGL texture storage cache, which is very simple and limited right now. This will be improved in future.

Another potential error has been fixed with the texture cache, which could prevent data loss when data is interchangably written to textures from both the GPU and CPU. It was possible that the dirty flag for a texture would be consumed without the data being synchronized on next use, due to the old overlap check. This check no longer consumes the dirty flag.

Please test a bunch of games to make sure they still work, and there are no performance regressions.

Ryujinx.Graphics.Gpu/Image/AutoDeleteCache.cs

@@ -43,7 +43,7 @@ namespace Ryujinx.Graphics.Gpu.Image
 
                 oldestTexture.SynchronizeMemory();
 
-                if (oldestTexture.IsModified && !oldestTexture.ConsumeModified())
+                if (oldestTexture.IsModified && !oldestTexture.CheckModified(true))
                 {
                     // The texture must be flushed if it falls out of the auto delete cache.
                     // Flushes out of the auto delete cache do not trigger write tracking,

Ryujinx.Graphics.Gpu/Image/Texture.cs

@@ -252,7 +252,7 @@ namespace Ryujinx.Graphics.Gpu.Image
                 if (!isView)
                 {
                     // Don't update this texture the next time we synchronize.
-                    ConsumeModified();
+                    CheckModified(true);
 
                     if (ScaleMode == TextureScaleMode.Scaled)
                     {
@@ -599,12 +599,13 @@ namespace Ryujinx.Graphics.Gpu.Image
 
         /// <summary>
         /// Checks if the memory for this texture was modified, and returns true if it was.
-        /// The modified flags are consumed as a result.
+        /// The modified flags are optionally consumed as a result.
         /// </summary>
+        /// <param name="consume">True to consume the dirty flags and reprotect, false to leave them as is</param>
         /// <returns>True if the texture was modified, false otherwise.</returns>
-        public bool ConsumeModified()
+        public bool CheckModified(bool consume)
         {
-            return Group.ConsumeDirty(this);
+            return Group.CheckDirty(this, consume);
         }
 
         /// <summary>
@@ -634,7 +635,7 @@ namespace Ryujinx.Graphics.Gpu.Image
             }
             else
             {
-                Group.ConsumeDirty(this);
+                Group.CheckDirty(this, true);
                 SynchronizeFull();
             }
         }
@@ -698,7 +699,7 @@ namespace Ryujinx.Graphics.Gpu.Image
         {
             BlacklistScale();
 
-            Group.ConsumeDirty(this);
+            Group.CheckDirty(this, true);
 
             IsModified = false;
 

Ryujinx.Graphics.Gpu/Image/TextureCache.cs

@@ -623,29 +623,49 @@ namespace Ryujinx.Graphics.Gpu.Image
                         hasLayerViews |= overlap.Info.GetSlices() < texture.Info.GetSlices();
                         hasMipViews |= overlap.Info.Levels < texture.Info.Levels;
                     }
-                    else if (overlapInCache || !setData)
+                    else
                     {
-                        if (info.GobBlocksInZ > 1 && info.GobBlocksInZ == overlap.Info.GobBlocksInZ)
+                        bool removeOverlap;
+                        bool modified = overlap.CheckModified(false);
+
+                        if (overlapInCache || !setData)
                         {
-                            // Allow overlapping slices of 3D textures. Could be improved in future by making sure the textures don't overlap.
-                            continue;
-                        }
+                            if (info.GobBlocksInZ > 1 && info.GobBlocksInZ == overlap.Info.GobBlocksInZ)
+                            {
+                                // Allow overlapping slices of 3D textures. Could be improved in future by making sure the textures don't overlap.
+                                continue;
+                            }
+
+                            // The overlap texture is going to contain garbage data after we draw, or is generally incompatible.
+                            // If the texture cannot be entirely contained in the new address space, and one of its view children is compatible with us,
+                            // it must be flushed before removal, so that the data is not lost.
+
+                            // If the texture was modified since its last use, then that data is probably meant to go into this texture.
+                            // If the data has been modified by the CPU, then it also shouldn't be flushed.
 
-                        // The overlap texture is going to contain garbage data after we draw, or is generally incompatible.
-                        // If the texture cannot be entirely contained in the new address space, and one of its view children is compatible with us,
-                        // it must be flushed before removal, so that the data is not lost.
+                            bool viewCompatibleChild = overlap.HasViewCompatibleChild(texture);
 
-                        // If the texture was modified since its last use, then that data is probably meant to go into this texture.
-                        // If the data has been modified by the CPU, then it also shouldn't be flushed.
-                        bool modified = overlap.ConsumeModified();
+                            bool flush = overlapInCache && !modified && !texture.Range.Contains(overlap.Range) && viewCompatibleChild;
 
-                        bool flush = overlapInCache && !modified && !texture.Range.Contains(overlap.Range) && overlap.HasViewCompatibleChild(texture);
+                            setData |= modified || flush;
 
-                        setData |= modified || flush;
+                            if (overlapInCache)
+                            {
+                                _cache.Remove(overlap, flush);
+                            }
+
+                            removeOverlap = modified && !viewCompatibleChild;
+                        }
+                        else
+                        {
+                            // If an incompatible overlapping texture has been modified, then it's data is likely destined for this texture,
+                            // and the overlapped texture will contain garbage. In this case, it should be removed to save memory.
+                            removeOverlap = modified;
+                        }
 
-                        if (overlapInCache)
+                        if (removeOverlap && overlap.Info.Target != Target.TextureBuffer)
                         {
-                            _cache.Remove(overlap, flush);
+                            overlap.RemoveFromPools(false);
                         }
                     }
                 }

Ryujinx.Graphics.Gpu/Image/TextureGroup.cs

@@ -83,11 +83,13 @@ namespace Ryujinx.Graphics.Gpu.Image
         }
 
         /// <summary>
-        /// Consume the dirty flags for a given texture. The state is shared between views of the same layers and levels.
+        /// Check and optionally consume the dirty flags for a given texture.
+        /// The state is shared between views of the same layers and levels.
         /// </summary>
         /// <param name="texture">The texture being used</param>
+        /// <param name="consume">True to consume the dirty flags and reprotect, false to leave them as is</param>
         /// <returns>True if a flag was dirty, false otherwise</returns>
-        public bool ConsumeDirty(Texture texture)
+        public bool CheckDirty(Texture texture, bool consume)
         {
             bool dirty = false;
 
@@ -101,7 +103,11 @@ namespace Ryujinx.Graphics.Gpu.Image
                     {
                         if (handle.Dirty)
                         {
-                            handle.Reprotect();
+                            if (consume)
+                            {
+                                handle.Reprotect();
+                            }
+
                             dirty = true;
                         }
                     }