Surface Flinger: Fix an oversight when closing a layer (#2192)

* Surface Flinger: Fix an oversight when closing a layer

As the title say.
I also took the liberty of changing the logic on how we select the
current layer being rendered to make it more explicit when opening and
creating layers.

NOTE: Found by Ac_k.

* check for RenderLayerId and not the dictionary size

This fix a possible race condition between the time you create a layer and set the one currently used for rendering

Ryujinx.HLE/HOS/Services/Am/AppletAE/AllSystemAppletProxiesService/SystemAppletProxy/ISelfController.cs

@@ -218,6 +218,7 @@ namespace Ryujinx.HLE.HOS.Services.Am.AppletAE.AllSystemAppletProxiesService.Sys
         public ResultCode CreateManagedDisplayLayer(ServiceCtx context)
         {
             context.Device.System.SurfaceFlinger.CreateLayer(_pid, out long layerId);
+            context.Device.System.SurfaceFlinger.SetRenderLayer(layerId);
 
             context.ResponseData.Write(layerId);
 
@@ -228,9 +229,9 @@ namespace Ryujinx.HLE.HOS.Services.Am.AppletAE.AllSystemAppletProxiesService.Sys
         // CreateManagedDisplaySeparableLayer() -> (u64, u64)
         public ResultCode CreateManagedDisplaySeparableLayer(ServiceCtx context)
         {
-            // NOTE: first create the recoding layer and then the display one because right now Surface Flinger only use the last id.
-            context.Device.System.SurfaceFlinger.CreateLayer(_pid, out long recordingLayerId);
             context.Device.System.SurfaceFlinger.CreateLayer(_pid, out long displayLayerId);
+            context.Device.System.SurfaceFlinger.CreateLayer(_pid, out long recordingLayerId);
+            context.Device.System.SurfaceFlinger.SetRenderLayer(displayLayerId);
 
             context.ResponseData.Write(displayLayerId);
             context.ResponseData.Write(recordingLayerId);

Ryujinx.HLE/HOS/Services/SurfaceFlinger/SurfaceFlinger.cs

@@ -7,6 +7,7 @@ using Ryujinx.HLE.HOS.Services.Nv.NvDrvServices.NvMap;
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
+using System.Linq;
 using System.Threading;
 
 namespace Ryujinx.HLE.HOS.Services.SurfaceFlinger
@@ -36,7 +37,7 @@ namespace Ryujinx.HLE.HOS.Services.SurfaceFlinger
 
         private readonly object Lock = new object();
 
-        public long LastId { get; private set; }
+        public long RenderLayerId { get; private set; }
 
         private class Layer
         {
@@ -57,7 +58,7 @@ namespace Ryujinx.HLE.HOS.Services.SurfaceFlinger
         {
             _device = device;
             _layers = new Dictionary<long, Layer>();
-            LastId  = 0;
+            RenderLayerId = 0;
 
             _composerThread = new Thread(HandleComposition)
             {
@@ -150,8 +151,6 @@ namespace Ryujinx.HLE.HOS.Services.SurfaceFlinger
                     Core             = core,
                     Owner            = pid
                 });
-
-                LastId = layerId;
             }
         }
 
@@ -166,7 +165,31 @@ namespace Ryujinx.HLE.HOS.Services.SurfaceFlinger
                     HOSBinderDriverServer.UnregisterBinderObject(layer.ProducerBinderId);
                 }
 
-                return _layers.Remove(layerId);
+                bool removed = _layers.Remove(layerId);
+
+                // If the layer was removed and the current in use, we need to change the current layer in use.
+                if (removed && RenderLayerId == layerId)
+                {
+                    // If no layer is availaible, reset to default value.
+                    if (_layers.Count == 0)
+                    {
+                        SetRenderLayer(0);
+                    }
+                    else
+                    {
+                        SetRenderLayer(_layers.Last().Key);
+                    }
+                }
+
+                return removed;
+            }
+        }
+
+        public void SetRenderLayer(long layerId)
+        {
+            lock (Lock)
+            {
+                RenderLayerId = layerId;
             }
         }
 
@@ -263,12 +286,12 @@ namespace Ryujinx.HLE.HOS.Services.SurfaceFlinger
             lock (Lock)
             {
                 // TODO: support multilayers (& multidisplay ?)
-                if (_layers.Count == 0)
+                if (RenderLayerId == 0)
                 {
                     return;
                 }
 
-                Layer layer = GetLayerByIdLocked(LastId);
+                Layer layer = GetLayerByIdLocked(RenderLayerId);
 
                 Status acquireStatus = layer.Consumer.AcquireBuffer(out BufferItem item, 0);
 

Ryujinx.HLE/HOS/Services/Vi/RootService/ApplicationDisplayService/IManagerDisplayService.cs

@@ -22,6 +22,7 @@ namespace Ryujinx.HLE.HOS.Services.Vi.RootService.ApplicationDisplayService
             long pid = context.Device.System.AppletState.AppletResourceUserIds.GetData<long>((int)appletResourceUserId);
 
             context.Device.System.SurfaceFlinger.CreateLayer(pid, out long layerId);
+            context.Device.System.SurfaceFlinger.SetRenderLayer(layerId);
 
             context.ResponseData.Write(layerId);
 

Ryujinx.HLE/HOS/Services/Vi/RootService/IApplicationDisplayService.cs

@@ -126,6 +126,8 @@ namespace Ryujinx.HLE.HOS.Services.Vi.RootService
 
             IBinder producer = context.Device.System.SurfaceFlinger.OpenLayer(context.Request.HandleDesc.PId, layerId);
 
+            context.Device.System.SurfaceFlinger.SetRenderLayer(layerId);
+
             Parcel parcel = new Parcel(0x28, 0x4);
 
             parcel.WriteObject(producer, "dispdrv\0");
@@ -164,6 +166,8 @@ namespace Ryujinx.HLE.HOS.Services.Vi.RootService
 
             IBinder producer = context.Device.System.SurfaceFlinger.CreateLayer(0, out long layerId);
 
+            context.Device.System.SurfaceFlinger.SetRenderLayer(layerId);
+
             Parcel parcel = new Parcel(0x28, 0x4);
 
             parcel.WriteObject(producer, "dispdrv\0");