Move gl_Layer to vertex shader if geometry is not supported (#4368)

* Set gl_Layer on vertex shader if it's set on the geometry shader and it does nothing else

* Shader cache version bump

* PR feedback

* Fix typo

Ryujinx.Graphics.GAL/Capabilities.cs

@@ -26,6 +26,7 @@ namespace Ryujinx.Graphics.GAL
         public readonly bool SupportsBlendEquationAdvanced;
         public readonly bool SupportsFragmentShaderInterlock;
         public readonly bool SupportsFragmentShaderOrderingIntel;
+        public readonly bool SupportsGeometryShader;
         public readonly bool SupportsGeometryShaderPassthrough;
         public readonly bool SupportsImageLoadFormatted;
         public readonly bool SupportsLayerVertexTessellation;
@@ -68,6 +69,7 @@ namespace Ryujinx.Graphics.GAL
             bool supportsBlendEquationAdvanced,
             bool supportsFragmentShaderInterlock,
             bool supportsFragmentShaderOrderingIntel,
+            bool supportsGeometryShader,
             bool supportsGeometryShaderPassthrough,
             bool supportsImageLoadFormatted,
             bool supportsLayerVertexTessellation,
@@ -107,6 +109,7 @@ namespace Ryujinx.Graphics.GAL
             SupportsBlendEquationAdvanced = supportsBlendEquationAdvanced;
             SupportsFragmentShaderInterlock = supportsFragmentShaderInterlock;
             SupportsFragmentShaderOrderingIntel = supportsFragmentShaderOrderingIntel;
+            SupportsGeometryShader = supportsGeometryShader;
             SupportsGeometryShaderPassthrough = supportsGeometryShaderPassthrough;
             SupportsImageLoadFormatted = supportsImageLoadFormatted;
             SupportsLayerVertexTessellation = supportsLayerVertexTessellation;

Ryujinx.Graphics.Gpu/Shader/DiskCache/DiskCacheHostStorage.cs

@@ -22,7 +22,7 @@ namespace Ryujinx.Graphics.Gpu.Shader.DiskCache
         private const ushort FileFormatVersionMajor = 1;
         private const ushort FileFormatVersionMinor = 2;
         private const uint FileFormatVersionPacked = ((uint)FileFormatVersionMajor << 16) | FileFormatVersionMinor;
-        private const uint CodeGenVersion = 4369;
+        private const uint CodeGenVersion = 4368;
 
         private const string SharedTocFileName = "shared.toc";
         private const string SharedDataFileName = "shared.data";
@@ -774,6 +774,8 @@ namespace Ryujinx.Graphics.Gpu.Shader.DiskCache
                 sBuffers,
                 textures,
                 images,
+                ShaderIdentification.None,
+                0,
                 dataInfo.Stage,
                 dataInfo.UsesInstanceId,
                 dataInfo.UsesDrawParameters,

Ryujinx.Graphics.Gpu/Shader/DiskCache/ParallelDiskCacheLoader.cs

@@ -633,6 +633,11 @@ namespace Ryujinx.Graphics.Gpu.Shader.DiskCache
                 }
             }
 
+            if (!_context.Capabilities.SupportsGeometryShader)
+            {
+                ShaderCache.TryRemoveGeometryStage(translatorContexts);
+            }
+
             CachedShaderStage[] shaders = new CachedShaderStage[guestShaders.Length];
             List<ShaderProgram> translatedStages = new List<ShaderProgram>();
 

Ryujinx.Graphics.Gpu/Shader/GpuAccessorBase.cs

@@ -126,6 +126,8 @@ namespace Ryujinx.Graphics.Gpu.Shader
 
         public bool QueryHostSupportsFragmentShaderOrderingIntel() => _context.Capabilities.SupportsFragmentShaderOrderingIntel;
 
+        public bool QueryHostSupportsGeometryShader() => _context.Capabilities.SupportsGeometryShader;
+
         public bool QueryHostSupportsGeometryShaderPassthrough() => _context.Capabilities.SupportsGeometryShaderPassthrough;
 
         public bool QueryHostSupportsImageLoadFormatted() => _context.Capabilities.SupportsImageLoadFormatted;

Ryujinx.Graphics.Gpu/Shader/ShaderCache.cs

@@ -353,6 +353,11 @@ namespace Ryujinx.Graphics.Gpu.Shader
                 }
             }
 
+            if (!_context.Capabilities.SupportsGeometryShader)
+            {
+                TryRemoveGeometryStage(translatorContexts);
+            }
+
             CachedShaderStage[] shaders = new CachedShaderStage[Constants.ShaderStages + 1];
             List<ShaderSource> shaderSources = new List<ShaderSource>();
 
@@ -421,6 +426,39 @@ namespace Ryujinx.Graphics.Gpu.Shader
             return gpShaders;
         }
 
+        /// <summary>
+        /// Tries to eliminate the geometry stage from the array of translator contexts.
+        /// </summary>
+        /// <param name="translatorContexts">Array of translator contexts</param>
+        public static void TryRemoveGeometryStage(TranslatorContext[] translatorContexts)
+        {
+            if (translatorContexts[4] != null)
+            {
+                // We have a geometry shader, but geometry shaders are not supported.
+                // Try to eliminate the geometry shader.
+
+                ShaderProgramInfo info = translatorContexts[4].Translate().Info;
+
+                if (info.Identification == ShaderIdentification.GeometryLayerPassthrough)
+                {
+                    // We managed to identify that this geometry shader is only used to set the output Layer value,
+                    // we can set the Layer on the previous stage instead (usually the vertex stage) and eliminate it.
+
+                    for (int i = 3; i >= 1; i--)
+                    {
+                        if (translatorContexts[i] != null)
+                        {
+                            translatorContexts[i].SetGeometryShaderLayerInputAttribute(info.GpLayerInputAttribute);
+                            translatorContexts[i].SetLastInVertexPipeline(translatorContexts[5] != null);
+                            break;
+                        }
+                    }
+
+                    translatorContexts[4] = null;
+                }
+            }
+        }
+
         /// <summary>
         /// Creates a shader source for use with the backend from a translated shader program.
         /// </summary>

Ryujinx.Graphics.OpenGL/OpenGLRenderer.cs

@@ -124,6 +124,7 @@ namespace Ryujinx.Graphics.OpenGL
                 supportsBlendEquationAdvanced: HwCapabilities.SupportsBlendEquationAdvanced,
                 supportsFragmentShaderInterlock: HwCapabilities.SupportsFragmentShaderInterlock,
                 supportsFragmentShaderOrderingIntel: HwCapabilities.SupportsFragmentShaderOrdering,
+                supportsGeometryShader: true,
                 supportsGeometryShaderPassthrough: HwCapabilities.SupportsGeometryShaderPassthrough,
                 supportsImageLoadFormatted: HwCapabilities.SupportsImageLoadFormatted,
                 supportsLayerVertexTessellation: HwCapabilities.SupportsShaderViewportLayerArray,

Ryujinx.Graphics.Shader/IGpuAccessor.cs

@@ -259,6 +259,15 @@ namespace Ryujinx.Graphics.Shader
             return false;
         }
 
+        /// <summary>
+        /// Queries host GPU geometry shader support.
+        /// </summary>
+        /// <returns>True if the GPU and driver supports geometry shaders, false otherwise</returns>
+        bool QueryHostSupportsGeometryShader()
+        {
+            return true;
+        }
+
         /// <summary>
         /// Queries host GPU geometry shader passthrough support.
         /// </summary>

Ryujinx.Graphics.Shader/ShaderIdentification.cs

@@ -0,0 +1,8 @@
+namespace Ryujinx.Graphics.Shader
+{
+    public enum ShaderIdentification
+    {
+        None,
+        GeometryLayerPassthrough
+    }
+}

Ryujinx.Graphics.Shader/ShaderProgramInfo.cs

@@ -10,6 +10,8 @@ namespace Ryujinx.Graphics.Shader
         public ReadOnlyCollection<TextureDescriptor> Textures { get; }
         public ReadOnlyCollection<TextureDescriptor> Images { get; }
 
+        public ShaderIdentification Identification { get; }
+        public int GpLayerInputAttribute { get; }
         public ShaderStage Stage { get; }
         public bool UsesInstanceId { get; }
         public bool UsesDrawParameters { get; }
@@ -22,6 +24,8 @@ namespace Ryujinx.Graphics.Shader
             BufferDescriptor[] sBuffers,
             TextureDescriptor[] textures,
             TextureDescriptor[] images,
+            ShaderIdentification identification,
+            int gpLayerInputAttribute,
             ShaderStage stage,
             bool usesInstanceId,
             bool usesDrawParameters,
@@ -34,6 +38,8 @@ namespace Ryujinx.Graphics.Shader
             Textures = Array.AsReadOnly(textures);
             Images = Array.AsReadOnly(images);
 
+            Identification = identification;
+            GpLayerInputAttribute = gpLayerInputAttribute;
             Stage = stage;
             UsesInstanceId = usesInstanceId;
             UsesDrawParameters = usesDrawParameters;

Ryujinx.Graphics.Shader/Translation/EmitterContext.cs

@@ -241,6 +241,13 @@ namespace Ryujinx.Graphics.Shader.Translation
 
                 this.Copy(Attribute(AttributeConsts.PositionZ), this.FPFusedMultiplyAdd(z, ConstF(0.5f), halfW));
             }
+
+            if (Config.Stage != ShaderStage.Geometry && Config.HasLayerInputAttribute)
+            {
+                Config.SetUsedFeature(FeatureFlags.RtLayer);
+
+                this.Copy(Attribute(AttributeConsts.Layer), Attribute(Config.GpLayerInputAttribute | AttributeConsts.LoadOutputMask));
+            }
         }
 
         public void PrepareForVertexReturn(out Operand oldXLocal, out Operand oldYLocal, out Operand oldZLocal)

Ryujinx.Graphics.Shader/Translation/ShaderConfig.cs

@@ -20,6 +20,8 @@ namespace Ryujinx.Graphics.Shader.Translation
         public bool LastInPipeline { get; private set; }
         public bool LastInVertexPipeline { get; private set; }
 
+        public bool HasLayerInputAttribute { get; private set; }
+        public int GpLayerInputAttribute { get; private set; }
         public int ThreadsPerInputPrimitive { get; }
 
         public OutputTopology OutputTopology { get; }
@@ -245,6 +247,22 @@ namespace Ryujinx.Graphics.Shader.Translation
             LayerOutputAttribute = attr;
         }
 
+        public void SetGeometryShaderLayerInputAttribute(int attr)
+        {
+            HasLayerInputAttribute = true;
+            GpLayerInputAttribute = attr;
+        }
+
+        public void SetLastInVertexPipeline(bool hasFragment)
+        {
+            if (!hasFragment)
+            {
+                LastInPipeline = true;
+            }
+
+            LastInVertexPipeline = true;
+        }
+
         public void SetInputUserAttributeFixedFunc(int index)
         {
             UsedInputAttributes |= 1 << index;
@@ -706,13 +724,15 @@ namespace Ryujinx.Graphics.Shader.Translation
             return FindDescriptorIndex(GetImageDescriptors(), texOp);
         }
 
-        public ShaderProgramInfo CreateProgramInfo()
+        public ShaderProgramInfo CreateProgramInfo(ShaderIdentification identification = ShaderIdentification.None)
         {
             return new ShaderProgramInfo(
                 GetConstantBufferDescriptors(),
                 GetStorageBufferDescriptors(),
                 GetTextureDescriptors(),
                 GetImageDescriptors(),
+                identification,
+                GpLayerInputAttribute,
                 Stage,
                 UsedFeatures.HasFlag(FeatureFlags.InstanceId),
                 UsedFeatures.HasFlag(FeatureFlags.DrawParameters),

Ryujinx.Graphics.Shader/Translation/ShaderIdentifier.cs

@@ -0,0 +1,145 @@
+using Ryujinx.Graphics.Shader.IntermediateRepresentation;
+using static Ryujinx.Graphics.Shader.IntermediateRepresentation.OperandHelper;
+
+namespace Ryujinx.Graphics.Shader.Translation
+{
+    static class ShaderIdentifier
+    {
+        public static ShaderIdentification Identify(Function[] functions, ShaderConfig config)
+        {
+            if (config.Stage == ShaderStage.Geometry &&
+                config.GpuAccessor.QueryPrimitiveTopology() == InputTopology.Triangles &&
+                !config.GpuAccessor.QueryHostSupportsGeometryShader() &&
+                IsLayerPassthroughGeometryShader(functions, out int layerInputAttr))
+            {
+                config.SetGeometryShaderLayerInputAttribute(layerInputAttr);
+
+                return ShaderIdentification.GeometryLayerPassthrough;
+            }
+
+            return ShaderIdentification.None;
+        }
+
+        private static bool IsLayerPassthroughGeometryShader(Function[] functions, out int layerInputAttr)
+        {
+            bool writesLayer = false;
+            layerInputAttr = 0;
+
+            if (functions.Length != 1)
+            {
+                return false;
+            }
+
+            int verticesCount = 0;
+            int totalVerticesCount = 0;
+
+            foreach (BasicBlock block in functions[0].Blocks)
+            {
+                // We are not expecting loops or any complex control flow here, so fail in those cases.
+                if (block.Branch != null && block.Branch.Index <= block.Index)
+                {
+                    return false;
+                }
+
+                foreach (INode node in block.Operations)
+                {
+                    if (!(node is Operation operation))
+                    {
+                        continue;
+                    }
+
+                    if (IsResourceWrite(operation.Inst))
+                    {
+                        return false;
+                    }
+
+                    if (operation.Inst == Instruction.StoreAttribute)
+                    {
+                        return false;
+                    }
+
+                    if (operation.Inst == Instruction.Copy && operation.Dest.Type == OperandType.Attribute)
+                    {
+                        Operand src = operation.GetSource(0);
+
+                        if (src.Type == OperandType.LocalVariable && src.AsgOp is Operation asgOp && asgOp.Inst == Instruction.LoadAttribute)
+                        {
+                            src = Attribute(asgOp.GetSource(0).Value);
+                        }
+
+                        if (src.Type == OperandType.Attribute)
+                        {
+                            if (operation.Dest.Value == AttributeConsts.Layer)
+                            {
+                                if ((src.Value & AttributeConsts.LoadOutputMask) != 0)
+                                {
+                                    return false;
+                                }
+
+                                writesLayer = true;
+                                layerInputAttr = src.Value;
+                            }
+                            else if (src.Value != operation.Dest.Value)
+                            {
+                                return false;
+                            }
+                        }
+                        else if (src.Type == OperandType.Constant)
+                        {
+                            int dstComponent = (operation.Dest.Value >> 2) & 3;
+                            float expectedValue = dstComponent == 3 ? 1f : 0f;
+
+                            if (src.AsFloat() != expectedValue)
+                            {
+                                return false;
+                            }
+                        }
+                        else
+                        {
+                            return false;
+                        }
+                    }
+                    else if (operation.Inst == Instruction.EmitVertex)
+                    {
+                        verticesCount++;
+                    }
+                    else if (operation.Inst == Instruction.EndPrimitive)
+                    {
+                        totalVerticesCount += verticesCount;
+                        verticesCount = 0;
+                    }
+                }
+            }
+
+            return totalVerticesCount + verticesCount == 3 && writesLayer;
+        }
+
+        private static bool IsResourceWrite(Instruction inst)
+        {
+            switch (inst)
+            {
+                case Instruction.AtomicAdd:
+                case Instruction.AtomicAnd:
+                case Instruction.AtomicCompareAndSwap:
+                case Instruction.AtomicMaxS32:
+                case Instruction.AtomicMaxU32:
+                case Instruction.AtomicMinS32:
+                case Instruction.AtomicMinU32:
+                case Instruction.AtomicOr:
+                case Instruction.AtomicSwap:
+                case Instruction.AtomicXor:
+                case Instruction.ImageAtomic:
+                case Instruction.ImageStore:
+                case Instruction.StoreGlobal:
+                case Instruction.StoreGlobal16:
+                case Instruction.StoreGlobal8:
+                case Instruction.StoreStorage:
+                case Instruction.StoreStorage16:
+                case Instruction.StoreStorage8:
+                    return true;
+            }
+
+            return false;
+        }
+    }
+}

Ryujinx.Graphics.Shader/Translation/Translator.cs

@@ -77,9 +77,11 @@ namespace Ryujinx.Graphics.Shader.Translation
                 funcs[i] = new Function(cfg.Blocks, $"fun{i}", false, inArgumentsCount, outArgumentsCount);
             }
 
+            var identification = ShaderIdentifier.Identify(funcs, config);
+
             var sInfo = StructuredProgram.MakeStructuredProgram(funcs, config);
 
-            var info = config.CreateProgramInfo();
+            var info = config.CreateProgramInfo(identification);
 
             return config.Options.TargetLanguage switch
             {

Ryujinx.Graphics.Shader/Translation/TranslatorContext.cs

@@ -138,6 +138,16 @@ namespace Ryujinx.Graphics.Shader.Translation
             _config.MergeFromtNextStage(nextStage._config);
         }
 
+        public void SetGeometryShaderLayerInputAttribute(int attr)
+        {
+            _config.SetGeometryShaderLayerInputAttribute(attr);
+        }
+
+        public void SetLastInVertexPipeline(bool hasFragment)
+        {
+            _config.SetLastInVertexPipeline(hasFragment);
+        }
+
         public ShaderProgram Translate(TranslatorContext other = null)
         {
             FunctionCode[] code = EmitShader(_program, _config, initializeOutputs: other == null, out _);

Ryujinx.Graphics.Vulkan/VulkanRenderer.cs

@@ -546,6 +546,7 @@ namespace Ryujinx.Graphics.Vulkan
                 supportsBlendEquationAdvanced: Capabilities.SupportsBlendEquationAdvanced,
                 supportsFragmentShaderInterlock: Capabilities.SupportsFragmentShaderInterlock,
                 supportsFragmentShaderOrderingIntel: false,
+                supportsGeometryShader: Capabilities.SupportsGeometryShader,
                 supportsGeometryShaderPassthrough: Capabilities.SupportsGeometryShaderPassthrough,
                 supportsImageLoadFormatted: features2.Features.ShaderStorageImageReadWithoutFormat,
                 supportsLayerVertexTessellation: featuresVk12.ShaderOutputLayer,