Better viewport flipping and depth mode detection method (#1556)

* Use a better viewport flipping approach

* New approach to detect depth mode

* nit: Sort method on the OpenGL backend

* Adjust spacing on comment

* Unswap near and far parameters based on ScaleZ

Ryujinx.Graphics.GAL/IPipeline.cs

@@ -50,8 +50,6 @@ namespace Ryujinx.Graphics.GAL
 
         void SetLogicOpState(bool enable, LogicalOp op);
 
-        void SetOrigin(Origin origin);
-
         void SetPointParameters(float size, bool isProgramPointSize, bool enablePointSprite, Origin origin);
 
         void SetPrimitiveRestart(bool enable, int index);

Ryujinx.Graphics.Gpu/Engine/Methods.cs

@@ -488,25 +488,12 @@ namespace Ryujinx.Graphics.Gpu.Engine
         /// <param name="state">Current GPU state</param>
         private void UpdateViewportTransform(GpuState state)
         {
-            DepthMode depthMode = state.Get<DepthMode>(MethodOffset.DepthMode);
+            var yControl = state.Get<YControl> (MethodOffset.YControl);
+            var face     = state.Get<FaceState>(MethodOffset.FaceState);
 
-            _context.Renderer.Pipeline.SetDepthMode(depthMode);
+            UpdateFrontFace(yControl, face.FrontFace);
 
-            YControl yControl = state.Get<YControl>(MethodOffset.YControl);
-
-            bool   flipY  = yControl.HasFlag(YControl.NegateY);
-            Origin origin = yControl.HasFlag(YControl.TriangleRastFlip) ? Origin.LowerLeft : Origin.UpperLeft;
-
-            _context.Renderer.Pipeline.SetOrigin(origin);
-
-            // The triangle rast flip flag only affects rasterization, the viewport is not flipped.
-            // Setting the origin mode to upper left on the host, however, not only affects rasterization,
-            // but also flips the viewport.
-            // We negate the effects of flipping the viewport by flipping it again using the viewport swizzle.
-            if (origin == Origin.UpperLeft)
-            {
-                flipY = !flipY;
-            }
+            bool flipY = yControl.HasFlag(YControl.NegateY);
 
             Span<Viewport> viewports = stackalloc Viewport[Constants.TotalViewports];
 
@@ -515,11 +502,42 @@ namespace Ryujinx.Graphics.Gpu.Engine
                 var transform = state.Get<ViewportTransform>(MethodOffset.ViewportTransform, index);
                 var extents   = state.Get<ViewportExtents>  (MethodOffset.ViewportExtents,   index);
 
-                float x = transform.TranslateX - MathF.Abs(transform.ScaleX);
-                float y = transform.TranslateY - MathF.Abs(transform.ScaleY);
+                float scaleX = MathF.Abs(transform.ScaleX);
+                float scaleY = transform.ScaleY;
 
-                float width  = MathF.Abs(transform.ScaleX) * 2;
-                float height = MathF.Abs(transform.ScaleY) * 2;
+                if (flipY)
+                {
+                    scaleY = -scaleY;
+                }
+
+                if (!_context.Capabilities.SupportsViewportSwizzle && transform.UnpackSwizzleY() == ViewportSwizzle.NegativeY)
+                {
+                    scaleY = -scaleY;
+                }
+
+                if (index == 0)
+                {
+                    // Try to guess the depth mode being used on the high level API
+                    // based on current transform.
+                    // It is setup like so by said APIs:
+                    // If depth mode is ZeroToOne:
+                    //  TranslateZ = Near
+                    //  ScaleZ = Far - Near
+                    // If depth mode is MinusOneToOne:
+                    //  TranslateZ = (Near + Far) / 2
+                    //  ScaleZ = (Far - Near) / 2
+                    // DepthNear/Far are sorted such as that Near is always less than Far.
+                    DepthMode depthMode = extents.DepthNear != transform.TranslateZ &&
+                                          extents.DepthFar  != transform.TranslateZ ? DepthMode.MinusOneToOne : DepthMode.ZeroToOne;
+
+                    _context.Renderer.Pipeline.SetDepthMode(depthMode);
+                }
+
+                float x = transform.TranslateX - scaleX;
+                float y = transform.TranslateY - scaleY;
+
+                float width  = scaleX * 2;
+                float height = scaleY * 2;
 
                 float scale = TextureManager.RenderTargetScale;
                 if (scale != 1f)
@@ -537,34 +555,17 @@ namespace Ryujinx.Graphics.Gpu.Engine
                 ViewportSwizzle swizzleZ = transform.UnpackSwizzleZ();
                 ViewportSwizzle swizzleW = transform.UnpackSwizzleW();
 
-                if (transform.ScaleX < 0)
-                {
-                    swizzleX ^= ViewportSwizzle.NegativeFlag;
-                }
-
-                if (flipY)
-                {
-                    swizzleY ^= ViewportSwizzle.NegativeFlag;
-                }
-
-                if (transform.ScaleY < 0)
-                {
-                    swizzleY ^= ViewportSwizzle.NegativeFlag;
-                }
+                float depthNear = extents.DepthNear;
+                float depthFar  = extents.DepthFar;
 
                 if (transform.ScaleZ < 0)
                 {
-                    swizzleZ ^= ViewportSwizzle.NegativeFlag;
+                    float temp = depthNear;
+                    depthNear  = depthFar;
+                    depthFar   = temp;
                 }
 
-                viewports[index] = new Viewport(
-                    region,
-                    swizzleX,
-                    swizzleY,
-                    swizzleZ,
-                    swizzleW,
-                    extents.DepthNear,
-                    extents.DepthFar);
+                viewports[index] = new Viewport(region, swizzleX, swizzleY, swizzleZ, swizzleW, depthNear, depthFar);
             }
 
             _context.Renderer.Pipeline.SetViewports(0, viewports);
@@ -832,11 +833,29 @@ namespace Ryujinx.Graphics.Gpu.Engine
         /// <param name="state">Current GPU state</param>
         private void UpdateFaceState(GpuState state)
         {
-            var face = state.Get<FaceState>(MethodOffset.FaceState);
+            var yControl = state.Get<YControl> (MethodOffset.YControl);
+            var face     = state.Get<FaceState>(MethodOffset.FaceState);
 
             _context.Renderer.Pipeline.SetFaceCulling(face.CullEnable, face.CullFace);
 
-            _context.Renderer.Pipeline.SetFrontFace(face.FrontFace);
+            UpdateFrontFace(yControl, face.FrontFace);
+        }
+
+        /// <summary>
+        /// Updates the front face based on the current front face and the origin.
+        /// </summary>
+        /// <param name="yControl">Y control register value, where the origin is located</param>
+        /// <param name="frontFace">Front face</param>
+        private void UpdateFrontFace(YControl yControl, FrontFace frontFace)
+        {
+            bool isUpperLeftOrigin = !yControl.HasFlag(YControl.TriangleRastFlip);
+
+            if (isUpperLeftOrigin)
+            {
+                frontFace = frontFace == FrontFace.CounterClockwise ? FrontFace.Clockwise : FrontFace.CounterClockwise;
+            }
+
+            _context.Renderer.Pipeline.SetFrontFace(frontFace);
         }
 
         /// <summary>

Ryujinx.Graphics.Gpu/Shader/GpuAccessor.cs

@@ -188,12 +188,6 @@ namespace Ryujinx.Graphics.Gpu.Shader
         /// <returns>True if the GPU and driver supports non-constant texture offsets, false otherwise</returns>
         public bool QuerySupportsNonConstantTextureOffset() => _context.Capabilities.SupportsNonConstantTextureOffset;
 
-        /// <summary>
-        /// Queries host GPU viewport swizzle support.
-        /// </summary>
-        /// <returns>True if the GPU and driver supports viewport swizzle, false otherwise</returns>
-        public bool QuerySupportsViewportSwizzle() => _context.Capabilities.SupportsViewportSwizzle;
-
         /// <summary>
         /// Queries texture format information, for shaders using image load or store.
         /// </summary>
@@ -257,24 +251,6 @@ namespace Ryujinx.Graphics.Gpu.Shader
             };
         }
 
-        public int QueryViewportSwizzle(int component)
-        {
-            YControl yControl = _state.Get<YControl>(MethodOffset.YControl);
-
-            bool flipY = yControl.HasFlag(YControl.NegateY) ^ !yControl.HasFlag(YControl.TriangleRastFlip);
-
-            ViewportTransform transform = _state.Get<ViewportTransform>(MethodOffset.ViewportTransform, 0);
-
-            return component switch
-            {
-                0 => (int)(transform.UnpackSwizzleX() ^ (transform.ScaleX < 0 ? ViewportSwizzle.NegativeFlag : 0)),
-                1 => (int)(transform.UnpackSwizzleY() ^ (transform.ScaleY < 0 ? ViewportSwizzle.NegativeFlag : 0) ^ (flipY ? ViewportSwizzle.NegativeFlag : 0)),
-                2 => (int)(transform.UnpackSwizzleZ() ^ (transform.ScaleZ < 0 ? ViewportSwizzle.NegativeFlag : 0)),
-                3 => (int)transform.UnpackSwizzleW(),
-                _ => throw new ArgumentOutOfRangeException(nameof(component))
-            };
-        }
-
         /// <summary>
         /// Gets the texture descriptor for a given texture on the pool.
         /// </summary>

Ryujinx.Graphics.OpenGL/Pipeline.cs

@@ -39,6 +39,7 @@ namespace Ryujinx.Graphics.OpenGL
         private TextureBase _rtColor0Texture;
         private TextureBase _rtDepthTexture;
 
+        private FrontFaceDirection _frontFace;
         private ClipOrigin _clipOrigin;
         private ClipDepthMode _clipDepthMode;
 
@@ -48,7 +49,7 @@ namespace Ryujinx.Graphics.OpenGL
 
         private bool _tfEnabled;
 
-        ColorF _blendConstant = new ColorF(0, 0, 0, 0);
+        private ColorF _blendConstant;
 
         internal Pipeline()
         {
@@ -570,20 +571,6 @@ namespace Ryujinx.Graphics.OpenGL
             GL.Enable(IndexedEnableCap.Blend, index);
         }
 
-        public void SetLogicOpState(bool enable, LogicalOp op)
-        {
-            if (enable)
-            {
-                GL.Enable(EnableCap.ColorLogicOp);
-
-                GL.LogicOp((LogicOp)op.Convert());
-            }
-            else
-            {
-                GL.Disable(EnableCap.ColorLogicOp);
-            }
-        }
-
         public void SetDepthBias(PolygonModeMask enables, float factor, float units, float clamp)
         {
             if ((enables & PolygonModeMask.Point) != 0)
@@ -676,7 +663,7 @@ namespace Ryujinx.Graphics.OpenGL
 
         public void SetFrontFace(FrontFace frontFace)
         {
-            GL.FrontFace(frontFace.Convert());
+            SetFrontFace(_frontFace = frontFace.Convert());
         }
 
         public void SetImage(int index, ShaderStage stage, ITexture texture)
@@ -706,11 +693,18 @@ namespace Ryujinx.Graphics.OpenGL
             _vertexArray.SetIndexBuffer(buffer.Handle);
         }
 
-        public void SetOrigin(Origin origin)
+        public void SetLogicOpState(bool enable, LogicalOp op)
         {
-            ClipOrigin clipOrigin = origin == Origin.UpperLeft ? ClipOrigin.UpperLeft : ClipOrigin.LowerLeft;
+            if (enable)
+            {
+                GL.Enable(EnableCap.ColorLogicOp);
 
-            SetOrigin(clipOrigin);
+                GL.LogicOp((LogicOp)op.Convert());
+            }
+            else
+            {
+                GL.Disable(EnableCap.ColorLogicOp);
+            }
         }
 
         public void SetPointParameters(float size, bool isProgramPointSize, bool enablePointSprite, Origin origin)
@@ -1030,7 +1024,9 @@ namespace Ryujinx.Graphics.OpenGL
                 Viewport viewport = viewports[index];
 
                 viewportArray[viewportElemIndex + 0] = viewport.Region.X;
-                viewportArray[viewportElemIndex + 1] = viewport.Region.Y;
+                viewportArray[viewportElemIndex + 1] = viewport.Region.Y + (viewport.Region.Height < 0 ? viewport.Region.Height : 0);
+                viewportArray[viewportElemIndex + 2] = viewport.Region.Width;
+                viewportArray[viewportElemIndex + 3] = MathF.Abs(viewport.Region.Height);
 
                 if (HwCapabilities.SupportsViewportSwizzle)
                 {
@@ -1042,13 +1038,14 @@ namespace Ryujinx.Graphics.OpenGL
                         viewport.SwizzleW.Convert());
                 }
 
-                viewportArray[viewportElemIndex + 2] = MathF.Abs(viewport.Region.Width);
-                viewportArray[viewportElemIndex + 3] = MathF.Abs(viewport.Region.Height);
-
                 depthRangeArray[index * 2 + 0] = viewport.DepthNear;
                 depthRangeArray[index * 2 + 1] = viewport.DepthFar;
             }
 
+            bool flipY = viewports.Length != 0 && viewports[0].Region.Height < 0;
+
+            SetOrigin(flipY ? ClipOrigin.UpperLeft : ClipOrigin.LowerLeft);
+
             GL.ViewportArray(first, viewports.Length, viewportArray);
 
             GL.DepthRangeArray(first, viewports.Length, depthRangeArray);
@@ -1097,9 +1094,24 @@ namespace Ryujinx.Graphics.OpenGL
                 _clipOrigin = origin;
 
                 GL.ClipControl(origin, _clipDepthMode);
+
+                SetFrontFace(_frontFace);
             }
         }
 
+        private void SetFrontFace(FrontFaceDirection frontFace)
+        {
+            // Changing clip origin will also change the front face to compensate
+            // for the flipped viewport, we flip it again here to compensate as
+            // this effect is undesirable for us.
+            if (_clipOrigin == ClipOrigin.UpperLeft)
+            {
+                frontFace = frontFace == FrontFaceDirection.Ccw ? FrontFaceDirection.Cw : FrontFaceDirection.Ccw;
+            }
+
+            GL.FrontFace(frontFace);
+        }
+
         private void EnsureVertexArray()
         {
             if (_vertexArray == null)

Ryujinx.Graphics.Shader/IGpuAccessor.cs

@@ -64,22 +64,9 @@
             return true;
         }
 
-        public bool QuerySupportsViewportSwizzle()
-        {
-            return true;
-        }
-
         public TextureFormat QueryTextureFormat(int handle)
         {
             return TextureFormat.R8G8B8A8Unorm;
         }
-
-        public int QueryViewportSwizzle(int component)
-        {
-            // Bit 0: Negate flag.
-            // Bits 2-1: Component.
-            // Example: 0b110 = W, 0b111 = -W, 0b000 = X, 0b010 = Y etc.
-            return component << 1;
-        }
     }
 }

Ryujinx.Graphics.Shader/Translation/EmitterContext.cs

@@ -73,34 +73,7 @@ namespace Ryujinx.Graphics.Shader.Translation
 
         public void PrepareForReturn()
         {
-            if (Config.Stage == ShaderStage.Vertex && (Config.Flags & TranslationFlags.VertexA) == 0)
-            {
-                // Here we attempt to implement viewport swizzle on the vertex shader.
-                // Perform permutation and negation of the output gl_Position components.
-                // Note that per-viewport swizzling can't be supported using this approach.
-                int swizzleX = Config.GpuAccessor.QueryViewportSwizzle(0);
-                int swizzleY = Config.GpuAccessor.QueryViewportSwizzle(1);
-                int swizzleZ = Config.GpuAccessor.QueryViewportSwizzle(2);
-                int swizzleW = Config.GpuAccessor.QueryViewportSwizzle(3);
-
-                bool nonStandardSwizzle = swizzleX != 0 || swizzleY != 2 || swizzleZ != 4 || swizzleW != 6;
-
-                if (!Config.GpuAccessor.QuerySupportsViewportSwizzle() && nonStandardSwizzle)
-                {
-                    Operand[] temp = new Operand[4];
-
-                    temp[0] = this.Copy(Attribute(AttributeConsts.PositionX));
-                    temp[1] = this.Copy(Attribute(AttributeConsts.PositionY));
-                    temp[2] = this.Copy(Attribute(AttributeConsts.PositionZ));
-                    temp[3] = this.Copy(Attribute(AttributeConsts.PositionW));
-
-                    this.Copy(Attribute(AttributeConsts.PositionX), this.FPNegate(temp[(swizzleX >> 1) & 3], (swizzleX & 1) != 0));
-                    this.Copy(Attribute(AttributeConsts.PositionY), this.FPNegate(temp[(swizzleY >> 1) & 3], (swizzleY & 1) != 0));
-                    this.Copy(Attribute(AttributeConsts.PositionZ), this.FPNegate(temp[(swizzleZ >> 1) & 3], (swizzleZ & 1) != 0));
-                    this.Copy(Attribute(AttributeConsts.PositionW), this.FPNegate(temp[(swizzleW >> 1) & 3], (swizzleW & 1) != 0));
-                }
-            }
-            else if (Config.Stage == ShaderStage.Fragment)
+            if (Config.Stage == ShaderStage.Fragment)
             {
                 if (Config.OmapDepth)
                 {