Add ATOMS, LDS, POPC, RED, STS and VOTE shader instructions, start changing the way how global memory is handled

Ryujinx.Graphics.Gpu/Memory/BufferManager.cs

@@ -298,23 +298,6 @@ namespace Ryujinx.Graphics.Gpu.Memory
                 BufferRange buffer = GetBufferRange(bounds.Address, bounds.Size);
 
                 _context.Renderer.Pipeline.BindUniformBuffer(index, ShaderStage.Compute, buffer);
-
-                if (index == 0)
-                {
-                    // TODO: Improve
-                    Span<byte> data = _context.PhysicalMemory.Read(bounds.Address + 0x310, 0x100);
-
-                    Span<int> words = MemoryMarshal.Cast<byte, int>(data);
-
-                    for (int offset = 0; offset < 0x40; offset += 4)
-                    {
-                        words[offset] &= 0x3f;
-                    }
-
-                    buffer = GetBufferRange(bounds.Address + 0x310, 0x100);
-
-                    buffer.Buffer.SetData(buffer.Offset, data);
-                }
             }
 
             // Force rebind after doing compute work.
@@ -460,23 +443,6 @@ namespace Ryujinx.Graphics.Gpu.Memory
             {
                 _context.Renderer.Pipeline.BindUniformBuffer(index, stage, buffer);
             }
-
-            if (!isStorage && index == 0)
-            {
-                // TODO: Improve
-                Span<byte> data = _context.PhysicalMemory.Read(bounds.Address + 0x110, 0x100);
-
-                Span<int> words = MemoryMarshal.Cast<byte, int>(data);
-
-                for (int offset = 0; offset < 0x40; offset += 4)
-                {
-                    words[offset] &= 0x3f;
-                }
-
-                buffer = GetBufferRange(bounds.Address + 0x110, 0x100);
-
-                buffer.Buffer.SetData(buffer.Offset, data);
-            }
         }
 
         public void CopyBuffer(GpuVa srcVa, GpuVa dstVa, ulong size)

Ryujinx.Graphics.Gpu/Memory/ConcurrentRangeList.cs

@@ -6,8 +6,6 @@ namespace Ryujinx.Graphics.Gpu.Memory
     {
         private List<T> _items;
 
-        public int Count => _items.Count;
-
         public ConcurrentRangeList()
         {
             _items = new List<T>();

Ryujinx.Graphics.Gpu/Memory/RangeList.cs

@@ -6,8 +6,6 @@ namespace Ryujinx.Graphics.Gpu.Memory
     {
         private List<T> _items;
 
-        public int Count => _items.Count;
-
         public RangeList()
         {
             _items = new List<T>();
@@ -57,20 +55,6 @@ namespace Ryujinx.Graphics.Gpu.Memory
             return false;
         }
 
-        public bool CanExitEarly(ulong address, ulong size)
-        {
-            int index = BinarySearch(address, size);
-
-            if (index >= 0)
-            {
-                T item = _items[index];
-
-                return address >= item.Address && address + size <= item.Address + item.Size;
-            }
-
-            return false;
-        }
-
         public T FindFirstOverlap(T item)
         {
             return FindFirstOverlap(item.Address, item.Size);

Ryujinx.Graphics.Shader/CodeGen/Constants.cs

@@ -0,0 +1,7 @@
+namespace Ryujinx.Graphics.Shader.CodeGen
+{
+    static class Constants
+    {
+        public const int MaxShaderStorageBuffers = 16;
+    }
+}

Ryujinx.Graphics.Shader/CodeGen/Glsl/Declarations.cs

@@ -16,7 +16,9 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl
         public static void Declare(CodeGenContext context, StructuredProgramInfo info)
         {
             context.AppendLine("#version 420 core");
+            context.AppendLine("#extension GL_ARB_gpu_shader_int64 : enable");
             context.AppendLine("#extension GL_ARB_shader_ballot : enable");
+            context.AppendLine("#extension GL_ARB_shader_group_vote : enable");
             context.AppendLine("#extension GL_ARB_shader_storage_buffer_object : enable");
 
             if (context.Config.Stage == ShaderStage.Compute)
@@ -66,9 +68,15 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl
 
             context.AppendLine();
 
-            context.AppendLine($"precise float {DefaultNames.LocalMemoryName}[0x100];");
+            context.AppendLine($"uint {DefaultNames.LocalMemoryName}[0x100];");
             context.AppendLine();
 
+            if (context.Config.Stage == ShaderStage.Compute)
+            {
+                context.AppendLine($"shared uint {DefaultNames.SharedMemoryName}[0x100];");
+                context.AppendLine();
+            }
+
             if (info.CBuffers.Count != 0)
             {
                 DeclareUniforms(context, info);
@@ -78,7 +86,7 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl
 
             if (info.SBuffers.Count != 0)
             {
-                DeclareStorage(context, info);
+                DeclareUsedStorage(context, info);
 
                 context.AppendLine();
             }
@@ -168,6 +176,11 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl
 
                 context.AppendLine(GetVarTypeName(decl.VarType) + " " + name + ";");
             }
+
+            if ((info.HelperFunctionsMask & HelperFunctionsMask.GlobalMemory) != 0)
+            {
+                context.AppendLine($"ivec2 {DefaultNames.GmemOffsetName};");
+            }
         }
 
         private static string GetVarTypeName(VariableType type)
@@ -205,24 +218,59 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl
             }
         }
 
-        private static void DeclareStorage(CodeGenContext context, StructuredProgramInfo info)
+        private static void DeclareAllStorage(CodeGenContext context, StructuredProgramInfo info)
         {
-            foreach (int sbufSlot in info.SBuffers.OrderBy(x => x))
+            string sbName = OperandManager.GetShaderStagePrefix(context.Config.Stage);
+
+            sbName += "_" + DefaultNames.StorageNamePrefix;
+
+            string blockName = $"{sbName}_{DefaultNames.BlockSuffix}";
+
+            context.AppendLine("layout (std430) buffer " + blockName);
+
+            context.EnterScope();
+
+            context.AppendLine("uint " + DefaultNames.DataName + "[];");
+
+            string arraySize = NumberFormatter.FormatInt(Constants.MaxShaderStorageBuffers);
+
+            context.LeaveScope($" {sbName}[{arraySize}];");
+
+            for (int sbufSlot = 0; sbufSlot < Constants.MaxShaderStorageBuffers; sbufSlot++)
             {
-                string sbName = OperandManager.GetShaderStagePrefix(context.Config.Stage);
+                context.SBufferDescriptors.Add(new BufferDescriptor($"{blockName}[{sbufSlot}]", sbufSlot));
+            }
+        }
 
-                sbName += "_" + DefaultNames.StorageNamePrefix + sbufSlot;
+        private static void DeclareUsedStorage(CodeGenContext context, StructuredProgramInfo info)
+        {
+            string sbName = OperandManager.GetShaderStagePrefix(context.Config.Stage);
 
-                context.SBufferDescriptors.Add(new BufferDescriptor(sbName, sbufSlot));
+            sbName += "_" + DefaultNames.StorageNamePrefix;
 
-                context.AppendLine("layout (std430) buffer " + sbName);
+            string blockName = $"{sbName}_{DefaultNames.BlockSuffix}";
 
-                context.EnterScope();
+            int maxSlot = 0;
 
-                context.AppendLine("precise float " + OperandManager.GetSbName(context.Config.Stage, sbufSlot) + "[];");
+            foreach (int sbufSlot in info.SBuffers)
+            {
+                context.SBufferDescriptors.Add(new BufferDescriptor($"{blockName}[{sbufSlot}]", sbufSlot));
 
-                context.LeaveScope(";");
+                if (maxSlot < sbufSlot)
+                {
+                    maxSlot = sbufSlot;
+                }
             }
+
+            context.AppendLine("layout (std430) buffer " + blockName);
+
+            context.EnterScope();
+
+            context.AppendLine("uint " + DefaultNames.DataName + "[];");
+
+            string arraySize = NumberFormatter.FormatInt(maxSlot + 1);
+
+            context.LeaveScope($" {sbName}[{arraySize}];");
         }
 
         private static void DeclareSamplers(CodeGenContext context, StructuredProgramInfo info)

Ryujinx.Graphics.Shader/CodeGen/Glsl/DefaultNames.cs

@@ -11,12 +11,18 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl
         public const string OAttributePrefix = "out_attr";
 
         public const string StorageNamePrefix = "s";
-        public const string StorageNameSuffix = "data";
+
+        public const string DataName = "data";
+
+        public const string BlockSuffix = "block";
 
         public const string UniformNamePrefix = "c";
         public const string UniformNameSuffix = "data";
 
-        public const string LocalMemoryName = "local_mem";
+        public const string LocalMemoryName  = "local_mem";
+        public const string SharedMemoryName = "shared_mem";
+
+        public const string GmemOffsetName  = "gmemOffset";
 
         public const string UndefinedName = "undef";
     }

Ryujinx.Graphics.Shader/CodeGen/Glsl/HelperFunctions/GlobalMemory.glsl

@@ -0,0 +1,18 @@
+ivec2 Helper_GetStorageBuffer(uint aLow, uint aHigh)
+{
+    uint64_t address = packUint2x32(uvec2(aLow, aHigh));
+    int i;
+    for (i = 0; i < 16; i++)
+    {
+        int offset = 0x40 + i * 4;
+        uint baseLow  = fp_c0_data[offset];
+        uint baseHigh = fp_c0_data[offset + 1];
+        uint size     = fp_c0_data[offset + 2];
+        uint64_t baseAddr = packUint2x32(uvec2(baseLow, baseHigh));
+        if (address >= baseAddr && address < baseAddr + packUint2x32(uvec2(size, 0)))
+        {
+            return ivec2(i, int(unpackUint2x32(address - (baseAddr & ~63ul)).x) >> 2);
+        }
+    }
+    return ivec2(0);
+}

Ryujinx.Graphics.Shader/CodeGen/Glsl/HelperFunctions/HelperFunctionNames.cs

@@ -2,6 +2,8 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl
 {
     static class HelperFunctionNames
     {
+        public static string GetStorageBuffer = "Helper_GetStorageBuffer";
+
         public static string Shuffle     = "Helper_Shuffle";
         public static string ShuffleDown = "Helper_ShuffleDown";
         public static string ShuffleUp   = "Helper_ShuffleUp";

Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstGen.cs

@@ -3,6 +3,7 @@ using Ryujinx.Graphics.Shader.StructuredIr;
 using System;
 
 using static Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions.InstGenHelper;
+using static Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions.InstGenMemory;
 using static Ryujinx.Graphics.Shader.StructuredIr.InstructionInfo;
 
 namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions
@@ -31,6 +32,8 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions
 
             if ((info.Type & InstType.Call) != 0)
             {
+                bool atomic = (info.Type & InstType.Atomic) != 0;
+
                 int arity = (int)(info.Type & InstType.ArityMask);
 
                 string args = string.Empty;
@@ -44,10 +47,29 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions
 
                     VariableType dstType = GetSrcVarType(inst, argIndex);
 
-                    args += GetSoureExpr(context, operation.GetSource(argIndex), dstType);
+                    if (argIndex == 0 && atomic)
+                    {
+                        switch (inst & Instruction.MrMask)
+                        {
+                            // TODO: Global.
+                            case Instruction.MrShared:  args += LoadShared (context, operation); break;
+                            case Instruction.MrStorage: args += LoadStorage(context, operation); break;
+                        }
+                    }
+                    else
+                    {
+                        args += GetSoureExpr(context, operation.GetSource(argIndex), dstType);
+                    }
                 }
 
-                return info.OpName + "(" + args + ")";
+                if (inst == Instruction.Ballot)
+                {
+                    return $"unpackUint2x32({info.OpName}({args})).x";
+                }
+                else
+                {
+                    return info.OpName + "(" + args + ")";
+                }
             }
             else if ((info.Type & InstType.Op) != 0)
             {
@@ -99,6 +121,9 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions
                     case Instruction.LoadLocal:
                         return InstGenMemory.LoadLocal(context, operation);
 
+                    case Instruction.LoadShared:
+                        return InstGenMemory.LoadShared(context, operation);
+
                     case Instruction.LoadStorage:
                         return InstGenMemory.LoadStorage(context, operation);
 
@@ -108,6 +133,9 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions
                     case Instruction.StoreLocal:
                         return InstGenMemory.StoreLocal(context, operation);
 
+                    case Instruction.StoreShared:
+                        return InstGenMemory.StoreShared(context, operation);
+
                     case Instruction.StoreStorage:
                         return InstGenMemory.StoreStorage(context, operation);
 

Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstGenHelper.cs

@@ -13,8 +13,19 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions
         {
             _infoTbl = new InstInfo[(int)Instruction.Count];
 
+            Add(Instruction.AtomicAdd,                InstType.AtomicBinary,   "atomicAdd");
+            Add(Instruction.AtomicAnd,                InstType.AtomicBinary,   "atomicAnd");
+            Add(Instruction.AtomicCompareAndSwap,     InstType.AtomicTernary,  "atomicCompSwap");
+            Add(Instruction.AtomicMaxS32,             InstType.AtomicBinary,   "atomicMax");
+            Add(Instruction.AtomicMaxU32,             InstType.AtomicBinary,   "atomicMax");
+            Add(Instruction.AtomicMinS32,             InstType.AtomicBinary,   "atomicMin");
+            Add(Instruction.AtomicMinU32,             InstType.AtomicBinary,   "atomicMin");
+            Add(Instruction.AtomicOr,                 InstType.AtomicBinary,   "atomicOr");
+            Add(Instruction.AtomicSwap,               InstType.AtomicBinary,   "atomicExchange");
+            Add(Instruction.AtomicXor,                InstType.AtomicBinary,   "atomicXor");
             Add(Instruction.Absolute,                 InstType.CallUnary,      "abs");
             Add(Instruction.Add,                      InstType.OpBinaryCom,    "+",               2);
+            Add(Instruction.Ballot,                   InstType.CallUnary,      "ballotARB");
             Add(Instruction.BitCount,                 InstType.CallUnary,      "bitCount");
             Add(Instruction.BitfieldExtractS32,       InstType.CallTernary,    "bitfieldExtract");
             Add(Instruction.BitfieldExtractU32,       InstType.CallTernary,    "bitfieldExtract");
@@ -59,6 +70,7 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions
             Add(Instruction.LoadAttribute,            InstType.Special);
             Add(Instruction.LoadConstant,             InstType.Special);
             Add(Instruction.LoadLocal,                InstType.Special);
+            Add(Instruction.LoadShared,               InstType.Special);
             Add(Instruction.LoadStorage,              InstType.Special);
             Add(Instruction.LogarithmB2,              InstType.CallUnary,      "log2");
             Add(Instruction.LogicalAnd,               InstType.OpBinaryCom,    "&&",              9);
@@ -87,6 +99,7 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions
             Add(Instruction.Sine,                     InstType.CallUnary,      "sin");
             Add(Instruction.SquareRoot,               InstType.CallUnary,      "sqrt");
             Add(Instruction.StoreLocal,               InstType.Special);
+            Add(Instruction.StoreShared,              InstType.Special);
             Add(Instruction.StoreStorage,             InstType.Special);
             Add(Instruction.Subtract,                 InstType.OpBinary,       "-",               2);
             Add(Instruction.SwizzleAdd,               InstType.CallTernary,    HelperFunctionNames.SwizzleAdd);
@@ -94,6 +107,9 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions
             Add(Instruction.TextureSize,              InstType.Special);
             Add(Instruction.Truncate,                 InstType.CallUnary,      "trunc");
             Add(Instruction.UnpackHalf2x16,           InstType.Special);
+            Add(Instruction.VoteAll,                  InstType.CallUnary,      "allInvocationsARB");
+            Add(Instruction.VoteAllEqual,             InstType.CallUnary,      "allInvocationsEqualARB");
+            Add(Instruction.VoteAny,                  InstType.CallUnary,      "anyInvocationARB");
         }
 
         private static void Add(Instruction inst, InstType flags, string opName = null, int precedence = 0)

Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstGenMemory.cs

@@ -1,5 +1,6 @@
 using Ryujinx.Graphics.Shader.IntermediateRepresentation;
 using Ryujinx.Graphics.Shader.StructuredIr;
+using Ryujinx.Graphics.Shader.Translation.Optimizations;
 using System;
 
 using static Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions.InstGenHelper;
@@ -118,26 +119,75 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions
             return OperandManager.GetConstantBufferName(src1, offsetExpr, context.Config.Stage);
         }
 
+        public static string LoadGlobal(CodeGenContext context, AstOperation operation)
+        {
+            IAstNode src1 = operation.GetSource(0);
+            IAstNode src2 = operation.GetSource(1);
+
+            string addrLowExpr  = GetSoureExpr(context, src1, GetSrcVarType(operation.Inst, 0));
+            string addrHighExpr = GetSoureExpr(context, src2, GetSrcVarType(operation.Inst, 1));
+
+            context.AppendLine($"{DefaultNames.GmemOffsetName} = {HelperFunctionNames.GetStorageBuffer}({addrLowExpr}, {addrHighExpr});");
+
+            return GetStorageBufferAccessor($"{DefaultNames.GmemOffsetName}.x", $"{DefaultNames.GmemOffsetName}.y", context.Config.Stage);
+        }
+
         public static string LoadLocal(CodeGenContext context, AstOperation operation)
+        {
+            return LoadLocalOrShared(context, operation, DefaultNames.LocalMemoryName);
+        }
+
+        public static string LoadShared(CodeGenContext context, AstOperation operation)
+        {
+            return LoadLocalOrShared(context, operation, DefaultNames.SharedMemoryName);
+        }
+
+        private static string LoadLocalOrShared(CodeGenContext context, AstOperation operation, string arrayName)
         {
             IAstNode src1 = operation.GetSource(0);
 
             string offsetExpr = GetSoureExpr(context, src1, GetSrcVarType(operation.Inst, 0));
 
-            return $"{DefaultNames.LocalMemoryName}[{offsetExpr}]";
+            return $"{arrayName}[{offsetExpr}]";
         }
 
         public static string LoadStorage(CodeGenContext context, AstOperation operation)
+        {
+            IAstNode src1 = operation.GetSource(0);
+
+            string offsetExpr = GetSoureExpr(context, src1, GetSrcVarType(operation.Inst, 0));
+
+            return GetStorageBufferAccessor(operation.Index, offsetExpr, context.Config.Stage);
+        }
+
+        public static string StoreGlobal(CodeGenContext context, AstOperation operation)
         {
             IAstNode src1 = operation.GetSource(0);
             IAstNode src2 = operation.GetSource(1);
+            IAstNode src3 = operation.GetSource(2);
 
-            string offsetExpr = GetSoureExpr(context, src2, GetSrcVarType(operation.Inst, 1));
+            string addrLowExpr  = GetSoureExpr(context, src1, GetSrcVarType(operation.Inst, 0));
+            string addrHighExpr = GetSoureExpr(context, src2, GetSrcVarType(operation.Inst, 1));
+            string valueExpr    = GetSoureExpr(context, src3, GetSrcVarType(operation.Inst, 2));
 
-            return OperandManager.GetStorageBufferName(src1, offsetExpr, context.Config.Stage);
+            context.AppendLine($"{DefaultNames.GmemOffsetName} = {HelperFunctionNames.GetStorageBuffer}({addrLowExpr}, {addrHighExpr});");
+
+            string sb = GetStorageBufferAccessor($"{DefaultNames.GmemOffsetName}.x", $"{DefaultNames.GmemOffsetName}.y", context.Config.Stage);
+
+            return $"{sb} = {valueExpr}";
         }
 
         public static string StoreLocal(CodeGenContext context, AstOperation operation)
+        {
+            return StoreLocalOrShared(context, operation, DefaultNames.LocalMemoryName);
+        }
+
+        public static string StoreShared(CodeGenContext context, AstOperation operation)
+        {
+            return StoreLocalOrShared(context, operation, DefaultNames.SharedMemoryName);
+        }
+
+        private static string StoreLocalOrShared(CodeGenContext context, AstOperation operation, string arrayName)
         {
             IAstNode src1 = operation.GetSource(0);
             IAstNode src2 = operation.GetSource(1);
@@ -146,26 +196,25 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions
 
             VariableType srcType = OperandManager.GetNodeDestType(src2);
 
-            string src = TypeConversion.ReinterpretCast(context, src2, srcType, VariableType.F32);
+            string src = TypeConversion.ReinterpretCast(context, src2, srcType, VariableType.U32);
 
-            return $"{DefaultNames.LocalMemoryName}[{offsetExpr}] = {src}";
+            return $"{arrayName}[{offsetExpr}] = {src}";
         }
 
         public static string StoreStorage(CodeGenContext context, AstOperation operation)
         {
             IAstNode src1 = operation.GetSource(0);
             IAstNode src2 = operation.GetSource(1);
-            IAstNode src3 = operation.GetSource(2);
 
-            string offsetExpr = GetSoureExpr(context, src2, GetSrcVarType(operation.Inst, 1));
+            string offsetExpr = GetSoureExpr(context, src1, GetSrcVarType(operation.Inst, 0));
 
-            VariableType srcType = OperandManager.GetNodeDestType(src3);
+            VariableType srcType = OperandManager.GetNodeDestType(src2);
 
-            string src = TypeConversion.ReinterpretCast(context, src3, srcType, VariableType.F32);
+            string src = TypeConversion.ReinterpretCast(context, src2, srcType, VariableType.U32);
 
-            string sbName = OperandManager.GetStorageBufferName(src1, offsetExpr, context.Config.Stage);
+            string sb = GetStorageBufferAccessor(operation.Index, offsetExpr, context.Config.Stage);
 
-            return $"{sbName} = {src}";
+            return $"{sb} = {src}";
         }
 
         public static string TextureSample(CodeGenContext context, AstOperation operation)
@@ -402,7 +451,7 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions
                Append(Src(VariableType.S32));
             }
 
-            texCall += ")" + (isGather || !isShadow ? GetMask(texOp.ComponentMask) : "");
+            texCall += ")" + (isGather || !isShadow ? GetMask(texOp.Index) : "");
 
             return texCall;
         }
@@ -428,22 +477,42 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions
 
             string src0Expr = GetSoureExpr(context, src0, GetSrcVarType(operation.Inst, 0));
 
-            return $"textureSize({samplerName}, {src0Expr}){GetMask(texOp.ComponentMask)}";
+            return $"textureSize({samplerName}, {src0Expr}){GetMask(texOp.Index)}";
         }
 
-        private static string GetMask(int compMask)
+        private static string GetStorageBufferAccessor(string slotExpr, string offsetExpr, ShaderStage stage)
         {
-            string mask = ".";
+            string sbName = OperandManager.GetShaderStagePrefix(stage);
 
-            for (int index = 0; index < 4; index++)
-            {
-                if ((compMask & (1 << index)) != 0)
-                {
-                    mask += "rgba".Substring(index, 1);
-                }
-            }
+            sbName += "_" + DefaultNames.StorageNamePrefix;
+
+            return $"{sbName}[{slotExpr}].{DefaultNames.DataName}[{offsetExpr}]";
+        }
+
+        private static string GetStorageBufferAccessor(int slot, string offsetExpr, ShaderStage stage)
+        {
+            string sbName = OperandManager.GetShaderStagePrefix(stage);
+
+            sbName += "_" + DefaultNames.StorageNamePrefix;
+
+            string mask = NumberFormatter.FormatUint(~(64u - 1));
 
-            return mask;
+            // Subtract the base address of the global memory, to get the
+            // storage buffer offset. The mask is used to keep the lower bits,
+            // since the bound storage buffer must match the host alignment
+            // restrictions.
+            int ubOffset = GlobalToStorage.GetStorageCbOffset(stage, slot);
+
+            string ubName = OperandManager.GetConstantBufferName(0, ubOffset, stage);
+
+            offsetExpr = $"{offsetExpr} - int((floatBitsToUint({ubName}) & {mask}) >> 2)";
+
+            return $"{sbName}[{NumberFormatter.FormatInt(slot)}].{DefaultNames.DataName}[{offsetExpr}]";
+        }
+
+        private static string GetMask(int index)
+        {
+            return '.' + "rgba".Substring(index, 1);
         }
     }
 }

Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstGenPacking.cs

@@ -24,22 +24,12 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions
 
             string srcExpr = GetSoureExpr(context, src, GetSrcVarType(operation.Inst, 0));
 
-            return $"unpackHalf2x16({srcExpr}){GetMask(operation.ComponentMask)}";
+            return $"unpackHalf2x16({srcExpr}){GetMask(operation.Index)}";
         }
 
-        private static string GetMask(int compMask)
+        private static string GetMask(int index)
         {
-            string mask = ".";
-
-            for (int index = 0; index < 2; index++)
-            {
-                if ((compMask & (1 << index)) != 0)
-                {
-                    mask += "xy".Substring(index, 1);
-                }
-            }
-
-            return mask;
+            return '.' + "xy".Substring(index, 1);
         }
     }
 }

Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstType.cs

@@ -8,8 +8,11 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions
         OpNullary   = Op | 0,
         OpUnary     = Op | 1,
         OpBinary    = Op | 2,
+        OpBinaryCom = Op | 2 | Commutative,
         OpTernary   = Op | 3,
-        OpBinaryCom = OpBinary | Commutative,
+
+        AtomicBinary  = CallBinary  | Atomic,
+        AtomicTernary = CallTernary | Atomic,
 
         CallNullary    = Call | 0,
         CallUnary      = Call | 1,
@@ -20,7 +23,8 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions
         Commutative = 1 << 8,
         Op          = 1 << 9,
         Call        = 1 << 10,
-        Special     = 1 << 11,
+        Atomic      = 1 << 11,
+        Special     = 1 << 12,
 
         ArityMask = 0xff
     }

Ryujinx.Graphics.Shader/CodeGen/Glsl/OperandManager.cs

@@ -51,13 +51,19 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl
             { AttributeConsts.FrontFacing,         new BuiltInAttribute("gl_FrontFacing",     VariableType.Bool) },
 
             // Special.
-            { AttributeConsts.FragmentOutputDepth, new BuiltInAttribute("gl_FragDepth",           VariableType.F32) },
-            { AttributeConsts.ThreadIdX,           new BuiltInAttribute("gl_LocalInvocationID.x", VariableType.U32) },
-            { AttributeConsts.ThreadIdY,           new BuiltInAttribute("gl_LocalInvocationID.y", VariableType.U32) },
-            { AttributeConsts.ThreadIdZ,           new BuiltInAttribute("gl_LocalInvocationID.z", VariableType.U32) },
-            { AttributeConsts.CtaIdX,              new BuiltInAttribute("gl_WorkGroupID.x",       VariableType.U32) },
-            { AttributeConsts.CtaIdY,              new BuiltInAttribute("gl_WorkGroupID.y",       VariableType.U32) },
-            { AttributeConsts.CtaIdZ,              new BuiltInAttribute("gl_WorkGroupID.z",       VariableType.U32) },
+            { AttributeConsts.FragmentOutputDepth, new BuiltInAttribute("gl_FragDepth",                           VariableType.F32) },
+            { AttributeConsts.ThreadIdX,           new BuiltInAttribute("gl_LocalInvocationID.x",                 VariableType.U32) },
+            { AttributeConsts.ThreadIdY,           new BuiltInAttribute("gl_LocalInvocationID.y",                 VariableType.U32) },
+            { AttributeConsts.ThreadIdZ,           new BuiltInAttribute("gl_LocalInvocationID.z",                 VariableType.U32) },
+            { AttributeConsts.CtaIdX,              new BuiltInAttribute("gl_WorkGroupID.x",                       VariableType.U32) },
+            { AttributeConsts.CtaIdY,              new BuiltInAttribute("gl_WorkGroupID.y",                       VariableType.U32) },
+            { AttributeConsts.CtaIdZ,              new BuiltInAttribute("gl_WorkGroupID.z",                       VariableType.U32) },
+            { AttributeConsts.LaneId,              new BuiltInAttribute("gl_SubGroupInvocationARB",               VariableType.U32) },
+            { AttributeConsts.EqMask,              new BuiltInAttribute("unpackUint2x32(gl_SubGroupEqMaskARB).x", VariableType.U32) },
+            { AttributeConsts.GeMask,              new BuiltInAttribute("unpackUint2x32(gl_SubGroupGeMaskARB).x", VariableType.U32) },
+            { AttributeConsts.GtMask,              new BuiltInAttribute("unpackUint2x32(gl_SubGroupGtMaskARB).x", VariableType.U32) },
+            { AttributeConsts.LeMask,              new BuiltInAttribute("unpackUint2x32(gl_SubGroupLeMaskARB).x", VariableType.U32) },
+            { AttributeConsts.LtMask,              new BuiltInAttribute("unpackUint2x32(gl_SubGroupLtMaskARB).x", VariableType.U32) },
         };
 
         private Dictionary<AstOperand, string> _locals;
@@ -87,7 +93,7 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl
                     return NumberFormatter.FormatInt(operand.Value);
 
                 case OperandType.ConstantBuffer:
-                    return GetConstantBufferName(operand, stage);
+                    return GetConstantBufferName(operand.CbufSlot, operand.CbufOffset, stage);
 
                 case OperandType.LocalVariable:
                     return _locals[operand];
@@ -99,25 +105,13 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl
             throw new ArgumentException($"Invalid operand type \"{operand.Type}\".");
         }
 
-        public static string GetConstantBufferName(AstOperand cbuf, ShaderStage stage)
+        public static string GetConstantBufferName(int slot, int offset, ShaderStage stage)
         {
-            string ubName = GetUbName(stage, cbuf.CbufSlot);
+            string ubName = GetUbName(stage, slot);
 
-            ubName += "[" + (cbuf.CbufOffset >> 2) + "]";
+            ubName += "[" + (offset >> 2) + "]";
 
-            return ubName + "." + GetSwizzleMask(cbuf.CbufOffset & 3);
-        }
-
-        public static string GetStorageBufferName(IAstNode slot, string offsetExpr, ShaderStage stage)
-        {
-            // Non-constant slots are not supported.
-            // It is expected that upstream stages are never going to generate non-constant
-            // slot access.
-            AstOperand operand = (AstOperand)slot;
-
-            string sbName = GetSbName(stage, operand.Value);
-
-            return $"{sbName}[{offsetExpr}]";
+            return ubName + "." + GetSwizzleMask(offset & 3);
         }
 
         public static string GetConstantBufferName(IAstNode slot, string offsetExpr, ShaderStage stage)
@@ -205,15 +199,6 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl
             return isOutAttr ? "// bad_attr0x" + value.ToString("X") : "0.0";
         }
 
-        public static string GetSbName(ShaderStage stage, int slot)
-        {
-            string sbName = GetShaderStagePrefix(stage);
-
-            sbName += "_" + DefaultNames.StorageNamePrefix + slot;
-
-            return sbName + "_" + DefaultNames.StorageNameSuffix;
-        }
-
         public static string GetUbName(ShaderStage stage, int slot)
         {
             string ubName = GetShaderStagePrefix(stage);

Ryujinx.Graphics.Shader/Decoders/AtomicOp.cs

@@ -0,0 +1,15 @@
+namespace Ryujinx.Graphics.Shader.Decoders
+{
+    enum AtomicOp
+    {
+        Add                = 0,
+        Minimum            = 1,
+        Maximum            = 2,
+        Increment          = 3,
+        Decrement          = 4,
+        BitwiseAnd         = 5,
+        BitwiseOr          = 6,
+        BitwiseExclusiveOr = 7,
+        Swap               = 8
+    }
+}

Ryujinx.Graphics.Shader/Decoders/IOpCodeAlu.cs

@@ -1,10 +1,7 @@
 namespace Ryujinx.Graphics.Shader.Decoders
 {
-    interface IOpCodeAlu : IOpCodeRd, IOpCodeRa
+    interface IOpCodeAlu : IOpCodeRd, IOpCodeRa, IOpCodePredicate39
     {
-        Register Predicate39 { get; }
-
-        bool InvertP     { get; }
         bool Extended    { get; }
         bool SetCondCode { get; }
         bool Saturate    { get; }

Ryujinx.Graphics.Shader/Decoders/IOpCodePredicate39.cs

@@ -0,0 +1,9 @@
+namespace Ryujinx.Graphics.Shader.Decoders
+{
+    interface IOpCodePredicate39
+    {
+        Register Predicate39 { get; }
+
+        bool InvertP { get; }
+    }
+}

Ryujinx.Graphics.Shader/Decoders/OpCodeAtom.cs

@@ -0,0 +1,39 @@
+using Ryujinx.Graphics.Shader.Instructions;
+
+namespace Ryujinx.Graphics.Shader.Decoders
+{
+    class OpCodeAtom : OpCode, IOpCodeRd, IOpCodeRa, IOpCodeReg
+    {
+        public Register Rd { get; }
+        public Register Ra { get; }
+        public Register Rb { get; }
+
+        public ReductionType Type { get; }
+
+        public int Offset { get; }
+
+        public bool Extended { get; }
+
+        public AtomicOp AtomicOp { get; }
+
+        public OpCodeAtom(InstEmitter emitter, ulong address, long opCode) : base(emitter, address, opCode)
+        {
+            Rd = new Register(opCode.Extract(0,  8), RegisterType.Gpr);
+            Ra = new Register(opCode.Extract(8,  8), RegisterType.Gpr);
+            Rb = new Register(opCode.Extract(20, 8), RegisterType.Gpr);
+
+            Type = (ReductionType)opCode.Extract(28, 2);
+
+            if (Type == ReductionType.FP32FtzRn)
+            {
+                Type = ReductionType.S64;
+            }
+
+            Offset = opCode.Extract(30, 22);
+
+            Extended = opCode.Extract(48);
+
+            AtomicOp = (AtomicOp)opCode.Extract(52, 4);
+        }
+    }
+}

Ryujinx.Graphics.Shader/Decoders/OpCodeRed.cs

@@ -0,0 +1,32 @@
+using Ryujinx.Graphics.Shader.Instructions;
+
+namespace Ryujinx.Graphics.Shader.Decoders
+{
+    class OpCodeRed : OpCode, IOpCodeRd, IOpCodeRa
+    {
+        public Register Rd { get; }
+        public Register Ra { get; }
+
+        public AtomicOp AtomicOp { get; }
+
+        public ReductionType Type { get; }
+
+        public int Offset { get; }
+
+        public bool Extended { get; }
+
+        public OpCodeRed(InstEmitter emitter, ulong address, long opCode) : base(emitter, address, opCode)
+        {
+            Rd = new Register(opCode.Extract(0, 8), RegisterType.Gpr);
+            Ra = new Register(opCode.Extract(8, 8), RegisterType.Gpr);
+
+            Type = (ReductionType)opCode.Extract(20, 3);
+
+            AtomicOp = (AtomicOp)opCode.Extract(23, 3);
+
+            Offset = opCode.Extract(28, 20);
+
+            Extended = opCode.Extract(48);
+        }
+    }
+}

Ryujinx.Graphics.Shader/Decoders/OpCodeTable.cs

@@ -32,6 +32,7 @@ namespace Ryujinx.Graphics.Shader.Decoders
 #region Instructions
             Set("1110111111011x", InstEmit.Ald,     typeof(OpCodeAttribute));
             Set("1110111111110x", InstEmit.Ast,     typeof(OpCodeAttribute));
+            Set("11101100xxxxxx", InstEmit.Atoms,   typeof(OpCodeAtom));
             Set("0100110000000x", InstEmit.Bfe,     typeof(OpCodeAluCbuf));
             Set("0011100x00000x", InstEmit.Bfe,     typeof(OpCodeAluImm));
             Set("0101110000000x", InstEmit.Bfe,     typeof(OpCodeAluReg));
@@ -122,6 +123,7 @@ namespace Ryujinx.Graphics.Shader.Decoders
             Set("1110111101000x", InstEmit.Ld,      typeof(OpCodeMemory));
             Set("1110111110010x", InstEmit.Ldc,     typeof(OpCodeLdc));
             Set("1110111011010x", InstEmit.Ldg,     typeof(OpCodeMemory));
+            Set("1110111101001x", InstEmit.Lds,     typeof(OpCodeMemory));
             Set("0100110001000x", InstEmit.Lop,     typeof(OpCodeLopCbuf));
             Set("0011100001000x", InstEmit.Lop,     typeof(OpCodeLopImm));
             Set("000001xxxxxxxx", InstEmit.Lop,     typeof(OpCodeLopImm32));
@@ -136,7 +138,11 @@ namespace Ryujinx.Graphics.Shader.Decoders
             Set("0101000010000x", InstEmit.Mufu,    typeof(OpCodeFArith));
             Set("1111101111100x", InstEmit.Out,     typeof(OpCode));
             Set("111000101010xx", InstEmit.Pbk,     typeof(OpCodeSsy));
+            Set("0100110000001x", InstEmit.Popc,    typeof(OpCodeAluCbuf));
+            Set("0011100x00001x", InstEmit.Popc,    typeof(OpCodeAluImm));
+            Set("0101110000001x", InstEmit.Popc,    typeof(OpCodeAluReg));
             Set("0101000010010x", InstEmit.Psetp,   typeof(OpCodePsetp));
+            Set("1110101111111x", InstEmit.Red,     typeof(OpCodeRed));
             Set("0100110010010x", InstEmit.Rro,     typeof(OpCodeFArithCbuf));
             Set("0011100x10010x", InstEmit.Rro,     typeof(OpCodeFArithImm));
             Set("0101110010010x", InstEmit.Rro,     typeof(OpCodeFArithReg));
@@ -154,6 +160,7 @@ namespace Ryujinx.Graphics.Shader.Decoders
             Set("111000101001xx", InstEmit.Ssy,     typeof(OpCodeSsy));
             Set("1110111101010x", InstEmit.St,      typeof(OpCodeMemory));
             Set("1110111011011x", InstEmit.Stg,     typeof(OpCodeMemory));
+            Set("1110111101011x", InstEmit.Sts,     typeof(OpCodeMemory));
             Set("11101011001xxx", InstEmit.Sust,    typeof(OpCodeImage));
             Set("1111000011111x", InstEmit.Sync,    typeof(OpCodeSync));
             Set("110000xxxx111x", InstEmit.Tex,     typeof(OpCodeTex));
@@ -168,6 +175,7 @@ namespace Ryujinx.Graphics.Shader.Decoders
             Set("1101111101001x", InstEmit.Txq,     typeof(OpCodeTex));
             Set("1101111101010x", InstEmit.TxqB,    typeof(OpCodeTex));
             Set("01011111xxxxxx", InstEmit.Vmad,    typeof(OpCodeVideo));
+            Set("0101000011011x", InstEmit.Vote,    typeof(OpCodeVote));
             Set("0100111xxxxxxx", InstEmit.Xmad,    typeof(OpCodeAluCbuf));
             Set("0011011x00xxxx", InstEmit.Xmad,    typeof(OpCodeAluImm));
             Set("010100010xxxxx", InstEmit.Xmad,    typeof(OpCodeAluRegCbuf));

Ryujinx.Graphics.Shader/Decoders/OpCodeVote.cs

@@ -0,0 +1,26 @@
+using Ryujinx.Graphics.Shader.Instructions;
+
+namespace Ryujinx.Graphics.Shader.Decoders
+{
+    class OpCodeVote : OpCode, IOpCodeRd, IOpCodePredicate39
+    {
+        public Register Rd          { get; }
+        public Register Predicate39 { get; }
+        public Register Predicate45 { get; }
+
+        public VoteOp VoteOp { get; }
+
+        public bool InvertP { get; }
+
+        public OpCodeVote(InstEmitter emitter, ulong address, long opCode) : base(emitter, address, opCode)
+        {
+            Rd          = new Register(opCode.Extract(0,  8), RegisterType.Gpr);
+            Predicate39 = new Register(opCode.Extract(39, 3), RegisterType.Predicate);
+            Predicate45 = new Register(opCode.Extract(45, 3), RegisterType.Predicate);
+
+            InvertP = opCode.Extract(42);
+
+            VoteOp = (VoteOp)opCode.Extract(48, 2);
+        }
+    }
+}

Ryujinx.Graphics.Shader/Decoders/ReductionType.cs

@@ -0,0 +1,12 @@
+namespace Ryujinx.Graphics.Shader.Decoders
+{
+    enum ReductionType
+    {
+        U32       = 0,
+        S32       = 1,
+        U64       = 2,
+        FP32FtzRn = 3,
+        U128      = 4,
+        S64       = 5
+    }
+}

Ryujinx.Graphics.Shader/Decoders/SystemRegister.cs

@@ -2,6 +2,7 @@ namespace Ryujinx.Graphics.Shader.Decoders
 {
     enum SystemRegister
     {
+        LaneId     = 0,
         YDirection = 0x12,
         ThreadId   = 0x20,
         ThreadIdX  = 0x21,
@@ -9,6 +10,11 @@ namespace Ryujinx.Graphics.Shader.Decoders
         ThreadIdZ  = 0x23,
         CtaIdX     = 0x25,
         CtaIdY     = 0x26,
-        CtaIdZ     = 0x27
+        CtaIdZ     = 0x27,
+        EqMask     = 0x38,
+        LtMask     = 0x39,
+        LeMask     = 0x3a,
+        GtMask     = 0x3b,
+        GeMask     = 0x3c
     }
 }

Ryujinx.Graphics.Shader/Decoders/VoteOp.cs

@@ -0,0 +1,9 @@
+namespace Ryujinx.Graphics.Shader.Decoders
+{
+    enum VoteOp
+    {
+        All      = 0,
+        Any      = 1,
+        AllEqual = 2
+    }
+}

Ryujinx.Graphics.Shader/Instructions/InstEmitAlu.cs

@@ -368,6 +368,19 @@ namespace Ryujinx.Graphics.Shader.Instructions
             SetZnFlags(context, dest, op.SetCondCode, op.Extended);
         }
 
+        public static void Popc(EmitterContext context)
+        {
+            OpCodeAlu op = (OpCodeAlu)context.CurrOp;
+
+            bool invert = op.RawOpCode.Extract(40);
+
+            Operand srcB = context.BitwiseNot(GetSrcB(context), invert);
+
+            Operand res = context.BitCount(srcB);
+
+            context.Copy(GetDest(context), res);
+        }
+
         public static void Psetp(EmitterContext context)
         {
             OpCodePsetp op = (OpCodePsetp)context.CurrOp;

Ryujinx.Graphics.Shader/Instructions/InstEmitHelper.cs

@@ -240,7 +240,7 @@ namespace Ryujinx.Graphics.Shader.Instructions
 
         public static Operand GetPredicate39(EmitterContext context)
         {
-            IOpCodeAlu op = (IOpCodeAlu)context.CurrOp;
+            IOpCodePredicate39 op = (IOpCodePredicate39)context.CurrOp;
 
             Operand local = Register(op.Predicate39);
 

Ryujinx.Graphics.Shader/Instructions/InstEmitMemory.cs

@@ -9,6 +9,13 @@ namespace Ryujinx.Graphics.Shader.Instructions
 {
     static partial class InstEmit
     {
+        private enum MemoryRegion
+        {
+            Global,
+            Local,
+            Shared
+        }
+
         public static void Ald(EmitterContext context)
         {
             OpCodeAttribute op = (OpCodeAttribute)context.CurrOp;
@@ -49,6 +56,21 @@ namespace Ryujinx.Graphics.Shader.Instructions
             }
         }
 
+        public static void Atoms(EmitterContext context)
+        {
+            OpCodeAtom op = (OpCodeAtom)context.CurrOp;
+
+            Operand mem = context.ShiftRightU32(GetSrcA(context), Const(2));
+
+            mem = context.IAdd(mem, Const(op.Offset));
+
+            Operand value = GetSrcB(context);
+
+            Operand res = EmitAtomicOp(context, Instruction.MrShared, op.AtomicOp, op.Type, mem, value);
+
+            context.Copy(GetDest(context), res);
+        }
+
         public static void Ipa(EmitterContext context)
         {
             OpCodeIpa op = (OpCodeIpa)context.CurrOp;
@@ -80,7 +102,7 @@ namespace Ryujinx.Graphics.Shader.Instructions
 
         public static void Ld(EmitterContext context)
         {
-            LoadLocalOrGlobal(context, isGlobal: false);
+            EmitLoad(context, MemoryRegion.Local);
         }
 
         public static void Ldc(EmitterContext context)
@@ -126,7 +148,12 @@ namespace Ryujinx.Graphics.Shader.Instructions
 
         public static void Ldg(EmitterContext context)
         {
-            LoadLocalOrGlobal(context, isGlobal: true);
+            EmitLoad(context, MemoryRegion.Global);
+        }
+
+        public static void Lds(EmitterContext context)
+        {
+            EmitLoad(context, MemoryRegion.Shared);
         }
 
         public static void Out(EmitterContext context)
@@ -152,17 +179,118 @@ namespace Ryujinx.Graphics.Shader.Instructions
             }
         }
 
+        public static void Red(EmitterContext context)
+        {
+            OpCodeRed op = (OpCodeRed)context.CurrOp;
+
+            Operand offset = context.IAdd(GetSrcA(context), Const(op.Offset));
+
+            Operand mem = context.ShiftRightU32(offset, Const(2));
+
+            EmitAtomicOp(context, Instruction.MrGlobal, op.AtomicOp, op.Type, mem, GetDest(context));
+        }
+
         public static void St(EmitterContext context)
         {
-            StoreLocalOrGlobal(context, isGlobal: false);
+            EmitStore(context, MemoryRegion.Local);
         }
 
         public static void Stg(EmitterContext context)
         {
-            StoreLocalOrGlobal(context, isGlobal: true);
+            EmitStore(context, MemoryRegion.Global);
+        }
+
+        public static void Sts(EmitterContext context)
+        {
+            EmitStore(context, MemoryRegion.Shared);
         }
 
-        private static void LoadLocalOrGlobal(EmitterContext context, bool isGlobal)
+        private static Operand EmitAtomicOp(
+            EmitterContext context,
+            Instruction    mr,
+            AtomicOp       op,
+            ReductionType  type,
+            Operand        mem,
+            Operand        value)
+        {
+            Operand res = null;
+
+            switch (op)
+            {
+                case AtomicOp.Add:
+                    if (type == ReductionType.S32 || type == ReductionType.U32)
+                    {
+                        res = context.AtomicAdd(mr, mem, value);
+                    }
+                    else
+                    {
+                        // Not supported or invalid.
+                    }
+                    break;
+                case AtomicOp.BitwiseAnd:
+                    if (type == ReductionType.S32 || type == ReductionType.U32)
+                    {
+                        res = context.AtomicAnd(mr, mem, value);
+                    }
+                    else
+                    {
+                        // Not supported or invalid.
+                    }
+                    break;
+                case AtomicOp.BitwiseExclusiveOr:
+                    if (type == ReductionType.S32 || type == ReductionType.U32)
+                    {
+                        res = context.AtomicXor(mr, mem, value);
+                    }
+                    else
+                    {
+                        // Not supported or invalid.
+                    }
+                    break;
+                case AtomicOp.BitwiseOr:
+                    if (type == ReductionType.S32 || type == ReductionType.U32)
+                    {
+                        res = context.AtomicOr(mr, mem, value);
+                    }
+                    else
+                    {
+                        // Not supported or invalid.
+                    }
+                    break;
+                case AtomicOp.Maximum:
+                    if (type == ReductionType.S32)
+                    {
+                        res = context.AtomicMaxS32(mr, mem, value);
+                    }
+                    else if (type == ReductionType.U32)
+                    {
+                        res = context.AtomicMaxU32(mr, mem, value);
+                    }
+                    else
+                    {
+                        // Not supported or invalid.
+                    }
+                    break;
+                case AtomicOp.Minimum:
+                    if (type == ReductionType.S32)
+                    {
+                        res = context.AtomicMinS32(mr, mem, value);
+                    }
+                    else if (type == ReductionType.U32)
+                    {
+                        res = context.AtomicMinU32(mr, mem, value);
+                    }
+                    else
+                    {
+                        // Not supported or invalid.
+                    }
+                    break;
+            }
+
+            return res;
+        }
+
+        private static void EmitLoad(EmitterContext context, MemoryRegion region)
         {
             OpCodeMemory op = (OpCodeMemory)context.CurrOp;
 
@@ -199,9 +327,14 @@ namespace Ryujinx.Graphics.Shader.Instructions
 
                 Operand offset = context.IAdd(wordOffset, Const(index));
 
-                Operand value = isGlobal
-                    ? context.LoadGlobal(offset)
-                    : context.LoadLocal (offset);
+                Operand value = null;
+
+                switch (region)
+                {
+                    case MemoryRegion.Global: value = context.LoadGlobal(offset); break;
+                    case MemoryRegion.Local:  value = context.LoadLocal (offset); break;
+                    case MemoryRegion.Shared: value = context.LoadShared(offset); break;
+                }
 
                 if (isSmallInt)
                 {
@@ -212,7 +345,7 @@ namespace Ryujinx.Graphics.Shader.Instructions
             }
         }
 
-        private static void StoreLocalOrGlobal(EmitterContext context, bool isGlobal)
+        private static void EmitStore(EmitterContext context, MemoryRegion region)
         {
             OpCodeMemory op = (OpCodeMemory)context.CurrOp;
 
@@ -241,31 +374,34 @@ namespace Ryujinx.Graphics.Shader.Instructions
             {
                 Register rd = new Register(op.Rd.Index + index, RegisterType.Gpr);
 
-                if (rd.IsRZ)
-                {
-                    break;
-                }
-
                 Operand value = Register(rd);
 
                 Operand offset = context.IAdd(wordOffset, Const(index));
 
                 if (isSmallInt)
                 {
-                    Operand word = isGlobal
-                         ? context.LoadGlobal(offset)
-                         : context.LoadLocal (offset);
+                    Operand word = null;
+
+                    switch (region)
+                    {
+                        case MemoryRegion.Global: word = context.LoadGlobal(offset); break;
+                        case MemoryRegion.Local:  word = context.LoadLocal (offset); break;
+                        case MemoryRegion.Shared: word = context.LoadShared(offset); break;
+                    }
 
                     value = InsertSmallInt(context, op.Size, bitOffset, word, value);
                 }
 
-                if (isGlobal)
+                switch (region)
                 {
-                    context.StoreGlobal(offset, value);
+                    case MemoryRegion.Global: context.StoreGlobal(offset, value); break;
+                    case MemoryRegion.Local:  context.StoreLocal (offset, value); break;
+                    case MemoryRegion.Shared: context.StoreShared(offset, value); break;
                 }
-                else
+
+                if (rd.IsRZ)
                 {
-                    context.StoreLocal(offset, value);
+                    break;
                 }
             }
         }

Ryujinx.Graphics.Shader/Instructions/InstEmitMove.cs

@@ -27,6 +27,8 @@ namespace Ryujinx.Graphics.Shader.Instructions
 
             switch (sysReg)
             {
+                case SystemRegister.LaneId: src = Attribute(AttributeConsts.LaneId); break;
+
                 // TODO: Use value from Y direction GPU register.
                 case SystemRegister.YDirection: src = ConstF(1); break;
 
@@ -50,6 +52,11 @@ namespace Ryujinx.Graphics.Shader.Instructions
                 case SystemRegister.CtaIdX:    src = Attribute(AttributeConsts.CtaIdX);    break;
                 case SystemRegister.CtaIdY:    src = Attribute(AttributeConsts.CtaIdY);    break;
                 case SystemRegister.CtaIdZ:    src = Attribute(AttributeConsts.CtaIdZ);    break;
+                case SystemRegister.EqMask:    src = Attribute(AttributeConsts.EqMask);    break;
+                case SystemRegister.LtMask:    src = Attribute(AttributeConsts.LtMask);    break;
+                case SystemRegister.LeMask:    src = Attribute(AttributeConsts.LeMask);    break;
+                case SystemRegister.GtMask:    src = Attribute(AttributeConsts.GtMask);    break;
+                case SystemRegister.GeMask:    src = Attribute(AttributeConsts.GeMask);    break;
 
                 default: src = Const(0); break;
             }

Ryujinx.Graphics.Shader/Instructions/InstEmitVote.cs

@@ -0,0 +1,48 @@
+using Ryujinx.Graphics.Shader.Decoders;
+using Ryujinx.Graphics.Shader.IntermediateRepresentation;
+using Ryujinx.Graphics.Shader.Translation;
+
+using static Ryujinx.Graphics.Shader.Instructions.InstEmitHelper;
+using static Ryujinx.Graphics.Shader.IntermediateRepresentation.OperandHelper;
+
+namespace Ryujinx.Graphics.Shader.Instructions
+{
+    static partial class InstEmit
+    {
+        public static void Vote(EmitterContext context)
+        {
+            OpCodeVote op = (OpCodeVote)context.CurrOp;
+
+            Operand pred = GetPredicate39(context);
+
+            Operand res = null;
+
+            switch (op.VoteOp)
+            {
+                case VoteOp.All:
+                    res = context.VoteAll(pred);
+                    break;
+                case VoteOp.Any:
+                    res = context.VoteAny(pred);
+                    break;
+                case VoteOp.AllEqual:
+                    res = context.VoteAllEqual(pred);
+                    break;
+            }
+
+            if (res != null)
+            {
+                context.Copy(Register(op.Predicate45), res);
+            }
+            else
+            {
+                // Invalid.
+            }
+
+            if (!op.Rd.IsRZ)
+            {
+                context.Copy(Register(op.Rd), context.Ballot(pred));
+            }
+        }
+    }
+}

Ryujinx.Graphics.Shader/IntermediateRepresentation/Instruction.cs

@@ -7,6 +7,17 @@ namespace Ryujinx.Graphics.Shader.IntermediateRepresentation
     {
         Absolute = 1,
         Add,
+        AtomicAdd,
+        AtomicAnd,
+        AtomicCompareAndSwap,
+        AtomicMinS32,
+        AtomicMinU32,
+        AtomicMaxS32,
+        AtomicMaxU32,
+        AtomicOr,
+        AtomicSwap,
+        AtomicXor,
+        Ballot,
         BitCount,
         BitfieldExtractS32,
         BitfieldExtractU32,
@@ -57,6 +68,7 @@ namespace Ryujinx.Graphics.Shader.IntermediateRepresentation
         LoadConstant,
         LoadGlobal,
         LoadLocal,
+        LoadShared,
         LoadStorage,
         LogarithmB2,
         LogicalAnd,
@@ -88,6 +100,7 @@ namespace Ryujinx.Graphics.Shader.IntermediateRepresentation
         SquareRoot,
         StoreGlobal,
         StoreLocal,
+        StoreShared,
         StoreStorage,
         Subtract,
         SwizzleAdd,
@@ -96,9 +109,44 @@ namespace Ryujinx.Graphics.Shader.IntermediateRepresentation
         Truncate,
         UnpackDouble2x32,
         UnpackHalf2x16,
+        VoteAll,
+        VoteAllEqual,
+        VoteAny,
 
         Count,
-        FP   = 1 << 16,
+
+        FP = 1 << 16,
+
+        MrShift = 17,
+
+        MrGlobal  = 0 << MrShift,
+        MrShared  = 1 << MrShift,
+        MrStorage = 2 << MrShift,
+        MrMask    = 3 << MrShift,
+
         Mask = 0xffff
     }
+
+    static class InstructionExtensions
+    {
+        public static bool IsAtomic(this Instruction inst)
+        {
+            switch (inst & Instruction.Mask)
+            {
+                case Instruction.AtomicAdd:
+                case Instruction.AtomicAnd:
+                case Instruction.AtomicCompareAndSwap:
+                case Instruction.AtomicMaxS32:
+                case Instruction.AtomicMaxU32:
+                case Instruction.AtomicMinS32:
+                case Instruction.AtomicMinU32:
+                case Instruction.AtomicOr:
+                case Instruction.AtomicSwap:
+                case Instruction.AtomicXor:
+                    return true;
+            }
+
+            return false;
+        }
+    }
 }

Ryujinx.Graphics.Shader/IntermediateRepresentation/Operation.cs

@@ -16,7 +16,7 @@ namespace Ryujinx.Graphics.Shader.IntermediateRepresentation
 
         public int SourcesCount => _sources.Length;
 
-        public int ComponentIndex { get; }
+        public int Index { get; }
 
         public Operation(Instruction inst, Operand dest, params Operand[] sources)
         {
@@ -39,11 +39,11 @@ namespace Ryujinx.Graphics.Shader.IntermediateRepresentation
 
         public Operation(
             Instruction      inst,
-            int              compIndex,
+            int              index,
             Operand          dest,
             params Operand[] sources) : this(inst, dest, sources)
         {
-            ComponentIndex = compIndex;
+            Index = index;
         }
 
         private Operand AssignDest(Operand dest)

Ryujinx.Graphics.Shader/Ryujinx.Graphics.Shader.csproj

@@ -1,6 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <ItemGroup>
+    <EmbeddedResource Include="CodeGen\Glsl\HelperFunctions\GlobalMemory.glsl" />
     <EmbeddedResource Include="CodeGen\Glsl\HelperFunctions\Shuffle.glsl" />
     <EmbeddedResource Include="CodeGen\Glsl\HelperFunctions\ShuffleDown.glsl" />
     <EmbeddedResource Include="CodeGen\Glsl\HelperFunctions\ShuffleUp.glsl" />

Ryujinx.Graphics.Shader/ShaderProgram.cs

@@ -1,3 +1,5 @@
+using System;
+
 namespace Ryujinx.Graphics.Shader
 {
     public class ShaderProgram
@@ -15,6 +17,11 @@ namespace Ryujinx.Graphics.Shader
             Code  = code;
         }
 
+        public void Prepend(string line)
+        {
+            Code = line + Environment.NewLine + Code;
+        }
+
         public void Replace(string name, string value)
         {
             Code = Code.Replace(name, value);

Ryujinx.Graphics.Shader/StructuredIr/AstOperation.cs

@@ -8,7 +8,7 @@ namespace Ryujinx.Graphics.Shader.StructuredIr
     {
         public Instruction Inst { get; }
 
-        public int ComponentMask { get; }
+        public int Index { get; }
 
         private IAstNode[] _sources;
 
@@ -24,12 +24,12 @@ namespace Ryujinx.Graphics.Shader.StructuredIr
                 AddUse(source, this);
             }
 
-            ComponentMask = 1;
+            Index = 0;
         }
 
-        public AstOperation(Instruction inst, int compMask, params IAstNode[] sources) : this(inst, sources)
+        public AstOperation(Instruction inst, int index, params IAstNode[] sources) : this(inst, sources)
         {
-            ComponentMask = compMask;
+            Index = index;
         }
 
         public IAstNode GetSource(int index)

Ryujinx.Graphics.Shader/StructuredIr/AstTextureOperation.cs

@@ -16,8 +16,8 @@ namespace Ryujinx.Graphics.Shader.StructuredIr
             TextureFlags      flags,
             int               handle,
             int               arraySize,
-            int               compMask,
-            params IAstNode[] sources) : base(inst, compMask, sources)
+            int               index,
+            params IAstNode[] sources) : base(inst, index, sources)
         {
             Type      = type;
             Flags     = flags;

Ryujinx.Graphics.Shader/StructuredIr/HelperFunctionsMask.cs

@@ -5,10 +5,11 @@ namespace Ryujinx.Graphics.Shader.StructuredIr
     [Flags]
     enum HelperFunctionsMask
     {
-        Shuffle     = 1 << 0,
-        ShuffleDown = 1 << 1,
-        ShuffleUp   = 1 << 2,
-        ShuffleXor  = 1 << 3,
-        SwizzleAdd  = 1 << 4
+        GlobalMemory = 1 << 0,
+        Shuffle      = 1 << 1,
+        ShuffleDown  = 1 << 2,
+        ShuffleUp    = 1 << 3,
+        ShuffleXor   = 1 << 4,
+        SwizzleAdd   = 1 << 5
     }
 }

Ryujinx.Graphics.Shader/StructuredIr/InstructionInfo.cs

@@ -25,8 +25,19 @@ namespace Ryujinx.Graphics.Shader.StructuredIr
             _infoTbl = new InstInfo[(int)Instruction.Count];
 
             //  Inst                                  Destination type     Source 1 type        Source 2 type        Source 3 type        Source 4 type
+            Add(Instruction.AtomicAdd,                VariableType.U32,    VariableType.U32,    VariableType.U32);
+            Add(Instruction.AtomicAnd,                VariableType.U32,    VariableType.U32,    VariableType.U32);
+            Add(Instruction.AtomicCompareAndSwap,     VariableType.U32,    VariableType.U32,    VariableType.U32,    VariableType.U32);
+            Add(Instruction.AtomicMaxS32,             VariableType.S32,    VariableType.S32,    VariableType.S32);
+            Add(Instruction.AtomicMaxU32,             VariableType.U32,    VariableType.U32,    VariableType.U32);
+            Add(Instruction.AtomicMinS32,             VariableType.S32,    VariableType.S32,    VariableType.S32);
+            Add(Instruction.AtomicMinU32,             VariableType.U32,    VariableType.U32,    VariableType.U32);
+            Add(Instruction.AtomicOr,                 VariableType.U32,    VariableType.U32,    VariableType.U32);
+            Add(Instruction.AtomicSwap,               VariableType.U32,    VariableType.U32,    VariableType.U32);
+            Add(Instruction.AtomicXor,                VariableType.U32,    VariableType.U32,    VariableType.U32);
             Add(Instruction.Absolute,                 VariableType.Scalar, VariableType.Scalar);
             Add(Instruction.Add,                      VariableType.Scalar, VariableType.Scalar, VariableType.Scalar);
+            Add(Instruction.Ballot,                   VariableType.U32,    VariableType.Bool);
             Add(Instruction.BitCount,                 VariableType.Int,    VariableType.Int);
             Add(Instruction.BitfieldExtractS32,       VariableType.S32,    VariableType.S32,    VariableType.S32,    VariableType.S32);
             Add(Instruction.BitfieldExtractU32,       VariableType.U32,    VariableType.U32,    VariableType.S32,    VariableType.S32);
@@ -69,9 +80,10 @@ namespace Ryujinx.Graphics.Shader.StructuredIr
             Add(Instruction.IsNan,                    VariableType.Bool,   VariableType.F32);
             Add(Instruction.LoadAttribute,            VariableType.F32,    VariableType.S32,    VariableType.S32);
             Add(Instruction.LoadConstant,             VariableType.F32,    VariableType.S32,    VariableType.S32);
-            Add(Instruction.LoadGlobal,               VariableType.F32,    VariableType.S32,    VariableType.S32);
+            Add(Instruction.LoadGlobal,               VariableType.U32,    VariableType.S32,    VariableType.S32);
             Add(Instruction.LoadLocal,                VariableType.F32,    VariableType.S32);
-            Add(Instruction.LoadStorage,              VariableType.F32,    VariableType.S32,    VariableType.S32);
+            Add(Instruction.LoadShared,               VariableType.U32,    VariableType.S32);
+            Add(Instruction.LoadStorage,              VariableType.U32,    VariableType.S32);
             Add(Instruction.LogarithmB2,              VariableType.Scalar, VariableType.Scalar);
             Add(Instruction.LogicalAnd,               VariableType.Bool,   VariableType.Bool,   VariableType.Bool);
             Add(Instruction.LogicalExclusiveOr,       VariableType.Bool,   VariableType.Bool,   VariableType.Bool);
@@ -95,15 +107,19 @@ namespace Ryujinx.Graphics.Shader.StructuredIr
             Add(Instruction.Round,                    VariableType.F32,    VariableType.F32);
             Add(Instruction.Sine,                     VariableType.Scalar, VariableType.Scalar);
             Add(Instruction.SquareRoot,               VariableType.Scalar, VariableType.Scalar);
-            Add(Instruction.StoreGlobal,              VariableType.None,   VariableType.S32,    VariableType.S32,    VariableType.F32);
+            Add(Instruction.StoreGlobal,              VariableType.None,   VariableType.S32,    VariableType.S32,    VariableType.U32);
             Add(Instruction.StoreLocal,               VariableType.None,   VariableType.S32,    VariableType.F32);
-            Add(Instruction.StoreStorage,             VariableType.None,   VariableType.S32,    VariableType.S32,    VariableType.F32);
+            Add(Instruction.StoreShared,              VariableType.None,   VariableType.S32,    VariableType.U32);
+            Add(Instruction.StoreStorage,             VariableType.None,   VariableType.S32,    VariableType.U32);
             Add(Instruction.Subtract,                 VariableType.Scalar, VariableType.Scalar, VariableType.Scalar);
             Add(Instruction.SwizzleAdd,               VariableType.F32,    VariableType.F32,    VariableType.F32,    VariableType.S32);
             Add(Instruction.TextureSample,            VariableType.F32);
             Add(Instruction.TextureSize,              VariableType.S32,    VariableType.S32,    VariableType.S32);
             Add(Instruction.Truncate,                 VariableType.F32,    VariableType.F32);
             Add(Instruction.UnpackHalf2x16,           VariableType.F32,    VariableType.U32);
+            Add(Instruction.VoteAll,                  VariableType.Bool,   VariableType.Bool);
+            Add(Instruction.VoteAllEqual,             VariableType.Bool,   VariableType.Bool);
+            Add(Instruction.VoteAny,                  VariableType.Bool,   VariableType.Bool);
         }
 
         private static void Add(Instruction inst, VariableType destType, params VariableType[] srcTypes)

Ryujinx.Graphics.Shader/StructuredIr/StructuredProgram.cs

@@ -51,8 +51,6 @@ namespace Ryujinx.Graphics.Shader.StructuredIr
                 sources[index] = context.GetOperandUse(operation.GetSource(index));
             }
 
-            int componentMask = 1 << operation.ComponentIndex;
-
             AstTextureOperation GetAstTextureOperation(TextureOperation texOp)
             {
                 return new AstTextureOperation(
@@ -61,7 +59,7 @@ namespace Ryujinx.Graphics.Shader.StructuredIr
                     texOp.Flags,
                     texOp.Handle,
                     4, // TODO: Non-hardcoded array size.
-                    componentMask,
+                    texOp.Index,
                     sources);
             }
 
@@ -80,16 +78,9 @@ namespace Ryujinx.Graphics.Shader.StructuredIr
 
                     context.Info.CBuffers.Add(slot.Value);
                 }
-                else if (inst == Instruction.LoadStorage)
+                else if (UsesStorage(inst))
                 {
-                    Operand slot = operation.GetSource(0);
-
-                    if (slot.Type != OperandType.Constant)
-                    {
-                        throw new InvalidOperationException("Found load or store with non-constant storage buffer slot.");
-                    }
-
-                    context.Info.SBuffers.Add(slot.Value);
+                    context.Info.SBuffers.Add(operation.Index);
                 }
 
                 AstAssignment assignment;
@@ -141,7 +132,7 @@ namespace Ryujinx.Graphics.Shader.StructuredIr
                 }
                 else if (!isCopy)
                 {
-                    source = new AstOperation(inst, componentMask, sources);
+                    source = new AstOperation(inst, operation.Index, sources);
                 }
                 else
                 {
@@ -166,19 +157,12 @@ namespace Ryujinx.Graphics.Shader.StructuredIr
             }
             else
             {
-                if (inst == Instruction.StoreStorage)
+                if (UsesStorage(inst))
                 {
-                    Operand slot = operation.GetSource(0);
-
-                    if (slot.Type != OperandType.Constant)
-                    {
-                        throw new InvalidOperationException("Found load or store with non-constant storage buffer slot.");
-                    }
-
-                    context.Info.SBuffers.Add(slot.Value);
+                    context.Info.SBuffers.Add(operation.Index);
                 }
 
-                context.AddNode(new AstOperation(inst, sources));
+                context.AddNode(new AstOperation(inst, operation.Index, sources));
             }
 
             // Those instructions needs to be emulated by using helper functions,
@@ -186,6 +170,10 @@ namespace Ryujinx.Graphics.Shader.StructuredIr
             // decide which helper functions are needed on the final generated code.
             switch (operation.Inst)
             {
+                case Instruction.LoadGlobal:
+                case Instruction.StoreGlobal:
+                    context.Info.HelperFunctionsMask |= HelperFunctionsMask.GlobalMemory;
+                    break;
                 case Instruction.Shuffle:
                     context.Info.HelperFunctionsMask |= HelperFunctionsMask.Shuffle;
                     break;
@@ -320,5 +308,15 @@ namespace Ryujinx.Graphics.Shader.StructuredIr
 
             throw new ArgumentException($"Unexpected instruction \"{inst}\".");
         }
+
+        private static bool UsesStorage(Instruction inst)
+        {
+            if (inst == Instruction.LoadStorage || inst == Instruction.StoreStorage)
+            {
+                return true;
+            }
+
+            return inst.IsAtomic() && (inst & Instruction.MrMask) == Instruction.MrStorage;
+        }
     }
 }

Ryujinx.Graphics.Shader/Translation/AttributeConsts.cs

@@ -42,5 +42,13 @@ namespace Ryujinx.Graphics.Shader.Translation
         public const int CtaIdX = 0x2000010;
         public const int CtaIdY = 0x2000014;
         public const int CtaIdZ = 0x2000018;
+
+        public const int LaneId = 0x2000020;
+
+        public const int EqMask = 0x2000024;
+        public const int GeMask = 0x2000028;
+        public const int GtMask = 0x200002c;
+        public const int LeMask = 0x2000030;
+        public const int LtMask = 0x2000034;
     }
 }

Ryujinx.Graphics.Shader/Translation/EmitterContextInsts.cs

@@ -6,6 +6,61 @@ namespace Ryujinx.Graphics.Shader.Translation
 {
     static class EmitterContextInsts
     {
+        public static Operand AtomicAdd(this EmitterContext context, Instruction mr, Operand a, Operand b)
+        {
+            return context.Add(Instruction.AtomicAdd | mr, Local(), a, b);
+        }
+
+        public static Operand AtomicAnd(this EmitterContext context, Instruction mr, Operand a, Operand b)
+        {
+            return context.Add(Instruction.AtomicAnd | mr, Local(), a, b);
+        }
+
+        public static Operand AtomicCompareAndSwap(this EmitterContext context, Instruction mr, Operand a, Operand b, Operand c)
+        {
+            return context.Add(Instruction.AtomicCompareAndSwap | mr, Local(), a, b, c);
+        }
+
+        public static Operand AtomicMaxS32(this EmitterContext context, Instruction mr, Operand a, Operand b)
+        {
+            return context.Add(Instruction.AtomicMaxS32 | mr, Local(), a, b);
+        }
+
+        public static Operand AtomicMaxU32(this EmitterContext context, Instruction mr, Operand a, Operand b)
+        {
+            return context.Add(Instruction.AtomicMaxU32 | mr, Local(), a, b);
+        }
+
+        public static Operand AtomicMinS32(this EmitterContext context, Instruction mr, Operand a, Operand b)
+        {
+            return context.Add(Instruction.AtomicMinS32 | mr, Local(), a, b);
+        }
+
+        public static Operand AtomicMinU32(this EmitterContext context, Instruction mr, Operand a, Operand b)
+        {
+            return context.Add(Instruction.AtomicMinU32 | mr, Local(), a, b);
+        }
+
+        public static Operand AtomicOr(this EmitterContext context, Instruction mr, Operand a, Operand b)
+        {
+            return context.Add(Instruction.AtomicOr | mr, Local(), a, b);
+        }
+
+        public static Operand AtomicSwap(this EmitterContext context, Instruction mr, Operand a, Operand b)
+        {
+            return context.Add(Instruction.AtomicSwap | mr, Local(), a, b);
+        }
+
+        public static Operand AtomicXor(this EmitterContext context, Instruction mr, Operand a, Operand b)
+        {
+            return context.Add(Instruction.AtomicXor | mr, Local(), a, b);
+        }
+
+        public static Operand Ballot(this EmitterContext context, Operand a)
+        {
+            return context.Add(Instruction.Ballot, Local(), a);
+        }
+
         public static Operand BitCount(this EmitterContext context, Operand a)
         {
             return context.Add(Instruction.BitCount, Local(), a);
@@ -411,6 +466,11 @@ namespace Ryujinx.Graphics.Shader.Translation
             return context.Add(Instruction.LoadLocal, Local(), a);
         }
 
+        public static Operand LoadShared(this EmitterContext context, Operand a)
+        {
+            return context.Add(Instruction.LoadShared, Local(), a);
+        }
+
         public static Operand PackHalf2x16(this EmitterContext context, Operand a, Operand b)
         {
             return context.Add(Instruction.PackHalf2x16, Local(), a, b);
@@ -468,6 +528,11 @@ namespace Ryujinx.Graphics.Shader.Translation
             return context.Add(Instruction.StoreLocal, null, a, b);
         }
 
+        public static Operand StoreShared(this EmitterContext context, Operand a, Operand b)
+        {
+            return context.Add(Instruction.StoreShared, null, a, b);
+        }
+
         public static Operand UnpackHalf2x16High(this EmitterContext context, Operand a)
         {
             return UnpackHalf2x16(context, a, 1);
@@ -486,5 +551,20 @@ namespace Ryujinx.Graphics.Shader.Translation
 
             return dest;
         }
+
+        public static Operand VoteAll(this EmitterContext context, Operand a)
+        {
+            return context.Add(Instruction.VoteAll, Local(), a);
+        }
+
+        public static Operand VoteAllEqual(this EmitterContext context, Operand a)
+        {
+            return context.Add(Instruction.VoteAllEqual, Local(), a);
+        }
+
+        public static Operand VoteAny(this EmitterContext context, Operand a)
+        {
+            return context.Add(Instruction.VoteAny, Local(), a);
+        }
     }
 }

Ryujinx.Graphics.Shader/Translation/Optimizations/ConstantFolding.cs

@@ -256,7 +256,7 @@ namespace Ryujinx.Graphics.Shader.Translation.Optimizations
         {
             int value = operation.GetSource(0).Value;
 
-            value = (value >> operation.ComponentIndex * 16) & 0xffff;
+            value = (value >> operation.Index * 16) & 0xffff;
 
             operation.TurnIntoCopy(ConstF(HalfConversion.HalfToSingle(value)));
         }

Ryujinx.Graphics.Shader/Translation/Optimizations/GlobalToStorage.cs

@@ -1,8 +1,6 @@
 using Ryujinx.Graphics.Shader.IntermediateRepresentation;
 using System.Collections.Generic;
 
-using static Ryujinx.Graphics.Shader.IntermediateRepresentation.OperandHelper;
-
 namespace Ryujinx.Graphics.Shader.Translation.Optimizations
 {
     static class GlobalToStorage
@@ -27,7 +25,8 @@ namespace Ryujinx.Graphics.Shader.Translation.Optimizations
                     continue;
                 }
 
-                if (operation.Inst == Instruction.LoadGlobal ||
+                if (operation.Inst.IsAtomic() ||
+                    operation.Inst == Instruction.LoadGlobal ||
                     operation.Inst == Instruction.StoreGlobal)
                 {
                     Operand source = operation.GetSource(0);
@@ -51,18 +50,31 @@ namespace Ryujinx.Graphics.Shader.Translation.Optimizations
 
             Operation storageOp;
 
-            if (operation.Inst == Instruction.LoadGlobal)
+            if (operation.Inst.IsAtomic())
+            {
+                Operand[] sources = new Operand[operation.SourcesCount];
+
+                for (int index = 0; index < operation.SourcesCount; index++)
+                {
+                    sources[index] = operation.GetSource(index);
+                }
+
+                Instruction inst = (operation.Inst & ~Instruction.MrMask) | Instruction.MrStorage;
+
+                storageOp = new Operation(inst, storageIndex, operation.Dest, sources);
+            }
+            else if (operation.Inst == Instruction.LoadGlobal)
             {
                 Operand source = operation.GetSource(0);
 
-                storageOp = new Operation(Instruction.LoadStorage, operation.Dest, Const(storageIndex), source);
+                storageOp = new Operation(Instruction.LoadStorage, storageIndex, operation.Dest, source);
             }
             else
             {
                 Operand src1 = operation.GetSource(0);
                 Operand src2 = operation.GetSource(1);
 
-                storageOp = new Operation(Instruction.StoreStorage, null, Const(storageIndex), src1, src2);
+                storageOp = new Operation(Instruction.StoreStorage, storageIndex, null, src1, src2);
             }
 
             for (int index = 0; index < operation.SourcesCount; index++)
@@ -114,6 +126,11 @@ namespace Ryujinx.Graphics.Shader.Translation.Optimizations
             return -1;
         }
 
+        public static int GetStorageCbOffset(ShaderStage stage, int slot)
+        {
+            return GetStorageBaseCbOffset(stage) + slot * StorageDescSize;
+        }
+
         private static int GetStorageBaseCbOffset(ShaderStage stage)
         {
             switch (stage)

Ryujinx.Graphics.Shader/Translation/Optimizations/Optimizer.cs

@@ -133,7 +133,7 @@ namespace Ryujinx.Graphics.Shader.Translation.Optimizations
 
                 if (operation.GetSource(0) == dest)
                 {
-                    operation.TurnIntoCopy(operation.ComponentIndex == 1 ? src1 : src0);
+                    operation.TurnIntoCopy(operation.Index == 1 ? src1 : src0);
 
                     modified = true;
                 }
@@ -251,7 +251,30 @@ namespace Ryujinx.Graphics.Shader.Translation.Optimizations
 
         private static bool IsUnused(INode node)
         {
-            return DestIsLocalVar(node) && node.Dest.UseOps.Count == 0;
+            return !HasSideEffects(node) && DestIsLocalVar(node) && node.Dest.UseOps.Count == 0;
+        }
+
+        private static bool HasSideEffects(INode node)
+        {
+            if (node is Operation operation)
+            {
+                switch (operation.Inst & Instruction.Mask)
+                {
+                    case Instruction.AtomicAdd:
+                    case Instruction.AtomicAnd:
+                    case Instruction.AtomicCompareAndSwap:
+                    case Instruction.AtomicMaxS32:
+                    case Instruction.AtomicMaxU32:
+                    case Instruction.AtomicMinS32:
+                    case Instruction.AtomicMinU32:
+                    case Instruction.AtomicOr:
+                    case Instruction.AtomicSwap:
+                    case Instruction.AtomicXor:
+                        return true;
+                }
+            }
+
+            return false;
         }
 
         private static bool DestIsLocalVar(INode node)

Ryujinx.Graphics.Shader/Translation/Translator.cs

@@ -15,25 +15,38 @@ namespace Ryujinx.Graphics.Shader.Translation
     {
         private const int HeaderSize = 0x50;
 
-        public static ShaderProgram Translate(Span<byte> code, TranslationConfig translationConfig)
+        public static Span<byte> ExtractCode(Span<byte> code, bool compute, out int headerSize)
         {
-            return Translate(code, Span<byte>.Empty, translationConfig);
+            if (compute)
+            {
+                headerSize = 0;
+            }
+            else
+            {
+                headerSize = HeaderSize;
+            }
+
+            Block[] cfg = Decoder.Decode(code, (ulong)headerSize);
+
+            ulong endAddress = 0;
+
+            foreach (Block block in cfg)
+            {
+                if (endAddress < block.EndAddress)
+                {
+                    endAddress = block.EndAddress;
+                }
+            }
+
+            return code.Slice(0, headerSize + (int)endAddress);
         }
 
-        public static ShaderProgram Translate(Span<byte> code, Span<byte> code2, TranslationConfig translationConfig)
+        public static ShaderProgram Translate(Span<byte> code, TranslationConfig translationConfig)
         {
             bool compute   = (translationConfig.Flags & TranslationFlags.Compute)   != 0;
             bool debugMode = (translationConfig.Flags & TranslationFlags.DebugMode) != 0;
 
-            Operation[] shaderOps = DecodeShader(code, compute, debugMode, out ShaderHeader header);
-
-            if (code2 != Span<byte>.Empty)
-            {
-                // Dual vertex shader.
-                Operation[] shaderOpsB = DecodeShader(code2, compute, debugMode, out header);
-
-                shaderOps = Combine(shaderOps, shaderOpsB);
-            }
+            Operation[] ops = DecodeShader(code, compute, debugMode, out ShaderHeader header);
 
             ShaderStage stage;
 
@@ -63,7 +76,29 @@ namespace Ryujinx.Graphics.Shader.Translation
                 maxOutputVertexCount,
                 outputTopology);
 
-            BasicBlock[] irBlocks = ControlFlowGraph.MakeCfg(shaderOps);
+            return Translate(ops, config);
+        }
+
+        public static ShaderProgram Translate(Span<byte> vpACode, Span<byte> vpBCode, TranslationConfig translationConfig)
+        {
+            bool debugMode = (translationConfig.Flags & TranslationFlags.DebugMode) != 0;
+
+            Operation[] vpAOps = DecodeShader(vpACode, compute: false, debugMode, out _);
+            Operation[] vpBOps = DecodeShader(vpBCode, compute: false, debugMode, out ShaderHeader header);
+
+            ShaderConfig config = new ShaderConfig(
+                header.Stage,
+                translationConfig.Flags,
+                translationConfig.MaxCBufferSize,
+                header.MaxOutputVertexCount,
+                header.OutputTopology);
+
+            return Translate(Combine(vpAOps, vpBOps), config);
+        }
+
+        private static ShaderProgram Translate(Operation[] ops, ShaderConfig config)
+        {
+            BasicBlock[] irBlocks = ControlFlowGraph.MakeCfg(ops);
 
             Dominance.FindDominators(irBlocks[0], irBlocks.Length);
 
@@ -71,7 +106,7 @@ namespace Ryujinx.Graphics.Shader.Translation
 
             Ssa.Rename(irBlocks);
 
-            Optimizer.Optimize(irBlocks, stage);
+            Optimizer.Optimize(irBlocks, config.Stage);
 
             StructuredProgramInfo sInfo = StructuredProgram.MakeStructuredProgram(irBlocks, config);
 
@@ -87,12 +122,7 @@ namespace Ryujinx.Graphics.Shader.Translation
 
             string glslCode = program.Code;
 
-            if (translationConfig.Version != 0)
-            {
-                glslCode = "// " + translationConfig.Version + Environment.NewLine + glslCode;
-            }
-
-            return new ShaderProgram(spInfo, stage, glslCode);
+            return new ShaderProgram(spInfo, config.Stage, glslCode);
         }
 
         private static Operation[] DecodeShader(Span<byte> code, bool compute, bool debugMode, out ShaderHeader header)