|
|
@@ -1,6 +1,7 @@
|
|
|
using Silk.NET.Vulkan;
|
|
|
using System;
|
|
|
using System.Collections.Generic;
|
|
|
+using System.Runtime.CompilerServices;
|
|
|
|
|
|
namespace Ryujinx.Graphics.Vulkan
|
|
|
{
|
|
|
@@ -8,22 +9,64 @@ namespace Ryujinx.Graphics.Vulkan
|
|
|
{
|
|
|
private const int MaxBarriersPerCall = 16;
|
|
|
|
|
|
+ private const AccessFlags BaseAccess = AccessFlags.ShaderReadBit | AccessFlags.ShaderWriteBit;
|
|
|
+ private const AccessFlags BufferAccess = AccessFlags.IndexReadBit | AccessFlags.VertexAttributeReadBit | AccessFlags.UniformReadBit;
|
|
|
+ private const AccessFlags CommandBufferAccess = AccessFlags.IndirectCommandReadBit;
|
|
|
+
|
|
|
private readonly VulkanRenderer _gd;
|
|
|
|
|
|
private readonly NativeArray<MemoryBarrier> _memoryBarrierBatch = new(MaxBarriersPerCall);
|
|
|
private readonly NativeArray<BufferMemoryBarrier> _bufferBarrierBatch = new(MaxBarriersPerCall);
|
|
|
private readonly NativeArray<ImageMemoryBarrier> _imageBarrierBatch = new(MaxBarriersPerCall);
|
|
|
|
|
|
- private readonly List<BarrierWithStageFlags<MemoryBarrier>> _memoryBarriers = new();
|
|
|
- private readonly List<BarrierWithStageFlags<BufferMemoryBarrier>> _bufferBarriers = new();
|
|
|
- private readonly List<BarrierWithStageFlags<ImageMemoryBarrier>> _imageBarriers = new();
|
|
|
+ private readonly List<BarrierWithStageFlags<MemoryBarrier, int>> _memoryBarriers = new();
|
|
|
+ private readonly List<BarrierWithStageFlags<BufferMemoryBarrier, int>> _bufferBarriers = new();
|
|
|
+ private readonly List<BarrierWithStageFlags<ImageMemoryBarrier, TextureStorage>> _imageBarriers = new();
|
|
|
private int _queuedBarrierCount;
|
|
|
|
|
|
+ private enum IncoherentBarrierType
|
|
|
+ {
|
|
|
+ None,
|
|
|
+ Texture,
|
|
|
+ All,
|
|
|
+ CommandBuffer
|
|
|
+ }
|
|
|
+
|
|
|
+ private PipelineStageFlags _incoherentBufferWriteStages;
|
|
|
+ private PipelineStageFlags _incoherentTextureWriteStages;
|
|
|
+ private PipelineStageFlags _extraStages;
|
|
|
+ private IncoherentBarrierType _queuedIncoherentBarrier;
|
|
|
+
|
|
|
public BarrierBatch(VulkanRenderer gd)
|
|
|
{
|
|
|
_gd = gd;
|
|
|
}
|
|
|
|
|
|
+ public static (AccessFlags Access, PipelineStageFlags Stages) GetSubpassAccessSuperset(VulkanRenderer gd)
|
|
|
+ {
|
|
|
+ AccessFlags access = BufferAccess;
|
|
|
+ PipelineStageFlags stages = PipelineStageFlags.AllGraphicsBit;
|
|
|
+
|
|
|
+ if (gd.TransformFeedbackApi != null)
|
|
|
+ {
|
|
|
+ access |= AccessFlags.TransformFeedbackWriteBitExt;
|
|
|
+ stages |= PipelineStageFlags.TransformFeedbackBitExt;
|
|
|
+ }
|
|
|
+
|
|
|
+ if (!gd.IsTBDR)
|
|
|
+ {
|
|
|
+ // Desktop GPUs can transform image barriers into memory barriers.
|
|
|
+
|
|
|
+ access |= AccessFlags.DepthStencilAttachmentWriteBit | AccessFlags.ColorAttachmentWriteBit;
|
|
|
+ access |= AccessFlags.DepthStencilAttachmentReadBit | AccessFlags.ColorAttachmentReadBit;
|
|
|
+
|
|
|
+ stages |= PipelineStageFlags.EarlyFragmentTestsBit | PipelineStageFlags.LateFragmentTestsBit;
|
|
|
+ stages |= PipelineStageFlags.ColorAttachmentOutputBit;
|
|
|
+ }
|
|
|
+
|
|
|
+ return (access, stages);
|
|
|
+ }
|
|
|
+
|
|
|
private readonly record struct StageFlags : IEquatable<StageFlags>
|
|
|
{
|
|
|
public readonly PipelineStageFlags Source;
|
|
|
@@ -36,47 +79,130 @@ namespace Ryujinx.Graphics.Vulkan
|
|
|
}
|
|
|
}
|
|
|
|
|
|
- private readonly struct BarrierWithStageFlags<T> where T : unmanaged
|
|
|
+ private readonly struct BarrierWithStageFlags<T, T2> where T : unmanaged
|
|
|
{
|
|
|
public readonly StageFlags Flags;
|
|
|
public readonly T Barrier;
|
|
|
+ public readonly T2 Resource;
|
|
|
|
|
|
public BarrierWithStageFlags(StageFlags flags, T barrier)
|
|
|
{
|
|
|
Flags = flags;
|
|
|
Barrier = barrier;
|
|
|
+ Resource = default;
|
|
|
}
|
|
|
|
|
|
- public BarrierWithStageFlags(PipelineStageFlags srcStageFlags, PipelineStageFlags dstStageFlags, T barrier)
|
|
|
+ public BarrierWithStageFlags(PipelineStageFlags srcStageFlags, PipelineStageFlags dstStageFlags, T barrier, T2 resource)
|
|
|
{
|
|
|
Flags = new StageFlags(srcStageFlags, dstStageFlags);
|
|
|
Barrier = barrier;
|
|
|
+ Resource = resource;
|
|
|
}
|
|
|
}
|
|
|
|
|
|
- private void QueueBarrier<T>(List<BarrierWithStageFlags<T>> list, T barrier, PipelineStageFlags srcStageFlags, PipelineStageFlags dstStageFlags) where T : unmanaged
|
|
|
+ private void QueueBarrier<T, T2>(List<BarrierWithStageFlags<T, T2>> list, T barrier, T2 resource, PipelineStageFlags srcStageFlags, PipelineStageFlags dstStageFlags) where T : unmanaged
|
|
|
{
|
|
|
- list.Add(new BarrierWithStageFlags<T>(srcStageFlags, dstStageFlags, barrier));
|
|
|
+ list.Add(new BarrierWithStageFlags<T, T2>(srcStageFlags, dstStageFlags, barrier, resource));
|
|
|
_queuedBarrierCount++;
|
|
|
}
|
|
|
|
|
|
public void QueueBarrier(MemoryBarrier barrier, PipelineStageFlags srcStageFlags, PipelineStageFlags dstStageFlags)
|
|
|
{
|
|
|
- QueueBarrier(_memoryBarriers, barrier, srcStageFlags, dstStageFlags);
|
|
|
+ QueueBarrier(_memoryBarriers, barrier, default, srcStageFlags, dstStageFlags);
|
|
|
}
|
|
|
|
|
|
public void QueueBarrier(BufferMemoryBarrier barrier, PipelineStageFlags srcStageFlags, PipelineStageFlags dstStageFlags)
|
|
|
{
|
|
|
- QueueBarrier(_bufferBarriers, barrier, srcStageFlags, dstStageFlags);
|
|
|
+ QueueBarrier(_bufferBarriers, barrier, default, srcStageFlags, dstStageFlags);
|
|
|
}
|
|
|
|
|
|
- public void QueueBarrier(ImageMemoryBarrier barrier, PipelineStageFlags srcStageFlags, PipelineStageFlags dstStageFlags)
|
|
|
+ public void QueueBarrier(ImageMemoryBarrier barrier, TextureStorage resource, PipelineStageFlags srcStageFlags, PipelineStageFlags dstStageFlags)
|
|
|
{
|
|
|
- QueueBarrier(_imageBarriers, barrier, srcStageFlags, dstStageFlags);
|
|
|
+ QueueBarrier(_imageBarriers, barrier, resource, srcStageFlags, dstStageFlags);
|
|
|
}
|
|
|
|
|
|
- public unsafe void Flush(CommandBuffer cb, bool insideRenderPass, Action endRenderPass)
|
|
|
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
|
|
|
+ public unsafe void FlushMemoryBarrier(ShaderCollection program, bool inRenderPass)
|
|
|
{
|
|
|
+ if (_queuedIncoherentBarrier > IncoherentBarrierType.None)
|
|
|
+ {
|
|
|
+ // We should emit a memory barrier if there's a write access in the program (current program, or program since last barrier)
|
|
|
+ bool hasTextureWrite = _incoherentTextureWriteStages != PipelineStageFlags.None;
|
|
|
+ bool hasBufferWrite = _incoherentBufferWriteStages != PipelineStageFlags.None;
|
|
|
+ bool hasBufferBarrier = _queuedIncoherentBarrier > IncoherentBarrierType.Texture;
|
|
|
+
|
|
|
+ if (hasTextureWrite || (hasBufferBarrier && hasBufferWrite))
|
|
|
+ {
|
|
|
+ AccessFlags access = BaseAccess;
|
|
|
+
|
|
|
+ PipelineStageFlags stages = inRenderPass ? PipelineStageFlags.AllGraphicsBit : PipelineStageFlags.AllCommandsBit;
|
|
|
+
|
|
|
+ if (hasBufferBarrier && hasBufferWrite)
|
|
|
+ {
|
|
|
+ access |= BufferAccess;
|
|
|
+
|
|
|
+ if (_gd.TransformFeedbackApi != null)
|
|
|
+ {
|
|
|
+ access |= AccessFlags.TransformFeedbackWriteBitExt;
|
|
|
+ stages |= PipelineStageFlags.TransformFeedbackBitExt;
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ if (_queuedIncoherentBarrier == IncoherentBarrierType.CommandBuffer)
|
|
|
+ {
|
|
|
+ access |= CommandBufferAccess;
|
|
|
+ stages |= PipelineStageFlags.DrawIndirectBit;
|
|
|
+ }
|
|
|
+
|
|
|
+ MemoryBarrier barrier = new MemoryBarrier()
|
|
|
+ {
|
|
|
+ SType = StructureType.MemoryBarrier,
|
|
|
+ SrcAccessMask = access,
|
|
|
+ DstAccessMask = access
|
|
|
+ };
|
|
|
+
|
|
|
+ QueueBarrier(barrier, stages, stages);
|
|
|
+
|
|
|
+ _incoherentTextureWriteStages = program?.IncoherentTextureWriteStages ?? PipelineStageFlags.None;
|
|
|
+
|
|
|
+ if (_queuedIncoherentBarrier > IncoherentBarrierType.Texture)
|
|
|
+ {
|
|
|
+ if (program != null)
|
|
|
+ {
|
|
|
+ _incoherentBufferWriteStages = program.IncoherentBufferWriteStages | _extraStages;
|
|
|
+ }
|
|
|
+ else
|
|
|
+ {
|
|
|
+ _incoherentBufferWriteStages = PipelineStageFlags.None;
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ _queuedIncoherentBarrier = IncoherentBarrierType.None;
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ public unsafe void Flush(CommandBufferScoped cbs, bool inRenderPass, RenderPassHolder rpHolder, Action endRenderPass)
|
|
|
+ {
|
|
|
+ Flush(cbs, null, inRenderPass, rpHolder, endRenderPass);
|
|
|
+ }
|
|
|
+
|
|
|
+ public unsafe void Flush(CommandBufferScoped cbs, ShaderCollection program, bool inRenderPass, RenderPassHolder rpHolder, Action endRenderPass)
|
|
|
+ {
|
|
|
+ if (program != null)
|
|
|
+ {
|
|
|
+ _incoherentBufferWriteStages |= program.IncoherentBufferWriteStages | _extraStages;
|
|
|
+ _incoherentTextureWriteStages |= program.IncoherentTextureWriteStages;
|
|
|
+ }
|
|
|
+
|
|
|
+ FlushMemoryBarrier(program, inRenderPass);
|
|
|
+
|
|
|
+ if (!inRenderPass && rpHolder != null)
|
|
|
+ {
|
|
|
+ // Render pass is about to begin. Queue any fences that normally interrupt the pass.
|
|
|
+ rpHolder.InsertForcedFences(cbs);
|
|
|
+ }
|
|
|
+
|
|
|
while (_queuedBarrierCount > 0)
|
|
|
{
|
|
|
int memoryCount = 0;
|
|
|
@@ -86,20 +212,20 @@ namespace Ryujinx.Graphics.Vulkan
|
|
|
bool hasBarrier = false;
|
|
|
StageFlags flags = default;
|
|
|
|
|
|
- static void AddBarriers<T>(
|
|
|
+ static void AddBarriers<T, T2>(
|
|
|
Span<T> target,
|
|
|
ref int queuedBarrierCount,
|
|
|
ref bool hasBarrier,
|
|
|
ref StageFlags flags,
|
|
|
ref int count,
|
|
|
- List<BarrierWithStageFlags<T>> list) where T : unmanaged
|
|
|
+ List<BarrierWithStageFlags<T, T2>> list) where T : unmanaged
|
|
|
{
|
|
|
int firstMatch = -1;
|
|
|
int end = list.Count;
|
|
|
|
|
|
for (int i = 0; i < list.Count; i++)
|
|
|
{
|
|
|
- BarrierWithStageFlags<T> barrier = list[i];
|
|
|
+ BarrierWithStageFlags<T, T2> barrier = list[i];
|
|
|
|
|
|
if (!hasBarrier)
|
|
|
{
|
|
|
@@ -162,21 +288,60 @@ namespace Ryujinx.Graphics.Vulkan
|
|
|
}
|
|
|
}
|
|
|
|
|
|
- if (insideRenderPass)
|
|
|
+ if (inRenderPass && _imageBarriers.Count > 0)
|
|
|
{
|
|
|
// Image barriers queued in the batch are meant to be globally scoped,
|
|
|
// but inside a render pass they're scoped to just the range of the render pass.
|
|
|
|
|
|
// On MoltenVK, we just break the rules and always use image barrier.
|
|
|
// On desktop GPUs, all barriers are globally scoped, so we just replace it with a generic memory barrier.
|
|
|
- // TODO: On certain GPUs, we need to split render pass so the barrier scope is global. When this is done,
|
|
|
- // notify the resource that it should add a barrier as soon as a render pass ends to avoid this in future.
|
|
|
+ // Generally, we want to avoid this from happening in the future, so flag the texture to immediately
|
|
|
+ // emit a barrier whenever the current render pass is bound again.
|
|
|
+
|
|
|
+ bool anyIsNonAttachment = false;
|
|
|
+
|
|
|
+ foreach (BarrierWithStageFlags<ImageMemoryBarrier, TextureStorage> barrier in _imageBarriers)
|
|
|
+ {
|
|
|
+ // If the binding is an attachment, don't add it as a forced fence.
|
|
|
+ bool isAttachment = rpHolder.ContainsAttachment(barrier.Resource);
|
|
|
+
|
|
|
+ if (!isAttachment)
|
|
|
+ {
|
|
|
+ rpHolder.AddForcedFence(barrier.Resource, barrier.Flags.Dest);
|
|
|
+ anyIsNonAttachment = true;
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ if (_gd.IsTBDR)
|
|
|
+ {
|
|
|
+ if (!_gd.IsMoltenVk)
|
|
|
+ {
|
|
|
+ if (!anyIsNonAttachment)
|
|
|
+ {
|
|
|
+ // This case is a feedback loop. To prevent this from causing an absolute performance disaster,
|
|
|
+ // remove the barriers entirely.
|
|
|
+ // If this is not here, there will be a lot of single draw render passes.
|
|
|
+ // TODO: explicit handling for feedback loops, likely outside this class.
|
|
|
|
|
|
- if (!_gd.IsMoltenVk)
|
|
|
+ _queuedBarrierCount -= _imageBarriers.Count;
|
|
|
+ _imageBarriers.Clear();
|
|
|
+ }
|
|
|
+ else
|
|
|
+ {
|
|
|
+ // TBDR GPUs are sensitive to barriers, so we need to end the pass to ensure the data is available.
|
|
|
+ // Metal already has hazard tracking so MVK doesn't need this.
|
|
|
+ endRenderPass();
|
|
|
+ inRenderPass = false;
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ else
|
|
|
{
|
|
|
+ // Generic pipeline memory barriers will work for desktop GPUs.
|
|
|
+ // They do require a few more access flags on the subpass dependency, though.
|
|
|
foreach (var barrier in _imageBarriers)
|
|
|
{
|
|
|
- _memoryBarriers.Add(new BarrierWithStageFlags<MemoryBarrier>(
|
|
|
+ _memoryBarriers.Add(new BarrierWithStageFlags<MemoryBarrier, int>(
|
|
|
barrier.Flags,
|
|
|
new MemoryBarrier()
|
|
|
{
|
|
|
@@ -190,6 +355,22 @@ namespace Ryujinx.Graphics.Vulkan
|
|
|
}
|
|
|
}
|
|
|
|
|
|
+ if (inRenderPass && _memoryBarriers.Count > 0)
|
|
|
+ {
|
|
|
+ PipelineStageFlags allFlags = PipelineStageFlags.None;
|
|
|
+
|
|
|
+ foreach (var barrier in _memoryBarriers)
|
|
|
+ {
|
|
|
+ allFlags |= barrier.Flags.Dest;
|
|
|
+ }
|
|
|
+
|
|
|
+ if (allFlags.HasFlag(PipelineStageFlags.DrawIndirectBit) || !_gd.SupportsRenderPassBarrier(allFlags))
|
|
|
+ {
|
|
|
+ endRenderPass();
|
|
|
+ inRenderPass = false;
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
AddBarriers(_memoryBarrierBatch.AsSpan(), ref _queuedBarrierCount, ref hasBarrier, ref flags, ref memoryCount, _memoryBarriers);
|
|
|
AddBarriers(_bufferBarrierBatch.AsSpan(), ref _queuedBarrierCount, ref hasBarrier, ref flags, ref bufferCount, _bufferBarriers);
|
|
|
AddBarriers(_imageBarrierBatch.AsSpan(), ref _queuedBarrierCount, ref hasBarrier, ref flags, ref imageCount, _imageBarriers);
|
|
|
@@ -198,14 +379,14 @@ namespace Ryujinx.Graphics.Vulkan
|
|
|
{
|
|
|
PipelineStageFlags srcStageFlags = flags.Source;
|
|
|
|
|
|
- if (insideRenderPass)
|
|
|
+ if (inRenderPass)
|
|
|
{
|
|
|
// Inside a render pass, barrier stages can only be from rasterization.
|
|
|
srcStageFlags &= ~PipelineStageFlags.ComputeShaderBit;
|
|
|
}
|
|
|
|
|
|
_gd.Api.CmdPipelineBarrier(
|
|
|
- cb,
|
|
|
+ cbs.CommandBuffer,
|
|
|
srcStageFlags,
|
|
|
flags.Dest,
|
|
|
0,
|
|
|
@@ -219,6 +400,41 @@ namespace Ryujinx.Graphics.Vulkan
|
|
|
}
|
|
|
}
|
|
|
|
|
|
+ private void QueueIncoherentBarrier(IncoherentBarrierType type)
|
|
|
+ {
|
|
|
+ if (type > _queuedIncoherentBarrier)
|
|
|
+ {
|
|
|
+ _queuedIncoherentBarrier = type;
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ public void QueueTextureBarrier()
|
|
|
+ {
|
|
|
+ QueueIncoherentBarrier(IncoherentBarrierType.Texture);
|
|
|
+ }
|
|
|
+
|
|
|
+ public void QueueMemoryBarrier()
|
|
|
+ {
|
|
|
+ QueueIncoherentBarrier(IncoherentBarrierType.All);
|
|
|
+ }
|
|
|
+
|
|
|
+ public void QueueCommandBufferBarrier()
|
|
|
+ {
|
|
|
+ QueueIncoherentBarrier(IncoherentBarrierType.CommandBuffer);
|
|
|
+ }
|
|
|
+
|
|
|
+ public void EnableTfbBarriers(bool enable)
|
|
|
+ {
|
|
|
+ if (enable)
|
|
|
+ {
|
|
|
+ _extraStages |= PipelineStageFlags.TransformFeedbackBitExt;
|
|
|
+ }
|
|
|
+ else
|
|
|
+ {
|
|
|
+ _extraStages &= ~PipelineStageFlags.TransformFeedbackBitExt;
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
public void Dispose()
|
|
|
{
|
|
|
_memoryBarrierBatch.Dispose();
|