diff --git a/src/graphics/shader/recompiler/cfg/ShaderCFG.cpp b/src/graphics/shader/recompiler/cfg/ShaderCFG.cpp index d2ec9b9..6e4eaf5 100644 --- a/src/graphics/shader/recompiler/cfg/ShaderCFG.cpp +++ b/src/graphics/shader/recompiler/cfg/ShaderCFG.cpp @@ -897,15 +897,50 @@ bool IsSyntheticMergeForwarder(const Graph& graph, uint32_t block_id, uint32_t m block->terminator.true_block == merge; } -bool IsInsideLoopConstruct(const Graph& graph, const NaturalLoop& loop, uint32_t block_id) { - return block_id != UINT32_MAX && block_id != loop.merge && block_id != loop.continue_block && - graph.Dominates(loop.header, block_id) && - (loop.merge == UINT32_MAX || !graph.Dominates(loop.merge, block_id)); +const NaturalLoop* FindInnermostContainingLoop(const Graph& graph, uint32_t block_id) { + const NaturalLoop* innermost = nullptr; + for (const auto& loop: graph.natural_loops) { + if (Contains(loop.body_blocks, block_id) && + (innermost == nullptr || loop.body_blocks.size() < innermost->body_blocks.size())) { + innermost = &loop; + } + } + return innermost; } -bool SelectionMergeLeavesContainingLoop(const Graph& graph, uint32_t header, uint32_t merge) { +bool IsInsideLoopConstruct(const Graph& graph, const NaturalLoop& loop, uint32_t block_id) { + return block_id != UINT32_MAX && block_id != loop.merge && block_id != loop.continue_block && + graph.Dominates(loop.header, block_id) && !graph.Dominates(loop.merge, block_id); +} + +bool IsInnermostLoopControlConditional(const Graph& graph, const BasicBlock& block) { + if (block.terminator.kind != TerminatorKind::ConditionalBranch) { + return false; + } + const auto* loop = FindInnermostContainingLoop(graph, block.id); + if (loop == nullptr || loop->merge == UINT32_MAX || loop->continue_block == UINT32_MAX) { + return false; + } + const auto true_target = block.terminator.true_block; + const auto false_target = block.terminator.false_block; + if (block.id == loop->continue_block) { + const auto is_repeat_target = [&](uint32_t target) { + return target == loop->header || target == loop->merge; + }; + return is_repeat_target(true_target) && is_repeat_target(false_target); + } + const auto is_control_target = [&](uint32_t target) { + return target == loop->merge || target == loop->continue_block; + }; + return (is_control_target(true_target) && + (is_control_target(false_target) || + IsInsideLoopConstruct(graph, *loop, false_target))) || + (is_control_target(false_target) && IsInsideLoopConstruct(graph, *loop, true_target)); +} + +bool MergeLeavesContainingLoop(const Graph& graph, uint32_t header, uint32_t merge) { for (const auto& loop: graph.natural_loops) { - if (IsInsideLoopConstruct(graph, loop, header) && + if (loop.header != header && IsInsideLoopConstruct(graph, loop, header) && !IsInsideLoopConstruct(graph, loop, merge)) { return true; } @@ -980,7 +1015,9 @@ bool SplitSharedMergeBlock(Graph& graph, uint32_t merge, bool SplitOneLoopMerge(Graph& graph) { const auto& loops = graph.natural_loops; for (const auto& loop: loops) { - if (SplitSharedMergeBlock(graph, loop.merge, loop.body_blocks)) { + const auto construct_blocks = DominatedBlocks(graph, loop.header, loop.merge); + const auto force_split = MergeLeavesContainingLoop(graph, loop.header, loop.merge); + if (SplitSharedMergeBlock(graph, loop.merge, construct_blocks, force_split)) { return true; } } @@ -1001,11 +1038,14 @@ bool SplitOneSelectionMerge(Graph& graph) { Contains(loop_headers, block_id)) { continue; } + if (IsInnermostLoopControlConditional(graph, *block)) { + continue; + } const auto merge = graph.FindNearestCommonPostDominator(block->terminator.true_block, block->terminator.false_block); const auto construct_blocks = DominatedBlocks(graph, block_id, merge); - const auto force_split = SelectionMergeLeavesContainingLoop(graph, block_id, merge); + const auto force_split = MergeLeavesContainingLoop(graph, block_id, merge); if (SplitSharedMergeBlock(graph, merge, construct_blocks, force_split)) { return true; } @@ -1395,6 +1435,9 @@ bool Structurize(Graph& graph, std::string* error) { block.terminator.loop_header) { continue; } + if (IsInnermostLoopControlConditional(graph, block)) { + continue; + } const auto merge = graph.FindNearestCommonPostDominator(block.terminator.true_block, block.terminator.false_block); diff --git a/tests/shaderCfgTests.cpp b/tests/shaderCfgTests.cpp index 1cf89e5..5c66a0f 100644 --- a/tests/shaderCfgTests.cpp +++ b/tests/shaderCfgTests.cpp @@ -5394,7 +5394,224 @@ void TestNewShaderRecompilerCfgSharedOuterAndLoopMerge() { CheckSpirvBinaryValidates(result.spirv); } -void TestNewShaderRecompilerCfgLoopSharedContinueSelectionMerges() { +void TestNewShaderRecompilerCfgLoopEarlyBreakNoSelection() { + const uint32_t shader[] = { + EncodeSopc(0x0a, 0, 129), // loop: s_cmp_lt_u32 s0, 1 + EncodeSopp(0x04, 4), // loop exit -> end + EncodeSopc(0x06, 1, 1), // s_cmp_eq_u32 s1, s1 + EncodeSopp(0x04, 2), // early break -> same loop end + EncodeSop2(0x00, 0, 0, 129), // s_add_u32 s0, s0, 1 + EncodeSopp(0x02, 0xfffau), // backedge -> loop header + 0xbf810000u, + }; + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); + Check(Common::ContainsStr(result.ir_dump, "mode=structured"), + "loop early-break CFG did not stay on structured path"); + Check(SpirvInstructionOpcodeCount(result.spirv, 246) != 0, + "loop early-break SPIR-V lacks OpLoopMerge"); + Check(SpirvInstructionOpcodeCount(result.spirv, 247) == 0, + "loop early-break SPIR-V unexpectedly used OpSelectionMerge"); + Check(SpirvInstructionOpcodeCount(result.spirv, 251) == 0, + "loop early-break CFG unexpectedly used dispatcher OpSwitch"); + CheckSpirvBinaryValidates(result.spirv); +} + +void TestNewShaderRecompilerCfgNestedLoopNonlocalExitDispatcher() { + const uint32_t shader[] = { + EncodeSopc(0x0a, 0, 129), // outer loop: s_cmp_lt_u32 s0, 1 + EncodeSopp(0x04, 9), // outer exit -> end + EncodeSopc(0x0a, 1, 129), // inner loop: s_cmp_lt_u32 s1, 1 + EncodeSopp(0x04, 5), // inner exit -> outer continue + EncodeSopc(0x06, 2, 2), // s_cmp_eq_u32 s2, s2 + EncodeSopp(0x05, 5), // nonlocal exit -> outer end + EncodeSMovB32(3, 129), // inner work + EncodeSop2(0x00, 1, 1, 129), // s_add_u32 s1, s1, 1 + EncodeSopp(0x02, 0xfff9u), // inner backedge + EncodeSop2(0x00, 0, 0, 129), // outer continue: s_add_u32 s0, s0, 1 + EncodeSopp(0x02, 0xfff5u), // outer backedge + 0xbf810000u, + }; + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); + Check(Common::ContainsStr(result.ir_dump, "mode=dispatcher"), + "nested-loop nonlocal exit did not select dispatcher fallback"); + Check(SpirvInstructionOpcodeCount(result.spirv, 251) != 0, + "nested-loop nonlocal exit dispatcher SPIR-V lacks OpSwitch"); + CheckSpirvBinaryValidates(result.spirv); +} + +void TestNewShaderRecompilerCfgNestedLoopLocalExitNoSelection() { + const uint32_t shader[] = { + EncodeSopc(0x0a, 0, 129), // outer loop: s_cmp_lt_u32 s0, 1 + EncodeSopp(0x04, 6), // outer exit -> end + EncodeSopc(0x0a, 1, 129), // inner loop: s_cmp_lt_u32 s1, 1 + EncodeSopp(0x04, 2), // inner exit -> outer continue + EncodeSMovB32(2, 129), // inner work + EncodeSopp(0x02, 0xfffcu), // inner backedge + EncodeSop2(0x00, 0, 0, 129), // outer continue: s_add_u32 s0, s0, 1 + EncodeSopp(0x02, 0xfff8u), // outer backedge + 0xbf810000u, + }; + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); + Check(Common::ContainsStr(result.ir_dump, "mode=structured"), + "nested local loop exit did not stay on structured path"); + Check(SpirvInstructionOpcodeCount(result.spirv, 246) >= 2, + "nested local loop exit SPIR-V lacks both OpLoopMerge instructions"); + Check(SpirvInstructionOpcodeCount(result.spirv, 247) == 0, + "nested local loop exit SPIR-V unexpectedly used OpSelectionMerge"); + Check(SpirvInstructionOpcodeCount(result.spirv, 251) == 0, + "nested local loop exit unexpectedly used dispatcher OpSwitch"); + CheckSpirvBinaryValidates(result.spirv); +} + +void TestNewShaderRecompilerCfgNestedLoopExitTailMergeSplit() { + const uint32_t shader[] = { + EncodeSopc(0x0a, 0, 129), // outer loop: s_cmp_lt_u32 s0, 1 + EncodeSopp(0x04, 11), // outer exit -> end + EncodeSopc(0x06, 1, 1), // inner loop first exit condition + EncodeSopp(0x05, 3), // first inner exit -> tail A + EncodeSopc(0x06, 2, 2), // inner loop second exit condition + EncodeSopp(0x05, 3), // second inner exit -> tail B + EncodeSopp(0x02, 0xfffbu), // inner backedge + EncodeSMovB32(3, 129), // tail A + EncodeSopp(0x02, 2), // tail A -> outer continue + EncodeSMovB32(4, 129), // tail B + EncodeSopp(0x02, 0), // tail B -> outer continue + EncodeSop2(0x00, 0, 0, 129), // outer continue: s_add_u32 s0, s0, 1 + EncodeSopp(0x02, 0xfff3u), // outer backedge + 0xbf810000u, + }; + + ShaderRecompiler::Decoder::Program program; + std::string error; + Check(ShaderRecompiler::Decoder::DecodeProgram(std::span {shader}, program, &error), + error.c_str()); + + ShaderRecompiler::CFG::Graph graph; + Check(ShaderRecompiler::CFG::BuildGraph(program, graph, &error), error.c_str()); + const auto original_block_count = graph.blocks.size(); + Check(ShaderRecompiler::CFG::Structurize(graph, &error), error.c_str()); + Check(graph.blocks.size() > original_block_count, + "nested loop exit tails did not create a private inner merge"); + + const auto* outer_header = graph.FindBlockByPc(0); + const auto* inner_header = graph.FindBlockByPc(8); + Check(outer_header != nullptr && inner_header != nullptr && + outer_header->terminator.loop_header && inner_header->terminator.loop_header, + "nested loop exit-tail fixture did not retain both loop headers"); + Check(inner_header->terminator.merge_block != outer_header->terminator.continue_block, + "inner loop merge still aliases the outer continue target"); + const auto* inner_merge = graph.FindBlock(inner_header->terminator.merge_block); + Check(inner_merge != nullptr && inner_merge->inst_begin == inner_merge->inst_end && + inner_merge->terminator.kind == ShaderRecompiler::CFG::TerminatorKind::Branch && + inner_merge->terminator.true_block == outer_header->terminator.continue_block, + "private inner merge does not forward to the outer continue target"); +} + +void TestNewShaderRecompilerCfgMixedContinueNonmergeExitDispatcher() { + const uint32_t shader[] = { + EncodeSopc(0x06, 7, 7), // entry branch bypasses loop -> exit X + EncodeSopp(0x05, 5), // entry -> X + EncodeSopc(0x0a, 0, 129), // loop: s_cmp_lt_u32 s0, 1 + EncodeSopp(0x04, 5), // loop exit -> Y + EncodeSopc(0x06, 1, 1), // inner condition + EncodeSopp(0x05, 1), // nonmerge exit -> X, else continue + EncodeSopp(0x02, 0xfffbu), // loop backedge + EncodeSMovB32(2, 129), // X + EncodeSopp(0x02, 2), // X -> end + EncodeSMovB32(3, 129), // Y + EncodeSopp(0x02, 0), // Y -> end + 0xbf810000u, + }; + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); + Check(Common::ContainsStr(result.ir_dump, "mode=dispatcher"), + "mixed continue/nonmerge exit did not select dispatcher fallback"); + Check(SpirvInstructionOpcodeCount(result.spirv, 251) != 0, + "mixed continue/nonmerge exit dispatcher SPIR-V lacks OpSwitch"); + CheckSpirvBinaryValidates(result.spirv); +} + +void TestNewShaderRecompilerCfgConditionalLatchNoSelection() { + const uint32_t shader[] = { + EncodeSopp(0x02, 0), // loop header -> conditional block + EncodeSopc(0x06, 0, 0), // s_cmp_eq_u32 s0, s0 + EncodeSopp(0x05, 1), // loop exit -> end + EncodeSopp(0x02, 0xfffcu), // separate latch -> loop header + 0xbf810000u, + }; + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); + Check(Common::ContainsStr(result.ir_dump, "mode=structured"), + "conditional latch did not stay on structured path"); + Check(SpirvInstructionOpcodeCount(result.spirv, 246) != 0, + "conditional latch SPIR-V lacks OpLoopMerge"); + Check(SpirvInstructionOpcodeCount(result.spirv, 247) == 0, + "conditional latch SPIR-V unexpectedly used OpSelectionMerge"); + Check(SpirvInstructionOpcodeCount(result.spirv, 251) == 0, + "conditional latch unexpectedly used dispatcher OpSwitch"); + CheckSpirvBinaryValidates(result.spirv); +} + +void TestNewShaderRecompilerCfgDirectConditionalLatchNoSelection() { + const uint32_t shader[] = { + EncodeSopp(0x02, 0), // loop header -> conditional latch + EncodeSopc(0x06, 0, 0), // s_cmp_eq_u32 s0, s0 + EncodeSopp(0x05, 0xfffdu), // direct latch backedge -> loop header + 0xbf810000u, + }; + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); + Check(Common::ContainsStr(result.ir_dump, "mode=structured"), + "direct conditional latch did not stay on structured path"); + Check(SpirvInstructionOpcodeCount(result.spirv, 246) != 0, + "direct conditional latch SPIR-V lacks OpLoopMerge"); + Check(SpirvInstructionOpcodeCount(result.spirv, 247) == 0, + "direct conditional latch SPIR-V unexpectedly used OpSelectionMerge"); + Check(SpirvInstructionOpcodeCount(result.spirv, 251) == 0, + "direct conditional latch unexpectedly used dispatcher OpSwitch"); + CheckSpirvBinaryValidates(result.spirv); +} + +void TestNewShaderRecompilerCfgLoopEarlyContinuesNoSelection() { const uint32_t shader[] = { EncodeSMovB32(0, 128), // s0 = 0 EncodeSopc(0x0a, 0, 130), // loop: s_cmp_lt_u32 s0, 2 @@ -5419,15 +5636,13 @@ void TestNewShaderRecompilerCfgLoopSharedContinueSelectionMerges() { std::string error; Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); Check(Common::ContainsStr(result.ir_dump, "mode=structured"), - "shared loop continue selections should stay on structured path"); - Check(!Common::ContainsStr(result.ir_dump, "duplicate structured merge block"), - "shared loop continue selections were not split before structurization"); - Check(SpirvContainsOpcode(result.spirv, 246), - "shared loop continue selections SPIR-V lacks OpLoopMerge"); - Check(SpirvContainsOpcode(result.spirv, 247), - "shared loop continue selections SPIR-V lacks OpSelectionMerge"); - Check(!SpirvContainsOpcode(result.spirv, 251), - "shared loop continue selections unexpectedly used dispatcher OpSwitch"); + "loop early continues should stay on structured path"); + Check(SpirvInstructionOpcodeCount(result.spirv, 246) != 0, + "loop early continues SPIR-V lacks OpLoopMerge"); + Check(SpirvInstructionOpcodeCount(result.spirv, 247) == 0, + "loop early continues SPIR-V unexpectedly used OpSelectionMerge"); + Check(SpirvInstructionOpcodeCount(result.spirv, 251) == 0, + "loop early continues unexpectedly used dispatcher OpSwitch"); CheckSpirvBinaryValidates(result.spirv); } @@ -7014,7 +7229,6 @@ int main() { using namespace Libs::Graphics; EnsureConfigInitialized(); - TestResourceDescriptorClassification(); TestNativeShaderResourceDependencies(); TestNormalizedImageContracts(); @@ -7086,7 +7300,14 @@ int main() { TestNewShaderRecompilerCfgLoopHeaderBufferLoadDispatcher(); TestNewShaderRecompilerCfgLoopHeaderDsAppendConsumeDispatcher(); TestNewShaderRecompilerCfgSharedOuterAndLoopMerge(); - TestNewShaderRecompilerCfgLoopSharedContinueSelectionMerges(); + TestNewShaderRecompilerCfgLoopEarlyBreakNoSelection(); + TestNewShaderRecompilerCfgNestedLoopNonlocalExitDispatcher(); + TestNewShaderRecompilerCfgNestedLoopLocalExitNoSelection(); + TestNewShaderRecompilerCfgNestedLoopExitTailMergeSplit(); + TestNewShaderRecompilerCfgMixedContinueNonmergeExitDispatcher(); + TestNewShaderRecompilerCfgConditionalLatchNoSelection(); + TestNewShaderRecompilerCfgDirectConditionalLatchNoSelection(); + TestNewShaderRecompilerCfgLoopEarlyContinuesNoSelection(); TestNewShaderRecompilerCfgDuplicateMergeStructuredSplit(); TestNewShaderRecompilerCfgIrreducibleDispatcher(); TestNewShaderRecompilerExecMaskHelpers();