diff --git a/src/graphics/shader/recompiler/cfg/ShaderCFG.cpp b/src/graphics/shader/recompiler/cfg/ShaderCFG.cpp index 6e4eaf5..b46101d 100644 --- a/src/graphics/shader/recompiler/cfg/ShaderCFG.cpp +++ b/src/graphics/shader/recompiler/cfg/ShaderCFG.cpp @@ -871,19 +871,19 @@ std::vector DominatedBlocks(const Graph& graph, uint32_t header, return blocks; } -uint32_t AppendSyntheticMergeBlock(Graph& graph, uint32_t old_merge) { - const auto* merge = graph.FindBlock(old_merge); +uint32_t AppendSyntheticBranchBlock(Graph& graph, uint32_t target) { + const auto* target_block = graph.FindBlock(target); BasicBlock block; block.id = static_cast(graph.blocks.size()); - block.start_pc = merge != nullptr ? merge->start_pc : 0u; + block.start_pc = target_block != nullptr ? target_block->start_pc : 0u; block.end_pc = block.start_pc; - block.inst_begin = merge != nullptr ? merge->inst_begin : 0u; + block.inst_begin = target_block != nullptr ? target_block->inst_begin : 0u; block.inst_end = block.inst_begin; - block.successors = {old_merge}; + block.successors = {target}; block.terminator.kind = TerminatorKind::Branch; block.terminator.condition = BranchCondition::Always; - block.terminator.true_block = old_merge; + block.terminator.true_block = target; graph.blocks.push_back(std::move(block)); return graph.blocks.back().id; } @@ -948,6 +948,80 @@ bool MergeLeavesContainingLoop(const Graph& graph, uint32_t header, uint32_t mer return false; } +bool CanonicalizeNaturalLoops(Graph& graph, std::string* error) { + const auto rewrite_budget = graph.blocks.size() * 2u + 16u; + for (size_t rewrite = 0; rewrite < rewrite_budget; rewrite++) { + bool changed = false; + for (const auto& loop: graph.natural_loops) { + std::vector latches; + for (const auto& edge: graph.back_edges) { + if (edge.to == loop.header) { + AddUnique(latches, edge.from); + } + } + if (latches.size() <= 1u) { + continue; + } + + const auto continue_block = AppendSyntheticBranchBlock(graph, loop.header); + for (auto latch: latches) { + auto* block = graph.FindBlock(latch); + if (block != nullptr) { + ReplaceValue(block->successors, loop.header, continue_block); + ReplaceTerminatorTarget(block->terminator, loop.header, continue_block); + } + } + RebuildPredecessors(graph); + RecomputeAnalyses(graph); + changed = true; + break; + } + if (changed) { + continue; + } + + for (const auto& loop: graph.natural_loops) { + const auto* header = graph.FindBlock(loop.header); + const auto is_loop_control_target = [&](uint32_t target) { + return target == loop.merge || target == loop.continue_block; + }; + if (header == nullptr || header->terminator.kind != TerminatorKind::ConditionalBranch || + is_loop_control_target(header->terminator.true_block) || + is_loop_control_target(header->terminator.false_block) || + !Contains(loop.body_blocks, header->terminator.true_block) || + !Contains(loop.body_blocks, header->terminator.false_block)) { + continue; + } + + const auto old_header = loop.header; + const auto predecessors = header->predecessors; + const auto new_header = AppendSyntheticBranchBlock(graph, old_header); + for (auto pred: predecessors) { + auto* block = graph.FindBlock(pred); + if (block != nullptr) { + ReplaceValue(block->successors, old_header, new_header); + ReplaceTerminatorTarget(block->terminator, old_header, new_header); + } + } + if (graph.entry_block == old_header) { + graph.entry_block = new_header; + } + MoveBlockBefore(graph, new_header, old_header); + RebuildPredecessors(graph); + RecomputeAnalyses(graph); + changed = true; + break; + } + if (!changed) { + return true; + } + } + + SetFailure(graph, FailureKind::StructuredControlFlow, graph.entry_block, + "CFG loop canonicalization exceeded rewrite budget", error); + return false; +} + bool SplitSharedMergeBlock(Graph& graph, uint32_t merge, const std::vector& construct_blocks, bool force_split = false) { @@ -983,7 +1057,7 @@ bool SplitSharedMergeBlock(Graph& graph, uint32_t merge, return false; } - const auto synthetic_merge = AppendSyntheticMergeBlock(graph, merge); + const auto synthetic_merge = AppendSyntheticBranchBlock(graph, merge); auto* synthetic_block = graph.FindBlock(synthetic_merge); if (synthetic_block != nullptr) { synthetic_block->predecessors = predecessors_to_split; @@ -1393,6 +1467,9 @@ bool Structurize(Graph& graph, std::string* error) { return false; } + if (!CanonicalizeNaturalLoops(graph, error)) { + return false; + } if (!SplitSharedMergeBlocks(graph, error)) { return false; } diff --git a/tests/shaderCfgTests.cpp b/tests/shaderCfgTests.cpp index 5c66a0f..4f0cfbb 100644 --- a/tests/shaderCfgTests.cpp +++ b/tests/shaderCfgTests.cpp @@ -5646,6 +5646,104 @@ void TestNewShaderRecompilerCfgLoopEarlyContinuesNoSelection() { CheckSpirvBinaryValidates(result.spirv); } +void TestNewShaderRecompilerCfgConditionalLoopHeaderSelection() { + const uint32_t shader[] = { + EncodeSopc(0x06, 0, 0), // loop body selection condition + EncodeSopp(0x05, 2), // select path B + EncodeSMovB32(1, 129), // path A + EncodeSopp(0x02, 1), // path A -> join + EncodeSMovB32(2, 129), // path B + EncodeSMovB32(3, 129), // join + EncodeSopc(0x06, 4, 4), // repeat condition + EncodeSopp(0x05, 0xfff8u), // repeat -> guest header + 0xbf810000u, + }; + + ShaderRecompiler::Decoder::Program decoded; + std::string error; + Check(ShaderRecompiler::Decoder::DecodeProgram(std::span {shader}, decoded, &error), + error.c_str()); + ShaderRecompiler::CFG::Graph graph; + Check(ShaderRecompiler::CFG::BuildGraph(decoded, graph, &error), error.c_str()); + const auto original_block_count = graph.blocks.size(); + Check(ShaderRecompiler::CFG::Structurize(graph, &error), error.c_str()); + Check(graph.blocks.size() > original_block_count, + "conditional guest loop header did not create a synthetic header"); + + uint32_t loop_headers = 0; + uint32_t selection_headers = 0; + for (const auto& block: graph.blocks) { + if (block.terminator.loop_header) { + loop_headers++; + Check(block.inst_begin == block.inst_end && + block.terminator.kind == ShaderRecompiler::CFG::TerminatorKind::Branch, + "canonical loop header is not an empty unconditional block"); + } else if (block.terminator.kind == + ShaderRecompiler::CFG::TerminatorKind::ConditionalBranch && + block.terminator.merge_block != UINT32_MAX) { + selection_headers++; + } + } + Check(loop_headers == 1u && selection_headers == 1u, + "guest conditional was not separated from the loop header"); + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + ShaderRecompiler::CompileResult result; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); + Check(SpirvInstructionOpcodeCount(result.spirv, 246) == 1u, + "conditional loop-header SPIR-V has the wrong loop-merge count"); + Check(SpirvInstructionOpcodeCount(result.spirv, 247) == 1u, + "conditional loop-header SPIR-V has the wrong selection-merge count"); + Check(SpirvInstructionOpcodeCount(result.spirv, 251) == 0u, + "conditional loop-header unexpectedly used dispatcher OpSwitch"); + CheckSpirvBinaryValidates(result.spirv); +} + +void TestNewShaderRecompilerCfgMultipleLoopLatches() { + const uint32_t shader[] = { + EncodeSopc(0x0a, 0, 129), // loop condition + EncodeSopp(0x04, 5), // loop exit -> end + EncodeSopc(0x06, 1, 1), // early repeat condition + EncodeSopp(0x05, 0xfffcu), // early repeat -> header + EncodeSMovB32(2, 129), // body + EncodeSMovB32(3, 129), // body tail + EncodeSopp(0x02, 0xfff9u), // ordinary latch -> header + 0xbf810000u, + }; + + ShaderRecompiler::Decoder::Program decoded; + std::string error; + Check(ShaderRecompiler::Decoder::DecodeProgram(std::span {shader}, decoded, &error), + error.c_str()); + ShaderRecompiler::CFG::Graph graph; + Check(ShaderRecompiler::CFG::BuildGraph(decoded, graph, &error), error.c_str()); + const auto original_block_count = graph.blocks.size(); + Check(graph.back_edges.size() == 2u, "multiple-latch fixture lacks two native backedges"); + Check(ShaderRecompiler::CFG::Structurize(graph, &error), error.c_str()); + Check(graph.blocks.size() == original_block_count + 1u, + "multiple native latches did not create one synthetic continue"); + Check(graph.back_edges.size() == 1u && graph.natural_loops.size() == 1u, + "multiple native latches were not coalesced to one SPIR-V backedge"); + const auto& loop = graph.natural_loops.front(); + const auto* continue_block = graph.FindBlock(loop.continue_block); + Check(continue_block != nullptr && continue_block->inst_begin == continue_block->inst_end && + continue_block->predecessors.size() == 2u, + "canonical continue does not join both native latches"); + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + ShaderRecompiler::CompileResult result; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); + Check(SpirvInstructionOpcodeCount(result.spirv, 246) == 1u, + "multiple-latch SPIR-V has the wrong loop-merge count"); + Check(SpirvInstructionOpcodeCount(result.spirv, 247) == 0u, + "multiple-latch SPIR-V unexpectedly used a selection merge"); + Check(SpirvInstructionOpcodeCount(result.spirv, 251) == 0u, + "multiple-latch SPIR-V unexpectedly used dispatcher OpSwitch"); + CheckSpirvBinaryValidates(result.spirv); +} + void TestNewShaderRecompilerCfgDuplicateMergeStructuredSplit() { const uint32_t shader[] = { EncodeSopc(0x06, 0, 0), // s_cmp_eq_u32 s0, s0 @@ -7308,6 +7406,8 @@ int main() { TestNewShaderRecompilerCfgConditionalLatchNoSelection(); TestNewShaderRecompilerCfgDirectConditionalLatchNoSelection(); TestNewShaderRecompilerCfgLoopEarlyContinuesNoSelection(); + TestNewShaderRecompilerCfgConditionalLoopHeaderSelection(); + TestNewShaderRecompilerCfgMultipleLoopLatches(); TestNewShaderRecompilerCfgDuplicateMergeStructuredSplit(); TestNewShaderRecompilerCfgIrreducibleDispatcher(); TestNewShaderRecompilerExecMaskHelpers();