fix: guard Linux memory fixes to only Linux

Reserve only free guest address ranges

src/graphics/shader/recompiler/cfg/ShaderCFG.cpp

@@ -871,19 +871,19 @@ std::vector<uint32_t> DominatedBlocks(const Graph& graph, uint32_t header,
 	return blocks;
 }
 
-uint32_t AppendSyntheticMergeBlock(Graph& graph, uint32_t old_merge) {
+uint32_t AppendSyntheticBranchBlock(Graph& graph, uint32_t target) {
-	const auto* merge = graph.FindBlock(old_merge);
+	const auto* target_block = graph.FindBlock(target);
 
 	BasicBlock block;
 	block.id                    = static_cast<uint32_t>(graph.blocks.size());
-	block.start_pc              = merge != nullptr ? merge->start_pc : 0u;
+	block.start_pc              = target_block != nullptr ? target_block->start_pc : 0u;
 	block.end_pc                = block.start_pc;
-	block.inst_begin            = merge != nullptr ? merge->inst_begin : 0u;
+	block.inst_begin            = target_block != nullptr ? target_block->inst_begin : 0u;
 	block.inst_end              = block.inst_begin;
-	block.successors            = {old_merge};
+	block.successors            = {target};
 	block.terminator.kind       = TerminatorKind::Branch;
 	block.terminator.condition  = BranchCondition::Always;
-	block.terminator.true_block = old_merge;
+	block.terminator.true_block = target;
 	graph.blocks.push_back(std::move(block));
 	return graph.blocks.back().id;
 }
@@ -897,15 +897,50 @@ bool IsSyntheticMergeForwarder(const Graph& graph, uint32_t block_id, uint32_t m
 	       block->terminator.true_block == merge;
 }
 
-bool IsInsideLoopConstruct(const Graph& graph, const NaturalLoop& loop, uint32_t block_id) {
+const NaturalLoop* FindInnermostContainingLoop(const Graph& graph, uint32_t block_id) {
-	return block_id != UINT32_MAX && block_id != loop.merge && block_id != loop.continue_block &&
+	const NaturalLoop* innermost = nullptr;
-	       graph.Dominates(loop.header, block_id) &&
+	for (const auto& loop: graph.natural_loops) {
-	       (loop.merge == UINT32_MAX || !graph.Dominates(loop.merge, block_id));
+		if (Contains(loop.body_blocks, block_id) &&
+		    (innermost == nullptr || loop.body_blocks.size() < innermost->body_blocks.size())) {
+			innermost = &loop;
+		}
+	}
+	return innermost;
 }
 
-bool SelectionMergeLeavesContainingLoop(const Graph& graph, uint32_t header, uint32_t merge) {
+bool IsInsideLoopConstruct(const Graph& graph, const NaturalLoop& loop, uint32_t block_id) {
+	return block_id != UINT32_MAX && block_id != loop.merge && block_id != loop.continue_block &&
+	       graph.Dominates(loop.header, block_id) && !graph.Dominates(loop.merge, block_id);
+}
+
+bool IsInnermostLoopControlConditional(const Graph& graph, const BasicBlock& block) {
+	if (block.terminator.kind != TerminatorKind::ConditionalBranch) {
+		return false;
+	}
+	const auto* loop = FindInnermostContainingLoop(graph, block.id);
+	if (loop == nullptr || loop->merge == UINT32_MAX || loop->continue_block == UINT32_MAX) {
+		return false;
+	}
+	const auto true_target  = block.terminator.true_block;
+	const auto false_target = block.terminator.false_block;
+	if (block.id == loop->continue_block) {
+		const auto is_repeat_target = [&](uint32_t target) {
+			return target == loop->header || target == loop->merge;
+		};
+		return is_repeat_target(true_target) && is_repeat_target(false_target);
+	}
+	const auto is_control_target = [&](uint32_t target) {
+		return target == loop->merge || target == loop->continue_block;
+	};
+	return (is_control_target(true_target) &&
+	        (is_control_target(false_target) ||
+	         IsInsideLoopConstruct(graph, *loop, false_target))) ||
+	       (is_control_target(false_target) && IsInsideLoopConstruct(graph, *loop, true_target));
+}
+
+bool MergeLeavesContainingLoop(const Graph& graph, uint32_t header, uint32_t merge) {
 	for (const auto& loop: graph.natural_loops) {
-		if (IsInsideLoopConstruct(graph, loop, header) &&
+		if (loop.header != header && IsInsideLoopConstruct(graph, loop, header) &&
 		    !IsInsideLoopConstruct(graph, loop, merge)) {
 			return true;
 		}
@@ -913,6 +948,80 @@ bool SelectionMergeLeavesContainingLoop(const Graph& graph, uint32_t header, uin
 	return false;
 }
 
+bool CanonicalizeNaturalLoops(Graph& graph, std::string* error) {
+	const auto rewrite_budget = graph.blocks.size() * 2u + 16u;
+	for (size_t rewrite = 0; rewrite < rewrite_budget; rewrite++) {
+		bool changed = false;
+		for (const auto& loop: graph.natural_loops) {
+			std::vector<uint32_t> latches;
+			for (const auto& edge: graph.back_edges) {
+				if (edge.to == loop.header) {
+					AddUnique(latches, edge.from);
+				}
+			}
+			if (latches.size() <= 1u) {
+				continue;
+			}
+
+			const auto continue_block = AppendSyntheticBranchBlock(graph, loop.header);
+			for (auto latch: latches) {
+				auto* block = graph.FindBlock(latch);
+				if (block != nullptr) {
+					ReplaceValue(block->successors, loop.header, continue_block);
+					ReplaceTerminatorTarget(block->terminator, loop.header, continue_block);
+				}
+			}
+			RebuildPredecessors(graph);
+			RecomputeAnalyses(graph);
+			changed = true;
+			break;
+		}
+		if (changed) {
+			continue;
+		}
+
+		for (const auto& loop: graph.natural_loops) {
+			const auto* header                 = graph.FindBlock(loop.header);
+			const auto  is_loop_control_target = [&](uint32_t target) {
+				return target == loop.merge || target == loop.continue_block;
+			};
+			if (header == nullptr || header->terminator.kind != TerminatorKind::ConditionalBranch ||
+			    is_loop_control_target(header->terminator.true_block) ||
+			    is_loop_control_target(header->terminator.false_block) ||
+			    !Contains(loop.body_blocks, header->terminator.true_block) ||
+			    !Contains(loop.body_blocks, header->terminator.false_block)) {
+				continue;
+			}
+
+			const auto old_header   = loop.header;
+			const auto predecessors = header->predecessors;
+			const auto new_header   = AppendSyntheticBranchBlock(graph, old_header);
+			for (auto pred: predecessors) {
+				auto* block = graph.FindBlock(pred);
+				if (block != nullptr) {
+					ReplaceValue(block->successors, old_header, new_header);
+					ReplaceTerminatorTarget(block->terminator, old_header, new_header);
+				}
+			}
+			if (graph.entry_block == old_header) {
+				graph.entry_block = new_header;
+			}
+			MoveBlockBefore(graph, new_header, old_header);
+			RebuildPredecessors(graph);
+			RecomputeAnalyses(graph);
+			changed = true;
+			break;
+		}
+		if (!changed) {
+			return true;
+		}
+	}
+
+	SetFailure(graph, FailureKind::StructuredControlFlow, graph.entry_block,
+	           "CFG loop canonicalization exceeded rewrite budget", error);
+	return false;
+}
+
 bool SplitSharedMergeBlock(Graph& graph, uint32_t merge,
                            const std::vector<uint32_t>& construct_blocks,
                            bool                         force_split = false) {
@@ -948,7 +1057,7 @@ bool SplitSharedMergeBlock(Graph& graph, uint32_t merge,
 		return false;
 	}
 
-	const auto synthetic_merge = AppendSyntheticMergeBlock(graph, merge);
+	const auto synthetic_merge = AppendSyntheticBranchBlock(graph, merge);
 	auto*      synthetic_block = graph.FindBlock(synthetic_merge);
 	if (synthetic_block != nullptr) {
 		synthetic_block->predecessors = predecessors_to_split;
@@ -980,14 +1089,111 @@ bool SplitSharedMergeBlock(Graph& graph, uint32_t merge,
 bool SplitOneLoopMerge(Graph& graph) {
 	const auto& loops = graph.natural_loops;
 	for (const auto& loop: loops) {
-		if (SplitSharedMergeBlock(graph, loop.merge, loop.body_blocks)) {
+		const auto construct_blocks = DominatedBlocks(graph, loop.header, loop.merge);
+		const auto force_split      = MergeLeavesContainingLoop(graph, loop.header, loop.merge);
+		if (SplitSharedMergeBlock(graph, loop.merge, construct_blocks, force_split)) {
 			return true;
 		}
 	}
 	return false;
 }
 
-bool SplitOneSelectionMerge(Graph& graph) {
+std::vector<uint32_t> SelectionRegion(const Graph& graph, const BasicBlock& header,
+                                      uint32_t merge) {
+	std::vector<uint32_t> region;
+	std::vector<uint32_t> pending = {header.terminator.true_block,
+	                                 header.terminator.false_block};
+	while (!pending.empty()) {
+		const auto block_id = pending.back();
+		pending.pop_back();
+		if (block_id == merge || Contains(region, block_id)) {
+			continue;
+		}
+		const auto* block = graph.FindBlock(block_id);
+		if (block == nullptr) {
+			continue;
+		}
+		AddUnique(region, block_id);
+		pending.insert(pending.end(), block->successors.begin(), block->successors.end());
+	}
+	SortUnique(region);
+	return region;
+}
+
+bool DuplicateSelectionRegion(Graph& graph, uint32_t header_id, uint32_t merge,
+                              const std::vector<uint32_t>& region, uint32_t block_budget) {
+	std::vector<uint32_t> cloned_blocks;
+	for (auto block_id: region) {
+		if (!graph.Dominates(header_id, block_id)) {
+			cloned_blocks.push_back(block_id);
+		}
+	}
+	if (cloned_blocks.empty() || graph.FindBlock(header_id) == nullptr || header_id >= merge ||
+	    graph.blocks.size() + cloned_blocks.size() + 1u > block_budget) {
+		return false;
+	}
+
+	const auto first_clone = static_cast<uint32_t>(graph.blocks.size());
+	std::map<uint32_t, uint32_t> clones;
+	for (uint32_t i = 0; i < cloned_blocks.size(); i++) {
+		clones.emplace(cloned_blocks[i], first_clone + i);
+	}
+
+	for (auto block_id: cloned_blocks) {
+		BasicBlock clone = *graph.FindBlock(block_id);
+		clone.id         = clones.at(block_id);
+		clone.predecessors.clear();
+		clone.dominators.clear();
+		clone.post_dominators.clear();
+		graph.blocks.push_back(std::move(clone));
+	}
+
+	const auto remap_block = [&](BasicBlock& block) {
+		const auto remap_target = [&](uint32_t& target) {
+			if (const auto it = clones.find(target); it != clones.end()) {
+				target = it->second;
+			}
+		};
+		for (auto& successor: block.successors) {
+			remap_target(successor);
+		}
+		remap_target(block.terminator.true_block);
+		remap_target(block.terminator.false_block);
+		remap_target(block.terminator.merge_block);
+		remap_target(block.terminator.continue_block);
+		for (auto& target: block.terminator.indirect_targets) {
+			remap_target(target);
+		}
+	};
+	for (auto block_id: region) {
+		const auto owned_id = clones.contains(block_id) ? clones.at(block_id) : block_id;
+		remap_block(*graph.FindBlock(owned_id));
+	}
+
+	const auto private_merge = AppendSyntheticBranchBlock(graph, merge);
+	auto&      header        = *graph.FindBlock(header_id);
+	remap_block(header);
+
+	for (auto block_id: region) {
+		const auto owned_id = clones.contains(block_id) ? clones.at(block_id) : block_id;
+		auto*      block    = graph.FindBlock(owned_id);
+		if (block != nullptr) {
+			ReplaceValue(block->successors, merge, private_merge);
+			ReplaceTerminatorTarget(block->terminator, merge, private_merge);
+		}
+	}
+	ReplaceValue(header.successors, merge, private_merge);
+	ReplaceTerminatorTarget(header.terminator, merge, private_merge);
+
+	for (uint32_t i = 0; i <= cloned_blocks.size(); i++) {
+		MoveBlockBefore(graph, first_clone + i, merge + i);
+	}
+	RebuildPredecessors(graph);
+	RecomputeAnalyses(graph);
+	return true;
+}
+
+bool SplitOneSelectionMerge(Graph& graph, uint32_t block_budget) {
 	std::vector<uint32_t> loop_headers;
 	loop_headers.reserve(graph.natural_loops.size());
 	for (const auto& loop: graph.natural_loops) {
@@ -1001,11 +1207,26 @@ bool SplitOneSelectionMerge(Graph& graph) {
 		    Contains(loop_headers, block_id)) {
 			continue;
 		}
+		if (IsInnermostLoopControlConditional(graph, *block)) {
+			continue;
+		}
 
 		const auto merge = graph.FindNearestCommonPostDominator(block->terminator.true_block,
 		                                                        block->terminator.false_block);
+		if (merge == UINT32_MAX || graph.FindBlock(merge) == nullptr) {
+			continue;
+		}
+		const auto region = SelectionRegion(graph, *block, merge);
+		if (std::any_of(region.begin(), region.end(),
+		                [&](uint32_t member) { return !graph.Dominates(block_id, member); })) {
+			if (graph.natural_loops.empty() &&
+			    DuplicateSelectionRegion(graph, block_id, merge, region, block_budget)) {
+				return true;
+			}
+			continue;
+		}
 		const auto construct_blocks = DominatedBlocks(graph, block_id, merge);
-		const auto force_split      = SelectionMergeLeavesContainingLoop(graph, block_id, merge);
+		const auto force_split      = MergeLeavesContainingLoop(graph, block_id, merge);
 		if (SplitSharedMergeBlock(graph, merge, construct_blocks, force_split)) {
 			return true;
 		}
@@ -1015,10 +1236,12 @@ bool SplitOneSelectionMerge(Graph& graph) {
 
 bool SplitSharedMergeBlocks(Graph& graph, std::string* error) {
 	const auto original_block_count = static_cast<uint32_t>(graph.blocks.size());
-	const auto split_budget =
+	const auto split_budget = std::max<uint32_t>(
-	    std::max<uint32_t>(16u, std::min<uint32_t>(128u, original_block_count));
+	    16u, std::min<uint32_t>(128u, original_block_count * 4u));
+	const auto block_budget = std::max<uint32_t>(
+	    32u, std::min<uint32_t>(512u, original_block_count * 8u));
 	for (uint32_t splits = 0; splits < split_budget; splits++) {
-		if (!SplitOneLoopMerge(graph) && !SplitOneSelectionMerge(graph)) {
+		if (!SplitOneLoopMerge(graph) && !SplitOneSelectionMerge(graph, block_budget)) {
 			return true;
 		}
 		RebuildPredecessors(graph);
@@ -1353,6 +1576,9 @@ bool Structurize(Graph& graph, std::string* error) {
 		return false;
 	}
 
+	if (!CanonicalizeNaturalLoops(graph, error)) {
+		return false;
+	}
 	if (!SplitSharedMergeBlocks(graph, error)) {
 		return false;
 	}
@@ -1395,6 +1621,9 @@ bool Structurize(Graph& graph, std::string* error) {
 		    block.terminator.loop_header) {
 			continue;
 		}
+		if (IsInnermostLoopControlConditional(graph, block)) {
+			continue;
+		}
 
 		const auto merge = graph.FindNearestCommonPostDominator(block.terminator.true_block,
 		                                                        block.terminator.false_block);

src/kernel/memoryAddressSpace.inc

@@ -1058,6 +1058,76 @@ private:
 		    {HOST_SYSTEM_RESERVED_MIN, HOST_SYSTEM_RESERVED_MAX + 1u},
 		    {HOST_USER_MIN, HOST_USER_MAX + 1u},
 		}};
+
+#if KYTY_PLATFORM == KYTY_PLATFORM_LINUX && !defined(__APPLE__)
+		std::vector<std::pair<uint64_t, uint64_t>> occupied;
+		FILE*                                      maps = fopen("/proc/self/maps", "r");
+		EXIT_IF(maps == nullptr);
+
+		char line[512];
+		while (fgets(line, sizeof(line), maps) != nullptr) {
+			unsigned long long mapping_start = 0;
+			unsigned long long mapping_end   = 0;
+			if (sscanf(line, "%llx-%llx", &mapping_start, &mapping_end) == 2) {
+				occupied.emplace_back(static_cast<uint64_t>(mapping_start),
+				                      static_cast<uint64_t>(mapping_end));
+			}
+		}
+		fclose(maps);
+
+		auto reserve_range = [this](uint64_t start, uint64_t end) {
+			start = AlignUp(start, PageSize());
+			end   = AlignDown(end, PageSize());
+			if (start == 0 || end <= start) {
+				return;
+			}
+
+			const auto size = end - start;
+			int        flags = MAP_PRIVATE | MAP_ANON | MAP_NORESERVE;
+#if defined(KYTY_LINKED_GUEST_ADDRESS_SPACE)
+			flags |= MAP_FIXED;
+#elif defined(MAP_FIXED_NOREPLACE)
+			flags |= MAP_FIXED_NOREPLACE;
+#endif
+
+			void* ptr = mmap(reinterpret_cast<void*>(start), size, PROT_NONE, flags, -1, 0);
+			if (ptr == MAP_FAILED || reinterpret_cast<uint64_t>(ptr) != start) {
+				if (ptr != MAP_FAILED) {
+					munmap(ptr, size);
+				}
+				return;
+			}
+
+			AddFreeUnlocked(start, size);
+			m_owned.emplace_back(start, size);
+		};
+
+		for (const auto& [region_start, region_end]: regions) {
+			auto current = region_start;
+
+			for (const auto& [mapping_start, mapping_end]: occupied) {
+				if (mapping_end <= current) {
+					continue;
+				}
+				if (mapping_start >= region_end) {
+					break;
+				}
+
+				if (mapping_start > current) {
+					reserve_range(current, std::min(mapping_start, region_end));
+				}
+
+				current = std::max(current, mapping_end);
+				if (current >= region_end) {
+					break;
+				}
+			}
+
+			if (current < region_end) {
+				reserve_range(current, region_end);
+			}
+		}
+#else
 		for (const auto& [start, end]: regions) {
 			int flags = MAP_PRIVATE | MAP_ANON | MAP_NORESERVE;
 #if defined(KYTY_LINKED_GUEST_ADDRESS_SPACE)
@@ -1070,13 +1140,14 @@ private:
 				if (ptr != MAP_FAILED) {
 					munmap(ptr, end - start);
 				}
-				EXIT("failed to reserve guest address space at 0x%016" PRIx64 ", size 0x%016" PRIx64
+				EXIT("failed to reserve guest address space at 0x%016" PRIx64
-				     "\n",
+				     ", size 0x%016" PRIx64 "\n",
 				     start, end - start);
 			}
 			AddFreeUnlocked(start, end - start);
 			m_owned.emplace_back(start, end - start);
 		}
+#endif
 #endif
 	}
 

tests/shaderCfgTests.cpp

@@ -5394,7 +5394,224 @@ void TestNewShaderRecompilerCfgSharedOuterAndLoopMerge() {
 	CheckSpirvBinaryValidates(result.spirv);
 }
 
-void TestNewShaderRecompilerCfgLoopSharedContinueSelectionMerges() {
+void TestNewShaderRecompilerCfgLoopEarlyBreakNoSelection() {
+	const uint32_t shader[] = {
+	    EncodeSopc(0x0a, 0, 129),    // loop: s_cmp_lt_u32 s0, 1
+	    EncodeSopp(0x04, 4),         // loop exit -> end
+	    EncodeSopc(0x06, 1, 1),      // s_cmp_eq_u32 s1, s1
+	    EncodeSopp(0x04, 2),         // early break -> same loop end
+	    EncodeSop2(0x00, 0, 0, 129), // s_add_u32 s0, s0, 1
+	    EncodeSopp(0x02, 0xfffau),   // backedge -> loop header
+	    0xbf810000u,
+	};
+
+	ShaderRecompiler::CompileOptions options;
+	options.stage   = ShaderType::Compute;
+	options.dump_ir = true;
+
+	ShaderRecompiler::CompileResult result;
+	std::string                     error;
+	Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str());
+	Check(Common::ContainsStr(result.ir_dump, "mode=structured"),
+	      "loop early-break CFG did not stay on structured path");
+	Check(SpirvInstructionOpcodeCount(result.spirv, 246) != 0,
+	      "loop early-break SPIR-V lacks OpLoopMerge");
+	Check(SpirvInstructionOpcodeCount(result.spirv, 247) == 0,
+	      "loop early-break SPIR-V unexpectedly used OpSelectionMerge");
+	Check(SpirvInstructionOpcodeCount(result.spirv, 251) == 0,
+	      "loop early-break CFG unexpectedly used dispatcher OpSwitch");
+	CheckSpirvBinaryValidates(result.spirv);
+}
+
+void TestNewShaderRecompilerCfgNestedLoopNonlocalExitDispatcher() {
+	const uint32_t shader[] = {
+	    EncodeSopc(0x0a, 0, 129),    // outer loop: s_cmp_lt_u32 s0, 1
+	    EncodeSopp(0x04, 9),         // outer exit -> end
+	    EncodeSopc(0x0a, 1, 129),    // inner loop: s_cmp_lt_u32 s1, 1
+	    EncodeSopp(0x04, 5),         // inner exit -> outer continue
+	    EncodeSopc(0x06, 2, 2),      // s_cmp_eq_u32 s2, s2
+	    EncodeSopp(0x05, 5),         // nonlocal exit -> outer end
+	    EncodeSMovB32(3, 129),       // inner work
+	    EncodeSop2(0x00, 1, 1, 129), // s_add_u32 s1, s1, 1
+	    EncodeSopp(0x02, 0xfff9u),   // inner backedge
+	    EncodeSop2(0x00, 0, 0, 129), // outer continue: s_add_u32 s0, s0, 1
+	    EncodeSopp(0x02, 0xfff5u),   // outer backedge
+	    0xbf810000u,
+	};
+
+	ShaderRecompiler::CompileOptions options;
+	options.stage   = ShaderType::Compute;
+	options.dump_ir = true;
+
+	ShaderRecompiler::CompileResult result;
+	std::string                     error;
+	Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str());
+	Check(Common::ContainsStr(result.ir_dump, "mode=dispatcher"),
+	      "nested-loop nonlocal exit did not select dispatcher fallback");
+	Check(SpirvInstructionOpcodeCount(result.spirv, 251) != 0,
+	      "nested-loop nonlocal exit dispatcher SPIR-V lacks OpSwitch");
+	CheckSpirvBinaryValidates(result.spirv);
+}
+
+void TestNewShaderRecompilerCfgNestedLoopLocalExitNoSelection() {
+	const uint32_t shader[] = {
+	    EncodeSopc(0x0a, 0, 129),    // outer loop: s_cmp_lt_u32 s0, 1
+	    EncodeSopp(0x04, 6),         // outer exit -> end
+	    EncodeSopc(0x0a, 1, 129),    // inner loop: s_cmp_lt_u32 s1, 1
+	    EncodeSopp(0x04, 2),         // inner exit -> outer continue
+	    EncodeSMovB32(2, 129),       // inner work
+	    EncodeSopp(0x02, 0xfffcu),   // inner backedge
+	    EncodeSop2(0x00, 0, 0, 129), // outer continue: s_add_u32 s0, s0, 1
+	    EncodeSopp(0x02, 0xfff8u),   // outer backedge
+	    0xbf810000u,
+	};
+
+	ShaderRecompiler::CompileOptions options;
+	options.stage   = ShaderType::Compute;
+	options.dump_ir = true;
+
+	ShaderRecompiler::CompileResult result;
+	std::string                     error;
+	Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str());
+	Check(Common::ContainsStr(result.ir_dump, "mode=structured"),
+	      "nested local loop exit did not stay on structured path");
+	Check(SpirvInstructionOpcodeCount(result.spirv, 246) >= 2,
+	      "nested local loop exit SPIR-V lacks both OpLoopMerge instructions");
+	Check(SpirvInstructionOpcodeCount(result.spirv, 247) == 0,
+	      "nested local loop exit SPIR-V unexpectedly used OpSelectionMerge");
+	Check(SpirvInstructionOpcodeCount(result.spirv, 251) == 0,
+	      "nested local loop exit unexpectedly used dispatcher OpSwitch");
+	CheckSpirvBinaryValidates(result.spirv);
+}
+
+void TestNewShaderRecompilerCfgNestedLoopExitTailMergeSplit() {
+	const uint32_t shader[] = {
+	    EncodeSopc(0x0a, 0, 129),    // outer loop: s_cmp_lt_u32 s0, 1
+	    EncodeSopp(0x04, 11),        // outer exit -> end
+	    EncodeSopc(0x06, 1, 1),      // inner loop first exit condition
+	    EncodeSopp(0x05, 3),         // first inner exit -> tail A
+	    EncodeSopc(0x06, 2, 2),      // inner loop second exit condition
+	    EncodeSopp(0x05, 3),         // second inner exit -> tail B
+	    EncodeSopp(0x02, 0xfffbu),   // inner backedge
+	    EncodeSMovB32(3, 129),       // tail A
+	    EncodeSopp(0x02, 2),         // tail A -> outer continue
+	    EncodeSMovB32(4, 129),       // tail B
+	    EncodeSopp(0x02, 0),         // tail B -> outer continue
+	    EncodeSop2(0x00, 0, 0, 129), // outer continue: s_add_u32 s0, s0, 1
+	    EncodeSopp(0x02, 0xfff3u),   // outer backedge
+	    0xbf810000u,
+	};
+
+	ShaderRecompiler::Decoder::Program program;
+	std::string                        error;
+	Check(ShaderRecompiler::Decoder::DecodeProgram(std::span {shader}, program, &error),
+	      error.c_str());
+
+	ShaderRecompiler::CFG::Graph graph;
+	Check(ShaderRecompiler::CFG::BuildGraph(program, graph, &error), error.c_str());
+	const auto original_block_count = graph.blocks.size();
+	Check(ShaderRecompiler::CFG::Structurize(graph, &error), error.c_str());
+	Check(graph.blocks.size() > original_block_count,
+	      "nested loop exit tails did not create a private inner merge");
+
+	const auto* outer_header = graph.FindBlockByPc(0);
+	const auto* inner_header = graph.FindBlockByPc(8);
+	Check(outer_header != nullptr && inner_header != nullptr &&
+	          outer_header->terminator.loop_header && inner_header->terminator.loop_header,
+	      "nested loop exit-tail fixture did not retain both loop headers");
+	Check(inner_header->terminator.merge_block != outer_header->terminator.continue_block,
+	      "inner loop merge still aliases the outer continue target");
+	const auto* inner_merge = graph.FindBlock(inner_header->terminator.merge_block);
+	Check(inner_merge != nullptr && inner_merge->inst_begin == inner_merge->inst_end &&
+	          inner_merge->terminator.kind == ShaderRecompiler::CFG::TerminatorKind::Branch &&
+	          inner_merge->terminator.true_block == outer_header->terminator.continue_block,
+	      "private inner merge does not forward to the outer continue target");
+}
+
+void TestNewShaderRecompilerCfgMixedContinueNonmergeExitDispatcher() {
+	const uint32_t shader[] = {
+	    EncodeSopc(0x06, 7, 7),      // entry branch bypasses loop -> exit X
+	    EncodeSopp(0x05, 5),         // entry -> X
+	    EncodeSopc(0x0a, 0, 129),    // loop: s_cmp_lt_u32 s0, 1
+	    EncodeSopp(0x04, 5),         // loop exit -> Y
+	    EncodeSopc(0x06, 1, 1),      // inner condition
+	    EncodeSopp(0x05, 1),         // nonmerge exit -> X, else continue
+	    EncodeSopp(0x02, 0xfffbu),   // loop backedge
+	    EncodeSMovB32(2, 129),       // X
+	    EncodeSopp(0x02, 2),         // X -> end
+	    EncodeSMovB32(3, 129),       // Y
+	    EncodeSopp(0x02, 0),         // Y -> end
+	    0xbf810000u,
+	};
+
+	ShaderRecompiler::CompileOptions options;
+	options.stage   = ShaderType::Compute;
+	options.dump_ir = true;
+
+	ShaderRecompiler::CompileResult result;
+	std::string                     error;
+	Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str());
+	Check(Common::ContainsStr(result.ir_dump, "mode=dispatcher"),
+	      "mixed continue/nonmerge exit did not select dispatcher fallback");
+	Check(SpirvInstructionOpcodeCount(result.spirv, 251) != 0,
+	      "mixed continue/nonmerge exit dispatcher SPIR-V lacks OpSwitch");
+	CheckSpirvBinaryValidates(result.spirv);
+}
+
+void TestNewShaderRecompilerCfgConditionalLatchNoSelection() {
+	const uint32_t shader[] = {
+	    EncodeSopp(0x02, 0),       // loop header -> conditional block
+	    EncodeSopc(0x06, 0, 0),    // s_cmp_eq_u32 s0, s0
+	    EncodeSopp(0x05, 1),       // loop exit -> end
+	    EncodeSopp(0x02, 0xfffcu), // separate latch -> loop header
+	    0xbf810000u,
+	};
+
+	ShaderRecompiler::CompileOptions options;
+	options.stage   = ShaderType::Compute;
+	options.dump_ir = true;
+
+	ShaderRecompiler::CompileResult result;
+	std::string                     error;
+	Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str());
+	Check(Common::ContainsStr(result.ir_dump, "mode=structured"),
+	      "conditional latch did not stay on structured path");
+	Check(SpirvInstructionOpcodeCount(result.spirv, 246) != 0,
+	      "conditional latch SPIR-V lacks OpLoopMerge");
+	Check(SpirvInstructionOpcodeCount(result.spirv, 247) == 0,
+	      "conditional latch SPIR-V unexpectedly used OpSelectionMerge");
+	Check(SpirvInstructionOpcodeCount(result.spirv, 251) == 0,
+	      "conditional latch unexpectedly used dispatcher OpSwitch");
+	CheckSpirvBinaryValidates(result.spirv);
+}
+
+void TestNewShaderRecompilerCfgDirectConditionalLatchNoSelection() {
+	const uint32_t shader[] = {
+	    EncodeSopp(0x02, 0),       // loop header -> conditional latch
+	    EncodeSopc(0x06, 0, 0),    // s_cmp_eq_u32 s0, s0
+	    EncodeSopp(0x05, 0xfffdu), // direct latch backedge -> loop header
+	    0xbf810000u,
+	};
+
+	ShaderRecompiler::CompileOptions options;
+	options.stage   = ShaderType::Compute;
+	options.dump_ir = true;
+
+	ShaderRecompiler::CompileResult result;
+	std::string                     error;
+	Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str());
+	Check(Common::ContainsStr(result.ir_dump, "mode=structured"),
+	      "direct conditional latch did not stay on structured path");
+	Check(SpirvInstructionOpcodeCount(result.spirv, 246) != 0,
+	      "direct conditional latch SPIR-V lacks OpLoopMerge");
+	Check(SpirvInstructionOpcodeCount(result.spirv, 247) == 0,
+	      "direct conditional latch SPIR-V unexpectedly used OpSelectionMerge");
+	Check(SpirvInstructionOpcodeCount(result.spirv, 251) == 0,
+	      "direct conditional latch unexpectedly used dispatcher OpSwitch");
+	CheckSpirvBinaryValidates(result.spirv);
+}
+
+void TestNewShaderRecompilerCfgLoopEarlyContinuesNoSelection() {
 	const uint32_t shader[] = {
 	    EncodeSMovB32(0, 128),       // s0 = 0
 	    EncodeSopc(0x0a, 0, 130),    // loop: s_cmp_lt_u32 s0, 2
@@ -5419,15 +5636,111 @@ void TestNewShaderRecompilerCfgLoopSharedContinueSelectionMerges() {
 	std::string                     error;
 	Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str());
 	Check(Common::ContainsStr(result.ir_dump, "mode=structured"),
-	      "shared loop continue selections should stay on structured path");
+	      "loop early continues should stay on structured path");
-	Check(!Common::ContainsStr(result.ir_dump, "duplicate structured merge block"),
+	Check(SpirvInstructionOpcodeCount(result.spirv, 246) != 0,
-	      "shared loop continue selections were not split before structurization");
+	      "loop early continues SPIR-V lacks OpLoopMerge");
-	Check(SpirvContainsOpcode(result.spirv, 246),
+	Check(SpirvInstructionOpcodeCount(result.spirv, 247) == 0,
-	      "shared loop continue selections SPIR-V lacks OpLoopMerge");
+	      "loop early continues SPIR-V unexpectedly used OpSelectionMerge");
-	Check(SpirvContainsOpcode(result.spirv, 247),
+	Check(SpirvInstructionOpcodeCount(result.spirv, 251) == 0,
-	      "shared loop continue selections SPIR-V lacks OpSelectionMerge");
+	      "loop early continues unexpectedly used dispatcher OpSwitch");
-	Check(!SpirvContainsOpcode(result.spirv, 251),
+	CheckSpirvBinaryValidates(result.spirv);
-	      "shared loop continue selections unexpectedly used dispatcher OpSwitch");
+}
+
+void TestNewShaderRecompilerCfgConditionalLoopHeaderSelection() {
+	const uint32_t shader[] = {
+	    EncodeSopc(0x06, 0, 0),    // loop body selection condition
+	    EncodeSopp(0x05, 2),       // select path B
+	    EncodeSMovB32(1, 129),     // path A
+	    EncodeSopp(0x02, 1),       // path A -> join
+	    EncodeSMovB32(2, 129),     // path B
+	    EncodeSMovB32(3, 129),     // join
+	    EncodeSopc(0x06, 4, 4),    // repeat condition
+	    EncodeSopp(0x05, 0xfff8u), // repeat -> guest header
+	    0xbf810000u,
+	};
+
+	ShaderRecompiler::Decoder::Program decoded;
+	std::string                        error;
+	Check(ShaderRecompiler::Decoder::DecodeProgram(std::span {shader}, decoded, &error),
+	      error.c_str());
+	ShaderRecompiler::CFG::Graph graph;
+	Check(ShaderRecompiler::CFG::BuildGraph(decoded, graph, &error), error.c_str());
+	const auto original_block_count = graph.blocks.size();
+	Check(ShaderRecompiler::CFG::Structurize(graph, &error), error.c_str());
+	Check(graph.blocks.size() > original_block_count,
+	      "conditional guest loop header did not create a synthetic header");
+
+	uint32_t loop_headers      = 0;
+	uint32_t selection_headers = 0;
+	for (const auto& block: graph.blocks) {
+		if (block.terminator.loop_header) {
+			loop_headers++;
+			Check(block.inst_begin == block.inst_end &&
+			          block.terminator.kind == ShaderRecompiler::CFG::TerminatorKind::Branch,
+			      "canonical loop header is not an empty unconditional block");
+		} else if (block.terminator.kind ==
+		               ShaderRecompiler::CFG::TerminatorKind::ConditionalBranch &&
+		           block.terminator.merge_block != UINT32_MAX) {
+			selection_headers++;
+		}
+	}
+	Check(loop_headers == 1u && selection_headers == 1u,
+	      "guest conditional was not separated from the loop header");
+
+	ShaderRecompiler::CompileOptions options;
+	options.stage = ShaderType::Compute;
+	ShaderRecompiler::CompileResult result;
+	Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str());
+	Check(SpirvInstructionOpcodeCount(result.spirv, 246) == 1u,
+	      "conditional loop-header SPIR-V has the wrong loop-merge count");
+	Check(SpirvInstructionOpcodeCount(result.spirv, 247) == 1u,
+	      "conditional loop-header SPIR-V has the wrong selection-merge count");
+	Check(SpirvInstructionOpcodeCount(result.spirv, 251) == 0u,
+	      "conditional loop-header unexpectedly used dispatcher OpSwitch");
+	CheckSpirvBinaryValidates(result.spirv);
+}
+
+void TestNewShaderRecompilerCfgMultipleLoopLatches() {
+	const uint32_t shader[] = {
+	    EncodeSopc(0x0a, 0, 129),  // loop condition
+	    EncodeSopp(0x04, 5),       // loop exit -> end
+	    EncodeSopc(0x06, 1, 1),    // early repeat condition
+	    EncodeSopp(0x05, 0xfffcu), // early repeat -> header
+	    EncodeSMovB32(2, 129),     // body
+	    EncodeSMovB32(3, 129),     // body tail
+	    EncodeSopp(0x02, 0xfff9u), // ordinary latch -> header
+	    0xbf810000u,
+	};
+
+	ShaderRecompiler::Decoder::Program decoded;
+	std::string                        error;
+	Check(ShaderRecompiler::Decoder::DecodeProgram(std::span {shader}, decoded, &error),
+	      error.c_str());
+	ShaderRecompiler::CFG::Graph graph;
+	Check(ShaderRecompiler::CFG::BuildGraph(decoded, graph, &error), error.c_str());
+	const auto original_block_count = graph.blocks.size();
+	Check(graph.back_edges.size() == 2u, "multiple-latch fixture lacks two native backedges");
+	Check(ShaderRecompiler::CFG::Structurize(graph, &error), error.c_str());
+	Check(graph.blocks.size() == original_block_count + 1u,
+	      "multiple native latches did not create one synthetic continue");
+	Check(graph.back_edges.size() == 1u && graph.natural_loops.size() == 1u,
+	      "multiple native latches were not coalesced to one SPIR-V backedge");
+	const auto& loop           = graph.natural_loops.front();
+	const auto* continue_block = graph.FindBlock(loop.continue_block);
+	Check(continue_block != nullptr && continue_block->inst_begin == continue_block->inst_end &&
+	          continue_block->predecessors.size() == 2u,
+	      "canonical continue does not join both native latches");
+
+	ShaderRecompiler::CompileOptions options;
+	options.stage = ShaderType::Compute;
+	ShaderRecompiler::CompileResult result;
+	Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str());
+	Check(SpirvInstructionOpcodeCount(result.spirv, 246) == 1u,
+	      "multiple-latch SPIR-V has the wrong loop-merge count");
+	Check(SpirvInstructionOpcodeCount(result.spirv, 247) == 0u,
+	      "multiple-latch SPIR-V unexpectedly used a selection merge");
+	Check(SpirvInstructionOpcodeCount(result.spirv, 251) == 0u,
+	      "multiple-latch SPIR-V unexpectedly used dispatcher OpSwitch");
 	CheckSpirvBinaryValidates(result.spirv);
 }
 
@@ -5457,6 +5770,85 @@ void TestNewShaderRecompilerCfgDuplicateMergeStructuredSplit() {
 	CheckSpirvBinaryValidates(result.spirv);
 }
 
+void TestNewShaderRecompilerCfgOverlappingEarlyExitLadder() {
+	const uint32_t shader[] = {
+	    EncodeSopc(0x06, 0, 0), // block 0
+	    EncodeSopp(0x04, 2),    // block 0 -> 2 or 1
+	    EncodeSopc(0x06, 1, 1), // block 1
+	    EncodeSopp(0x04, 6),    // block 1 -> 5 or 2
+	    EncodeSopc(0x06, 2, 2), // block 2
+	    EncodeSopp(0x04, 4),    // block 2 -> 5 or 3
+	    EncodeSopc(0x06, 3, 3), // block 3
+	    EncodeSopp(0x04, 2),    // block 3 -> 5 or 4
+	    EncodeSMovB32(4, 129),  // block 4
+	    0xbf810000u,             // block 4 -> 6
+	    EncodeSMovB32(5, 129),  // block 5
+	    0xbf810000u,             // block 5 -> 6
+	};
+
+	ShaderRecompiler::Decoder::Program decoded;
+	std::string                        error;
+	Check(ShaderRecompiler::Decoder::DecodeProgram(std::span {shader}, decoded, &error),
+	      error.c_str());
+	ShaderRecompiler::CFG::Graph graph;
+	Check(ShaderRecompiler::CFG::BuildGraph(decoded, graph, &error), error.c_str());
+	Check(graph.blocks.size() == 7u && graph.blocks[0].successors == std::vector<uint32_t>({1, 2}) &&
+	          graph.blocks[0].terminator.true_block == 2u &&
+	          graph.blocks[0].terminator.false_block == 1u &&
+	          graph.blocks[1].successors == std::vector<uint32_t>({2, 5}) &&
+	          graph.blocks[1].terminator.true_block == 5u &&
+	          graph.blocks[1].terminator.false_block == 2u &&
+	          graph.blocks[2].successors == std::vector<uint32_t>({3, 5}) &&
+	          graph.blocks[2].terminator.true_block == 5u &&
+	          graph.blocks[2].terminator.false_block == 3u &&
+	          graph.blocks[3].successors == std::vector<uint32_t>({4, 5}) &&
+	          graph.blocks[3].terminator.true_block == 5u &&
+	          graph.blocks[3].terminator.false_block == 4u &&
+	          graph.blocks[4].successors == std::vector<uint32_t>({6}) &&
+	          graph.blocks[5].successors == std::vector<uint32_t>({6}),
+	      "overlapping early-exit fixture does not match the observed shader CFG");
+	Check(ShaderRecompiler::CFG::Structurize(graph, &error), error.c_str());
+	std::vector<bool>     reachable(graph.blocks.size());
+	std::vector<uint32_t> pending = {graph.entry_block};
+	while (!pending.empty()) {
+		const auto block_id = pending.back();
+		pending.pop_back();
+		if (reachable[block_id]) {
+			continue;
+		}
+		reachable[block_id] = true;
+		pending.insert(pending.end(), graph.blocks[block_id].successors.begin(),
+		               graph.blocks[block_id].successors.end());
+	}
+	Check(std::all_of(reachable.begin(), reachable.end(), [](bool value) { return value; }),
+	      "overlapping early-exit structurization left unreachable blocks");
+	std::vector<uint32_t> merges;
+	for (const auto& block: graph.blocks) {
+		if (block.terminator.kind == ShaderRecompiler::CFG::TerminatorKind::ConditionalBranch) {
+			Check(block.terminator.merge_block != UINT32_MAX &&
+			          std::find(merges.begin(), merges.end(), block.terminator.merge_block) ==
+			              merges.end(),
+			      "overlapping early-exit structurization retained a shared merge");
+			merges.push_back(block.terminator.merge_block);
+		}
+	}
+
+	ShaderRecompiler::CompileOptions options;
+	options.stage   = ShaderType::Pixel;
+	options.dump_ir = true;
+	ShaderRecompiler::CompileResult result;
+	Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str());
+	Check(Common::ContainsStr(result.ir_dump, "mode=structured"),
+	      "overlapping early-exit ladder did not stay on the structured path");
+	Check(!Common::ContainsStr(result.ir_dump, "duplicate structured merge block"),
+	      "overlapping early-exit ladder retained a shared merge");
+	Check(SpirvInstructionOpcodeCount(result.spirv, 247) >= 4u,
+	      "overlapping early-exit ladder lost its selections");
+	Check(SpirvInstructionOpcodeCount(result.spirv, 251) == 0u,
+	      "overlapping early-exit ladder used dispatcher OpSwitch");
+	CheckSpirvBinaryValidates(result.spirv);
+}
+
 void TestNewShaderRecompilerCfgIrreducibleDispatcher() {
 	const uint32_t shader[] = {
 	    EncodeSopp(0x05, 2),       // entry -> B, fallthrough A
@@ -7014,7 +7406,6 @@ int main() {
 	using namespace Libs::Graphics;
 
 	EnsureConfigInitialized();
-
 	TestResourceDescriptorClassification();
 	TestNativeShaderResourceDependencies();
 	TestNormalizedImageContracts();
@@ -7086,8 +7477,18 @@ int main() {
 	TestNewShaderRecompilerCfgLoopHeaderBufferLoadDispatcher();
 	TestNewShaderRecompilerCfgLoopHeaderDsAppendConsumeDispatcher();
 	TestNewShaderRecompilerCfgSharedOuterAndLoopMerge();
-	TestNewShaderRecompilerCfgLoopSharedContinueSelectionMerges();
+	TestNewShaderRecompilerCfgLoopEarlyBreakNoSelection();
+	TestNewShaderRecompilerCfgNestedLoopNonlocalExitDispatcher();
+	TestNewShaderRecompilerCfgNestedLoopLocalExitNoSelection();
+	TestNewShaderRecompilerCfgNestedLoopExitTailMergeSplit();
+	TestNewShaderRecompilerCfgMixedContinueNonmergeExitDispatcher();
+	TestNewShaderRecompilerCfgConditionalLatchNoSelection();
+	TestNewShaderRecompilerCfgDirectConditionalLatchNoSelection();
+	TestNewShaderRecompilerCfgLoopEarlyContinuesNoSelection();
+	TestNewShaderRecompilerCfgConditionalLoopHeaderSelection();
+	TestNewShaderRecompilerCfgMultipleLoopLatches();
 	TestNewShaderRecompilerCfgDuplicateMergeStructuredSplit();
+	TestNewShaderRecompilerCfgOverlappingEarlyExitLadder();
 	TestNewShaderRecompilerCfgIrreducibleDispatcher();
 	TestNewShaderRecompilerExecMaskHelpers();
 	TestComputeShaderInputWaveSize();