cmake: preserve spaces in clang-cl linker paths (#26)

Pass linker flags as individual options so CMake keeps the PDB and lld map paths intact when the build directory contains spaces.

src/CMakeLists.txt

@@ -312,6 +312,9 @@ function(add_kyty_full_emulator_test target source)
 		target_link_libraries(${target} onecore)
 		add_custom_command(TARGET ${target} POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different "${KYTY_THIRD_PARTY_DIR}/winpthread/bin/libwinpthread-1.dll" $<TARGET_FILE_DIR:${target}>/libwinpthread-1.dll)
 	endif()
+	# The macOS x86_64 guest address space needs its .zerofill segments anchored
+	# by linker flags, or the kernel kills the binary on load (posix_spawn EIO).
+	configure_macos_guest_address_space(${target})
 endfunction()
 
 function(configure_macos_guest_address_space target)
@@ -431,7 +434,6 @@ target_sources(shader_recompiler_compute_tests PRIVATE
 add_kyty_full_emulator_test(virtual_memory_allocation_tests ../tests/VirtualMemoryAllocationTests.cpp)
 target_compile_definitions(virtual_memory_allocation_tests PRIVATE
 	KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS=1)
-configure_macos_guest_address_space(virtual_memory_allocation_tests)
 
 # These tests use exceptions.
 if(NOT KYTY_CLANG_CL)
@@ -513,7 +515,12 @@ set(KYTY_EMULATOR_MAP_LINK_PATH "${CMAKE_CURRENT_BINARY_DIR}/${KYTY_EMULATOR_MAP
 set(KYTY_EMULATOR_PDB_LINK_PATH "${CMAKE_CURRENT_BINARY_DIR}/kyty_emulator.pdb")
 
 if(KYTY_CLANG_CL)
-	set_target_properties(kyty_emulator PROPERTIES LINK_FLAGS "/DYNAMICBASE:NO /DEBUG:FULL /PDB:${KYTY_EMULATOR_PDB_LINK_PATH} /lldmap:${KYTY_EMULATOR_MAP_LINK_PATH}")
+	target_link_options(kyty_emulator PRIVATE
+		"/DYNAMICBASE:NO"
+		"/DEBUG:FULL"
+		"/PDB:${KYTY_EMULATOR_PDB_LINK_PATH}"
+		"/lldmap:${KYTY_EMULATOR_MAP_LINK_PATH}"
+	)
 	add_custom_command(TARGET kyty_emulator POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different "${KYTY_THIRD_PARTY_DIR}/winpthread/bin/libwinpthread-1.dll" $<TARGET_FILE_DIR:kyty_emulator>/libwinpthread-1.dll)
 elseif(WIN32 OR LINUX)
 	set_target_properties(kyty_emulator PROPERTIES LINK_FLAGS "${KYTY_LD_OPTIONS} -Wl,-Map=${KYTY_EMULATOR_MAP_LINK_PATH}")

src/graphics/guest_gpu/pm4.cpp

@@ -110,7 +110,6 @@ void DumpPm4PacketStream(Common::File* file, uint32_t* cmd_buffer, uint32_t star
 	auto* cmd = cmd_buffer + start_dw;
 	auto  dw  = num_dw;
 	while (dw != 0) {
-		EXIT_NOT_IMPLEMENTED(dw < 2);
 		EXIT_NOT_IMPLEMENTED(dw > num_dw);
 
 		auto cmd_id = *cmd++;
@@ -120,6 +119,9 @@ void DumpPm4PacketStream(Common::File* file, uint32_t* cmd_buffer, uint32_t star
 		uint32_t len = 0;
 
 		const auto packet_type = static_cast<PacketType>(cmd_id >> 30u);
+		// Type-2 packets are header-only padding; every other packet type requires a body.
+		EXIT_NOT_IMPLEMENTED(dw < 2 && packet_type != PacketType::Type2);
+
 		switch (packet_type) {
 			case PacketType::Type3: {
 				const bool    sh_gx = (cmd_id & 0x2u) == 0;

src/graphics/shader/recompiler/cfg/ShaderCFG.cpp

@@ -871,19 +871,19 @@ std::vector<uint32_t> DominatedBlocks(const Graph& graph, uint32_t header,
 	return blocks;
 }
 
-uint32_t AppendSyntheticMergeBlock(Graph& graph, uint32_t old_merge) {
-	const auto* merge = graph.FindBlock(old_merge);
+uint32_t AppendSyntheticBranchBlock(Graph& graph, uint32_t target) {
+	const auto* target_block = graph.FindBlock(target);
 
 	BasicBlock block;
 	block.id                    = static_cast<uint32_t>(graph.blocks.size());
-	block.start_pc              = merge != nullptr ? merge->start_pc : 0u;
+	block.start_pc              = target_block != nullptr ? target_block->start_pc : 0u;
 	block.end_pc                = block.start_pc;
-	block.inst_begin            = merge != nullptr ? merge->inst_begin : 0u;
+	block.inst_begin            = target_block != nullptr ? target_block->inst_begin : 0u;
 	block.inst_end              = block.inst_begin;
-	block.successors            = {old_merge};
+	block.successors            = {target};
 	block.terminator.kind       = TerminatorKind::Branch;
 	block.terminator.condition  = BranchCondition::Always;
-	block.terminator.true_block = old_merge;
+	block.terminator.true_block = target;
 	graph.blocks.push_back(std::move(block));
 	return graph.blocks.back().id;
 }
@@ -897,15 +897,50 @@ bool IsSyntheticMergeForwarder(const Graph& graph, uint32_t block_id, uint32_t m
 	       block->terminator.true_block == merge;
 }
 
-bool IsInsideLoopConstruct(const Graph& graph, const NaturalLoop& loop, uint32_t block_id) {
-	return block_id != UINT32_MAX && block_id != loop.merge && block_id != loop.continue_block &&
-	       graph.Dominates(loop.header, block_id) &&
-	       (loop.merge == UINT32_MAX || !graph.Dominates(loop.merge, block_id));
+const NaturalLoop* FindInnermostContainingLoop(const Graph& graph, uint32_t block_id) {
+	const NaturalLoop* innermost = nullptr;
+	for (const auto& loop: graph.natural_loops) {
+		if (Contains(loop.body_blocks, block_id) &&
+		    (innermost == nullptr || loop.body_blocks.size() < innermost->body_blocks.size())) {
+			innermost = &loop;
+		}
+	}
+	return innermost;
 }
 
-bool SelectionMergeLeavesContainingLoop(const Graph& graph, uint32_t header, uint32_t merge) {
+bool IsInsideLoopConstruct(const Graph& graph, const NaturalLoop& loop, uint32_t block_id) {
+	return block_id != UINT32_MAX && block_id != loop.merge && block_id != loop.continue_block &&
+	       graph.Dominates(loop.header, block_id) && !graph.Dominates(loop.merge, block_id);
+}
+
+bool IsInnermostLoopControlConditional(const Graph& graph, const BasicBlock& block) {
+	if (block.terminator.kind != TerminatorKind::ConditionalBranch) {
+		return false;
+	}
+	const auto* loop = FindInnermostContainingLoop(graph, block.id);
+	if (loop == nullptr || loop->merge == UINT32_MAX || loop->continue_block == UINT32_MAX) {
+		return false;
+	}
+	const auto true_target  = block.terminator.true_block;
+	const auto false_target = block.terminator.false_block;
+	if (block.id == loop->continue_block) {
+		const auto is_repeat_target = [&](uint32_t target) {
+			return target == loop->header || target == loop->merge;
+		};
+		return is_repeat_target(true_target) && is_repeat_target(false_target);
+	}
+	const auto is_control_target = [&](uint32_t target) {
+		return target == loop->merge || target == loop->continue_block;
+	};
+	return (is_control_target(true_target) &&
+	        (is_control_target(false_target) ||
+	         IsInsideLoopConstruct(graph, *loop, false_target))) ||
+	       (is_control_target(false_target) && IsInsideLoopConstruct(graph, *loop, true_target));
+}
+
+bool MergeLeavesContainingLoop(const Graph& graph, uint32_t header, uint32_t merge) {
 	for (const auto& loop: graph.natural_loops) {
-		if (IsInsideLoopConstruct(graph, loop, header) &&
+		if (loop.header != header && IsInsideLoopConstruct(graph, loop, header) &&
 		    !IsInsideLoopConstruct(graph, loop, merge)) {
 			return true;
 		}
@@ -913,6 +948,80 @@ bool SelectionMergeLeavesContainingLoop(const Graph& graph, uint32_t header, uin
 	return false;
 }
 
+bool CanonicalizeNaturalLoops(Graph& graph, std::string* error) {
+	const auto rewrite_budget = graph.blocks.size() * 2u + 16u;
+	for (size_t rewrite = 0; rewrite < rewrite_budget; rewrite++) {
+		bool changed = false;
+		for (const auto& loop: graph.natural_loops) {
+			std::vector<uint32_t> latches;
+			for (const auto& edge: graph.back_edges) {
+				if (edge.to == loop.header) {
+					AddUnique(latches, edge.from);
+				}
+			}
+			if (latches.size() <= 1u) {
+				continue;
+			}
+
+			const auto continue_block = AppendSyntheticBranchBlock(graph, loop.header);
+			for (auto latch: latches) {
+				auto* block = graph.FindBlock(latch);
+				if (block != nullptr) {
+					ReplaceValue(block->successors, loop.header, continue_block);
+					ReplaceTerminatorTarget(block->terminator, loop.header, continue_block);
+				}
+			}
+			RebuildPredecessors(graph);
+			RecomputeAnalyses(graph);
+			changed = true;
+			break;
+		}
+		if (changed) {
+			continue;
+		}
+
+		for (const auto& loop: graph.natural_loops) {
+			const auto* header                 = graph.FindBlock(loop.header);
+			const auto  is_loop_control_target = [&](uint32_t target) {
+				return target == loop.merge || target == loop.continue_block;
+			};
+			if (header == nullptr || header->terminator.kind != TerminatorKind::ConditionalBranch ||
+			    is_loop_control_target(header->terminator.true_block) ||
+			    is_loop_control_target(header->terminator.false_block) ||
+			    !Contains(loop.body_blocks, header->terminator.true_block) ||
+			    !Contains(loop.body_blocks, header->terminator.false_block)) {
+				continue;
+			}
+
+			const auto old_header   = loop.header;
+			const auto predecessors = header->predecessors;
+			const auto new_header   = AppendSyntheticBranchBlock(graph, old_header);
+			for (auto pred: predecessors) {
+				auto* block = graph.FindBlock(pred);
+				if (block != nullptr) {
+					ReplaceValue(block->successors, old_header, new_header);
+					ReplaceTerminatorTarget(block->terminator, old_header, new_header);
+				}
+			}
+			if (graph.entry_block == old_header) {
+				graph.entry_block = new_header;
+			}
+			MoveBlockBefore(graph, new_header, old_header);
+			RebuildPredecessors(graph);
+			RecomputeAnalyses(graph);
+			changed = true;
+			break;
+		}
+		if (!changed) {
+			return true;
+		}
+	}
+
+	SetFailure(graph, FailureKind::StructuredControlFlow, graph.entry_block,
+	           "CFG loop canonicalization exceeded rewrite budget", error);
+	return false;
+}
+
 bool SplitSharedMergeBlock(Graph& graph, uint32_t merge,
                            const std::vector<uint32_t>& construct_blocks,
                            bool                         force_split = false) {
@@ -948,7 +1057,7 @@ bool SplitSharedMergeBlock(Graph& graph, uint32_t merge,
 		return false;
 	}
 
-	const auto synthetic_merge = AppendSyntheticMergeBlock(graph, merge);
+	const auto synthetic_merge = AppendSyntheticBranchBlock(graph, merge);
 	auto*      synthetic_block = graph.FindBlock(synthetic_merge);
 	if (synthetic_block != nullptr) {
 		synthetic_block->predecessors = predecessors_to_split;
@@ -980,14 +1089,111 @@ bool SplitSharedMergeBlock(Graph& graph, uint32_t merge,
 bool SplitOneLoopMerge(Graph& graph) {
 	const auto& loops = graph.natural_loops;
 	for (const auto& loop: loops) {
-		if (SplitSharedMergeBlock(graph, loop.merge, loop.body_blocks)) {
+		const auto construct_blocks = DominatedBlocks(graph, loop.header, loop.merge);
+		const auto force_split      = MergeLeavesContainingLoop(graph, loop.header, loop.merge);
+		if (SplitSharedMergeBlock(graph, loop.merge, construct_blocks, force_split)) {
 			return true;
 		}
 	}
 	return false;
 }
 
-bool SplitOneSelectionMerge(Graph& graph) {
+std::vector<uint32_t> SelectionRegion(const Graph& graph, const BasicBlock& header,
+                                      uint32_t merge) {
+	std::vector<uint32_t> region;
+	std::vector<uint32_t> pending = {header.terminator.true_block,
+	                                 header.terminator.false_block};
+	while (!pending.empty()) {
+		const auto block_id = pending.back();
+		pending.pop_back();
+		if (block_id == merge || Contains(region, block_id)) {
+			continue;
+		}
+		const auto* block = graph.FindBlock(block_id);
+		if (block == nullptr) {
+			continue;
+		}
+		AddUnique(region, block_id);
+		pending.insert(pending.end(), block->successors.begin(), block->successors.end());
+	}
+	SortUnique(region);
+	return region;
+}
+
+bool DuplicateSelectionRegion(Graph& graph, uint32_t header_id, uint32_t merge,
+                              const std::vector<uint32_t>& region, uint32_t block_budget) {
+	std::vector<uint32_t> cloned_blocks;
+	for (auto block_id: region) {
+		if (!graph.Dominates(header_id, block_id)) {
+			cloned_blocks.push_back(block_id);
+		}
+	}
+	if (cloned_blocks.empty() || graph.FindBlock(header_id) == nullptr || header_id >= merge ||
+	    graph.blocks.size() + cloned_blocks.size() + 1u > block_budget) {
+		return false;
+	}
+
+	const auto first_clone = static_cast<uint32_t>(graph.blocks.size());
+	std::map<uint32_t, uint32_t> clones;
+	for (uint32_t i = 0; i < cloned_blocks.size(); i++) {
+		clones.emplace(cloned_blocks[i], first_clone + i);
+	}
+
+	for (auto block_id: cloned_blocks) {
+		BasicBlock clone = *graph.FindBlock(block_id);
+		clone.id         = clones.at(block_id);
+		clone.predecessors.clear();
+		clone.dominators.clear();
+		clone.post_dominators.clear();
+		graph.blocks.push_back(std::move(clone));
+	}
+
+	const auto remap_block = [&](BasicBlock& block) {
+		const auto remap_target = [&](uint32_t& target) {
+			if (const auto it = clones.find(target); it != clones.end()) {
+				target = it->second;
+			}
+		};
+		for (auto& successor: block.successors) {
+			remap_target(successor);
+		}
+		remap_target(block.terminator.true_block);
+		remap_target(block.terminator.false_block);
+		remap_target(block.terminator.merge_block);
+		remap_target(block.terminator.continue_block);
+		for (auto& target: block.terminator.indirect_targets) {
+			remap_target(target);
+		}
+	};
+	for (auto block_id: region) {
+		const auto owned_id = clones.contains(block_id) ? clones.at(block_id) : block_id;
+		remap_block(*graph.FindBlock(owned_id));
+	}
+
+	const auto private_merge = AppendSyntheticBranchBlock(graph, merge);
+	auto&      header        = *graph.FindBlock(header_id);
+	remap_block(header);
+
+	for (auto block_id: region) {
+		const auto owned_id = clones.contains(block_id) ? clones.at(block_id) : block_id;
+		auto*      block    = graph.FindBlock(owned_id);
+		if (block != nullptr) {
+			ReplaceValue(block->successors, merge, private_merge);
+			ReplaceTerminatorTarget(block->terminator, merge, private_merge);
+		}
+	}
+	ReplaceValue(header.successors, merge, private_merge);
+	ReplaceTerminatorTarget(header.terminator, merge, private_merge);
+
+	for (uint32_t i = 0; i <= cloned_blocks.size(); i++) {
+		MoveBlockBefore(graph, first_clone + i, merge + i);
+	}
+	RebuildPredecessors(graph);
+	RecomputeAnalyses(graph);
+	return true;
+}
+
+bool SplitOneSelectionMerge(Graph& graph, uint32_t block_budget) {
 	std::vector<uint32_t> loop_headers;
 	loop_headers.reserve(graph.natural_loops.size());
 	for (const auto& loop: graph.natural_loops) {
@@ -1001,11 +1207,26 @@ bool SplitOneSelectionMerge(Graph& graph) {
 		    Contains(loop_headers, block_id)) {
 			continue;
 		}
+		if (IsInnermostLoopControlConditional(graph, *block)) {
+			continue;
+		}
 
 		const auto merge = graph.FindNearestCommonPostDominator(block->terminator.true_block,
 		                                                        block->terminator.false_block);
+		if (merge == UINT32_MAX || graph.FindBlock(merge) == nullptr) {
+			continue;
+		}
+		const auto region = SelectionRegion(graph, *block, merge);
+		if (std::any_of(region.begin(), region.end(),
+		                [&](uint32_t member) { return !graph.Dominates(block_id, member); })) {
+			if (graph.natural_loops.empty() &&
+			    DuplicateSelectionRegion(graph, block_id, merge, region, block_budget)) {
+				return true;
+			}
+			continue;
+		}
 		const auto construct_blocks = DominatedBlocks(graph, block_id, merge);
-		const auto force_split      = SelectionMergeLeavesContainingLoop(graph, block_id, merge);
+		const auto force_split      = MergeLeavesContainingLoop(graph, block_id, merge);
 		if (SplitSharedMergeBlock(graph, merge, construct_blocks, force_split)) {
 			return true;
 		}
@@ -1015,10 +1236,12 @@ bool SplitOneSelectionMerge(Graph& graph) {
 
 bool SplitSharedMergeBlocks(Graph& graph, std::string* error) {
 	const auto original_block_count = static_cast<uint32_t>(graph.blocks.size());
-	const auto split_budget =
-	    std::max<uint32_t>(16u, std::min<uint32_t>(128u, original_block_count));
+	const auto split_budget = std::max<uint32_t>(
+	    16u, std::min<uint32_t>(128u, original_block_count * 4u));
+	const auto block_budget = std::max<uint32_t>(
+	    32u, std::min<uint32_t>(512u, original_block_count * 8u));
 	for (uint32_t splits = 0; splits < split_budget; splits++) {
-		if (!SplitOneLoopMerge(graph) && !SplitOneSelectionMerge(graph)) {
+		if (!SplitOneLoopMerge(graph) && !SplitOneSelectionMerge(graph, block_budget)) {
 			return true;
 		}
 		RebuildPredecessors(graph);
@@ -1353,6 +1576,9 @@ bool Structurize(Graph& graph, std::string* error) {
 		return false;
 	}
 
+	if (!CanonicalizeNaturalLoops(graph, error)) {
+		return false;
+	}
 	if (!SplitSharedMergeBlocks(graph, error)) {
 		return false;
 	}
@@ -1395,6 +1621,9 @@ bool Structurize(Graph& graph, std::string* error) {
 		    block.terminator.loop_header) {
 			continue;
 		}
+		if (IsInnermostLoopControlConditional(graph, block)) {
+			continue;
+		}
 
 		const auto merge = graph.FindNearestCommonPostDominator(block.terminator.true_block,
 		                                                        block.terminator.false_block);

src/graphics/shader/recompiler/ir/ScalarProvenance.cpp

@@ -45,6 +45,8 @@ namespace {
 
 constexpr uint32_t ScalarRegisters = 128;
 constexpr uint32_t VectorRegisters = 256;
+// Clamp X/Y/Z are consecutive three-bit fields; the high bit of each selects a border mode.
+constexpr uint32_t SamplerBorderClampMask = (1u << 2u) | (1u << 5u) | (1u << 8u);
 
 struct ScalarState {
 	std::array<uint32_t, ScalarRegisters> regs          = {};
@@ -647,6 +649,24 @@ private:
 		return AddDescriptor(descriptor);
 	}
 
+	uint32_t AddSamplerDescriptor(const ScalarState& state, uint32_t base) {
+		if (base >= ScalarRegisters || 4u > ScalarRegisters - base) {
+			return ScalarProvenance::Unknown;
+		}
+		DescriptorValue descriptor;
+		descriptor.dword_count = 4;
+		for (uint32_t i = 0; i < 4; i++) {
+			descriptor.dwords[i] = state.regs[base + i];
+		}
+		if (auto d0 = descriptor.dwords[0];
+		    d0 < m_graph.values.size() && m_graph.values[d0].op == ScalarValueOp::Constant &&
+		    (m_graph.values[d0].imm & SamplerBorderClampMask) == 0) {
+			// Without a border clamp, the border color and table index in dword 3 are unused.
+			descriptor.dwords[3] = Constant(0);
+		}
+		return AddDescriptor(descriptor);
+	}
+
 	uint32_t AddFlatAddressDescriptor(const Instruction& inst, const ScalarState& state) {
 		const uint32_t first = FlatStore(inst.op) ? 1u : 0u;
 		if (inst.src_count < first + 2u) {
@@ -704,7 +724,7 @@ private:
 			inst.memory.resource_source = AddDescriptor(state, inst.memory.resource * 4u, 8);
 			if (inst.op == Opcode::ImageSample || inst.op == Opcode::ImageGather4 ||
 			    inst.op == Opcode::ImageGetLod) {
-				inst.memory.sampler_source = AddDescriptor(state, inst.memory.sampler * 4u, 4);
+				inst.memory.sampler_source = AddSamplerDescriptor(state, inst.memory.sampler * 4u);
 			}
 		}
 	}

src/kernel/memory.cpp

@@ -818,6 +818,11 @@ static void                               MemoryPoolSubtractCommitted(uint64_t l
 // Keep host mappings, physical blocks, placeholders, and virtual ranges in step.
 static std::recursive_mutex g_memory_operation_mutex;
 
+// The base address the PS5 kernel hands out for hint-less user mappings. Guest code can
+// assume mappings it did not place explicitly are at or above this (Sony's libc rejects a
+// heap below it), so hint-less searches must not fall back to the low system-managed range.
+static constexpr uint64_t GUEST_DEFAULT_MAP_BASE = 0x200000000ull;
+
 static uint64_t FindGuestFreeRange(uint64_t search_addr, uint64_t size, uint64_t alignment) {
 	EXIT_IF(g_guest_address_space == nullptr || g_virtual_ranges == nullptr);
 
@@ -845,8 +850,11 @@ static uint64_t FindGuestFreeRange(uint64_t search_addr, uint64_t size, uint64_t
 	if (search_addr != 0) {
 		return find_in(search_addr, HOST_USER_MAX + 1u);
 	}
-	auto addr = find_in(HOST_SYSTEM_MANAGED_MIN, HOST_SYSTEM_MANAGED_MAX + 1u);
-	return addr != 0 ? addr : find_in(HOST_USER_MIN, HOST_USER_MAX + 1u);
+	auto addr = find_in(GUEST_DEFAULT_MAP_BASE, HOST_SYSTEM_MANAGED_MAX + 1u);
+	if (addr == 0) {
+		addr = find_in(HOST_USER_MIN, HOST_USER_MAX + 1u);
+	}
+	return addr;
 }
 
 bool TryWriteBacking(uint64_t vaddr, const void* data, uint64_t size) {

src/kernel/memoryAddressSpace.inc

@@ -1058,6 +1058,76 @@ private:
 		    {HOST_SYSTEM_RESERVED_MIN, HOST_SYSTEM_RESERVED_MAX + 1u},
 		    {HOST_USER_MIN, HOST_USER_MAX + 1u},
 		}};
+
+#if KYTY_PLATFORM == KYTY_PLATFORM_LINUX && !defined(__APPLE__)
+		std::vector<std::pair<uint64_t, uint64_t>> occupied;
+		FILE*                                      maps = fopen("/proc/self/maps", "r");
+		EXIT_IF(maps == nullptr);
+
+		char line[512];
+		while (fgets(line, sizeof(line), maps) != nullptr) {
+			unsigned long long mapping_start = 0;
+			unsigned long long mapping_end   = 0;
+			if (sscanf(line, "%llx-%llx", &mapping_start, &mapping_end) == 2) {
+				occupied.emplace_back(static_cast<uint64_t>(mapping_start),
+				                      static_cast<uint64_t>(mapping_end));
+			}
+		}
+		fclose(maps);
+
+		auto reserve_range = [this](uint64_t start, uint64_t end) {
+			start = AlignUp(start, PageSize());
+			end   = AlignDown(end, PageSize());
+			if (start == 0 || end <= start) {
+				return;
+			}
+
+			const auto size = end - start;
+			int        flags = MAP_PRIVATE | MAP_ANON | MAP_NORESERVE;
+#if defined(KYTY_LINKED_GUEST_ADDRESS_SPACE)
+			flags |= MAP_FIXED;
+#elif defined(MAP_FIXED_NOREPLACE)
+			flags |= MAP_FIXED_NOREPLACE;
+#endif
+
+			void* ptr = mmap(reinterpret_cast<void*>(start), size, PROT_NONE, flags, -1, 0);
+			if (ptr == MAP_FAILED || reinterpret_cast<uint64_t>(ptr) != start) {
+				if (ptr != MAP_FAILED) {
+					munmap(ptr, size);
+				}
+				return;
+			}
+
+			AddFreeUnlocked(start, size);
+			m_owned.emplace_back(start, size);
+		};
+
+		for (const auto& [region_start, region_end]: regions) {
+			auto current = region_start;
+
+			for (const auto& [mapping_start, mapping_end]: occupied) {
+				if (mapping_end <= current) {
+					continue;
+				}
+				if (mapping_start >= region_end) {
+					break;
+				}
+
+				if (mapping_start > current) {
+					reserve_range(current, std::min(mapping_start, region_end));
+				}
+
+				current = std::max(current, mapping_end);
+				if (current >= region_end) {
+					break;
+				}
+			}
+
+			if (current < region_end) {
+				reserve_range(current, region_end);
+			}
+		}
+#else
 		for (const auto& [start, end]: regions) {
 			int flags = MAP_PRIVATE | MAP_ANON | MAP_NORESERVE;
 #if defined(KYTY_LINKED_GUEST_ADDRESS_SPACE)
@@ -1070,13 +1140,14 @@ private:
 				if (ptr != MAP_FAILED) {
 					munmap(ptr, end - start);
 				}
-				EXIT("failed to reserve guest address space at 0x%016" PRIx64 ", size 0x%016" PRIx64
-				     "\n",
+				EXIT("failed to reserve guest address space at 0x%016" PRIx64
+				     ", size 0x%016" PRIx64 "\n",
 				     start, end - start);
 			}
 			AddFreeUnlocked(start, end - start);
 			m_owned.emplace_back(start, end - start);
 		}
+#endif
 #endif
 	}
 

src/kernel/pthread.cpp

@@ -65,6 +65,10 @@
 
 namespace Libs {
 
+namespace LibcInternalExt {
+void RunThreadAtexitDestructors();
+} // namespace LibcInternalExt
+
 namespace LibKernel {
 
 LIB_NAME("libkernel", "libkernel");
@@ -3347,6 +3351,8 @@ int PthreadGetCurrentPriorityForKernel() {
 static void CleanupThread(void* arg) {
 	auto* thread = static_cast<Pthread>(arg);
 
+	LibcInternalExt::RunThreadAtexitDestructors();
+
 	auto thread_dtors = g_pthread_context->GetThreadDtors();
 
 	if (thread_dtors != nullptr) {

src/libs/libC.cpp

@@ -633,6 +633,15 @@ LIB_VERSION("LibcInternalExt", 1, "LibcInternal", 1, 1);
 static uint64_t g_mspace_atomic_id_mask = 0;
 static uint64_t g_mstate_table[64]      = {0};
 
+using thread_atexit_destructor_t = KYTY_SYSV_ABI void (*)(void*);
+
+struct ThreadAtexitDestructor {
+	thread_atexit_destructor_t destructor;
+	void*                      object;
+};
+
+static thread_local std::vector<ThreadAtexitDestructor> g_thread_atexit_destructors;
+
 struct Info {
 	uint64_t  size;
 	uint32_t  unknown1;
@@ -650,25 +659,29 @@ void KYTY_SYSV_ABI LibcHeapGetTraceInfo(Info* info) {
 	info->mstate_table          = g_mstate_table;
 }
 
-uint64_t KYTY_SYSV_ABI LibcInternalExtUnknownQBS714Jr3g(uint64_t arg0, uint64_t arg1, uint64_t arg2,
-                                                        uint64_t arg3, uint64_t arg4,
-                                                        uint64_t arg5) {
+int KYTY_SYSV_ABI LibcInternalExtCxaThreadAtexit(thread_atexit_destructor_t destructor, void* object,
+                                                 void* /*module_id*/) {
 	PRINT_NAME();
 
-	LOGF("\t arg0 = 0x%016" PRIx64 "\n"
-	     "\t arg1 = 0x%016" PRIx64 "\n"
-	     "\t arg2 = 0x%016" PRIx64 "\n"
-	     "\t arg3 = 0x%016" PRIx64 "\n"
-	     "\t arg4 = 0x%016" PRIx64 "\n"
-	     "\t arg5 = 0x%016" PRIx64 "\n",
-	     arg0, arg1, arg2, arg3, arg4, arg5);
+	g_thread_atexit_destructors.push_back({destructor, object});
 
 	return 0;
 }
 
+void RunThreadAtexitDestructors() {
+	while (!g_thread_atexit_destructors.empty()) {
+		auto destructor = g_thread_atexit_destructors.back();
+		g_thread_atexit_destructors.pop_back();
+
+		if (destructor.destructor != nullptr) {
+			destructor.destructor(destructor.object);
+		}
+	}
+}
+
 LIB_DEFINE(InitLibcInternalExt_1) {
 	LIB_FUNC("NWtTN10cJzE", LibcInternalExt::LibcHeapGetTraceInfo);
-	LIB_FUNC("qBS714-Jr3g", LibcInternalExt::LibcInternalExtUnknownQBS714Jr3g);
+	LIB_FUNC("qBS714-Jr3g", LibcInternalExt::LibcInternalExtCxaThreadAtexit);
 }
 
 } // namespace LibcInternalExt

src/loader/runtimeLinker.cpp

@@ -38,7 +38,10 @@
 #include <windows.h>
 #else
 #include <dlfcn.h>
-#if KYTY_PLATFORM == KYTY_PLATFORM_LINUX && !defined(__APPLE__)
+#if defined(__APPLE__)
+#include <mach/mach.h>
+#include <mach/mach_vm.h>
+#elif KYTY_PLATFORM == KYTY_PLATFORM_LINUX
 #include <sys/uio.h>
 #include <unistd.h>
 #endif
@@ -722,7 +725,26 @@ static bool IsReadableRange(uint64_t addr, uint64_t size) {
 		}
 		current = std::min(region_end, end);
 	}
-#elif KYTY_PLATFORM == KYTY_PLATFORM_LINUX && !defined(__APPLE__)
+#elif defined(__APPLE__)
+	// Walk the Mach regions covering the range and require read permission. The fatal
+	// report dumps memory behind raw register values, and a fault inside the reporter
+	// re-enters the signal handler and wedges the reporting thread.
+	uint64_t current = addr;
+	while (current < end) {
+		mach_vm_address_t              region_addr = current;
+		mach_vm_size_t                 region_size = 0;
+		vm_region_basic_info_data_64_t info {};
+		mach_msg_type_number_t         count       = VM_REGION_BASIC_INFO_COUNT_64;
+		mach_port_t                    object_name = MACH_PORT_NULL;
+		if (mach_vm_region(mach_task_self(), &region_addr, &region_size, VM_REGION_BASIC_INFO_64,
+		                   reinterpret_cast<vm_region_info_t>(&info), &count,
+		                   &object_name) != KERN_SUCCESS ||
+		    region_addr > current || (info.protection & VM_PROT_READ) == 0) {
+			return false;
+		}
+		current = region_addr + region_size;
+	}
+#elif KYTY_PLATFORM == KYTY_PLATFORM_LINUX
 	const auto page_size = static_cast<uint64_t>(sysconf(_SC_PAGESIZE));
 	if (page_size == 0) {
 		return false;
@@ -752,7 +774,7 @@ static bool IsReadableRange(uint64_t addr, uint64_t size) {
 }
 
 static bool IsDumpableRange(uint64_t addr, uint64_t size) {
-#if KYTY_PLATFORM == KYTY_PLATFORM_LINUX && !defined(__APPLE__)
+#if KYTY_PLATFORM == KYTY_PLATFORM_LINUX
 	return IsReadableRange(addr, size);
 #else
 	(void)size;

tests/VirtualMemoryAllocationTests.cpp

@@ -1356,6 +1356,135 @@ void TestLargeDirectMapAliasesAcrossChunks() {
 	std::printf("[host]    %-48s ok\n", test);
 }
 
+void TestHintlessDirectMapUsesCanonicalGuestBase() {
+	// Mirrors the allocation Sony's libc.prx makes for its internal heap: 4 MiB of
+	// direct memory, 2 MiB aligned, mapped with no address hint. The PS5 kernel never
+	// places hint-less user mappings below 0x200000000 and guest code relies on that
+	// (libc fails its mspace setup for a lower heap address, and the first malloc then
+	// dereferences a null mspace). Writes through the mapping must also stick.
+	const char* test = "HintlessDirectMapUsesCanonicalGuestBase";
+
+	constexpr uint64_t Len   = 0x400000;
+	constexpr uint64_t Align = 0x200000;
+
+	int64_t phys_addr = 0;
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelAllocateDirectMemory(0, 0x260000000ull, Len, Align, 12,
+	                                                            &phys_addr),
+	        "KernelAllocateDirectMemory");
+
+	void* address = nullptr;
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelMapNamedDirectMemory(&address, Len, SceKernelProtCpuRw,
+	                                                            0, phys_addr, Align, "libc_heap"),
+	        "KernelMapNamedDirectMemory");
+	const auto base = reinterpret_cast<uint64_t>(address);
+	{
+		char message[128] = {};
+		std::snprintf(message, sizeof(message),
+		              "hint-less direct map landed below the PS5 base: 0x%016" PRIx64, base);
+		Check(test, base >= 0x200000000ull, message);
+	}
+
+	auto* header = reinterpret_cast<uint64_t*>(base);
+	header[0]    = 0x4d53504143453030ull; // "MSPACE00"
+	header[7]    = 0x58585858ull;         // magic at +0x38, like the libc mspace
+	*reinterpret_cast<uint64_t*>(base + Len - 8) = 0x454e444d41524bull;
+
+	Check(test, header[0] == 0x4d53504143453030ull, "immediate readback of header[0] failed");
+	Check(test, header[7] == 0x58585858ull, "immediate readback of header[7] failed");
+	Check(test, *reinterpret_cast<const uint64_t*>(base + Len - 8) == 0x454e444d41524bull,
+	      "immediate readback of tail failed");
+
+	uint64_t backing = 0;
+	Check(test, Libs::LibKernel::Memory::TryReadBacking(base + 0x38, &backing, sizeof(backing)),
+	      "TryReadBacking(header+0x38)");
+	Check(test, backing == 0x58585858ull, "backing store does not see the guest write at +0x38");
+
+	CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(base, Len), "KernelMunmap");
+	CheckOk(test, Libs::LibKernel::Memory::KernelReleaseDirectMemory(phys_addr, Len),
+	        "KernelReleaseDirectMemory");
+
+	std::printf("[host]    %-48s ok\n", test);
+}
+
+void TestDirectMemoryContentPersistsAcrossRemap() {
+	const char* test = "DirectMemoryContentPersistsAcrossRemap";
+
+	constexpr uint64_t MapSize = SceKernelPageSize * 4;
+
+	int64_t phys_addr = 0;
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelAllocateDirectMemory(
+	            SceKernelDirectMemoryStart, Libs::LibKernel::Memory::KernelGetDirectMemorySize(),
+	            MapSize, SceKernelPageSize, SceKernelMtypeC, &phys_addr),
+	        "KernelAllocateDirectMemory");
+
+	// Direct memory is physical: contents must survive unmapping and remapping, including
+	// a remap of a sub-range at a nonzero physical offset.
+	void* address = nullptr;
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelMapNamedDirectMemory(&address, MapSize,
+	                                                            SceKernelProtCpuRw, 0, phys_addr,
+	                                                            SceKernelPageSize, "persist_a"),
+	        "KernelMapNamedDirectMemory(first)");
+	const auto base = reinterpret_cast<uint64_t>(address);
+	for (uint64_t offset = 0; offset < MapSize; offset += sizeof(uint64_t)) {
+		*reinterpret_cast<uint64_t*>(base + offset) = offset ^ 0x4b5954595045525aull; // "KYTYPERZ"
+	}
+	CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(base, MapSize), "KernelMunmap(first)");
+
+	void* remap = nullptr;
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelMapNamedDirectMemory(&remap, MapSize,
+	                                                            SceKernelProtCpuRw, 0, phys_addr,
+	                                                            SceKernelPageSize, "persist_b"),
+	        "KernelMapNamedDirectMemory(remap)");
+	const auto remap_base = reinterpret_cast<uint64_t>(remap);
+	for (uint64_t offset = 0; offset < MapSize; offset += sizeof(uint64_t)) {
+		const auto expected = offset ^ 0x4b5954595045525aull;
+		const auto actual   = *reinterpret_cast<const uint64_t*>(remap_base + offset);
+		if (actual != expected) {
+			char message[160] = {};
+			std::snprintf(message, sizeof(message),
+			              "content lost across remap at offset 0x%" PRIx64 ": expected 0x%016" PRIx64
+			              ", read 0x%016" PRIx64,
+			              offset, expected, actual);
+			Fail(test, message);
+		}
+	}
+	CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(remap_base, MapSize), "KernelMunmap(remap)");
+
+	// Sub-range remap at a nonzero physical offset: page 2 of the original allocation.
+	void* partial = nullptr;
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelMapNamedDirectMemory(
+	            &partial, SceKernelPageSize, SceKernelProtCpuRw, 0,
+	            phys_addr + static_cast<int64_t>(SceKernelPageSize * 2), SceKernelPageSize,
+	            "persist_c"),
+	        "KernelMapNamedDirectMemory(partial)");
+	const auto partial_base = reinterpret_cast<uint64_t>(partial);
+	for (uint64_t offset = 0; offset < SceKernelPageSize; offset += sizeof(uint64_t)) {
+		const auto expected = (SceKernelPageSize * 2 + offset) ^ 0x4b5954595045525aull;
+		const auto actual   = *reinterpret_cast<const uint64_t*>(partial_base + offset);
+		if (actual != expected) {
+			char message[160] = {};
+			std::snprintf(message, sizeof(message),
+			              "content lost in partial remap at offset 0x%" PRIx64
+			              ": expected 0x%016" PRIx64 ", read 0x%016" PRIx64,
+			              offset, expected, actual);
+			Fail(test, message);
+		}
+	}
+	CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(partial_base, SceKernelPageSize),
+	        "KernelMunmap(partial)");
+
+	CheckOk(test, Libs::LibKernel::Memory::KernelReleaseDirectMemory(phys_addr, MapSize),
+	        "KernelReleaseDirectMemory");
+
+	std::printf("[host]    %-48s ok\n", test);
+}
+
 void TestDirectMapUnmapReusesHostAddress() {
 	const char* test = "DirectMapUnmapReusesHostAddress";
 
@@ -2150,6 +2279,8 @@ int main() {
 	RunTest(TestDirectAlignmentStaysWithinSearchRange);
 	RunTest(TestDefaultDirectMapUsesSystemAddressRange);
 	RunTest(TestLargeDirectMapAliasesAcrossChunks);
+	RunTest(TestHintlessDirectMapUsesCanonicalGuestBase);
+	RunTest(TestDirectMemoryContentPersistsAcrossRemap);
 	RunTest(TestDirectMapUnmapReusesHostAddress);
 	RunTest(TestFixedReserveReplacesPartialDirectMapping);
 	RunTest(TestFixedReserveRollbackConsumesRestoredPlaceholder);

tests/shaderCfgTests.cpp

@@ -5394,7 +5394,224 @@ void TestNewShaderRecompilerCfgSharedOuterAndLoopMerge() {
 	CheckSpirvBinaryValidates(result.spirv);
 }
 
-void TestNewShaderRecompilerCfgLoopSharedContinueSelectionMerges() {
+void TestNewShaderRecompilerCfgLoopEarlyBreakNoSelection() {
+	const uint32_t shader[] = {
+	    EncodeSopc(0x0a, 0, 129),    // loop: s_cmp_lt_u32 s0, 1
+	    EncodeSopp(0x04, 4),         // loop exit -> end
+	    EncodeSopc(0x06, 1, 1),      // s_cmp_eq_u32 s1, s1
+	    EncodeSopp(0x04, 2),         // early break -> same loop end
+	    EncodeSop2(0x00, 0, 0, 129), // s_add_u32 s0, s0, 1
+	    EncodeSopp(0x02, 0xfffau),   // backedge -> loop header
+	    0xbf810000u,
+	};
+
+	ShaderRecompiler::CompileOptions options;
+	options.stage   = ShaderType::Compute;
+	options.dump_ir = true;
+
+	ShaderRecompiler::CompileResult result;
+	std::string                     error;
+	Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str());
+	Check(Common::ContainsStr(result.ir_dump, "mode=structured"),
+	      "loop early-break CFG did not stay on structured path");
+	Check(SpirvInstructionOpcodeCount(result.spirv, 246) != 0,
+	      "loop early-break SPIR-V lacks OpLoopMerge");
+	Check(SpirvInstructionOpcodeCount(result.spirv, 247) == 0,
+	      "loop early-break SPIR-V unexpectedly used OpSelectionMerge");
+	Check(SpirvInstructionOpcodeCount(result.spirv, 251) == 0,
+	      "loop early-break CFG unexpectedly used dispatcher OpSwitch");
+	CheckSpirvBinaryValidates(result.spirv);
+}
+
+void TestNewShaderRecompilerCfgNestedLoopNonlocalExitDispatcher() {
+	const uint32_t shader[] = {
+	    EncodeSopc(0x0a, 0, 129),    // outer loop: s_cmp_lt_u32 s0, 1
+	    EncodeSopp(0x04, 9),         // outer exit -> end
+	    EncodeSopc(0x0a, 1, 129),    // inner loop: s_cmp_lt_u32 s1, 1
+	    EncodeSopp(0x04, 5),         // inner exit -> outer continue
+	    EncodeSopc(0x06, 2, 2),      // s_cmp_eq_u32 s2, s2
+	    EncodeSopp(0x05, 5),         // nonlocal exit -> outer end
+	    EncodeSMovB32(3, 129),       // inner work
+	    EncodeSop2(0x00, 1, 1, 129), // s_add_u32 s1, s1, 1
+	    EncodeSopp(0x02, 0xfff9u),   // inner backedge
+	    EncodeSop2(0x00, 0, 0, 129), // outer continue: s_add_u32 s0, s0, 1
+	    EncodeSopp(0x02, 0xfff5u),   // outer backedge
+	    0xbf810000u,
+	};
+
+	ShaderRecompiler::CompileOptions options;
+	options.stage   = ShaderType::Compute;
+	options.dump_ir = true;
+
+	ShaderRecompiler::CompileResult result;
+	std::string                     error;
+	Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str());
+	Check(Common::ContainsStr(result.ir_dump, "mode=dispatcher"),
+	      "nested-loop nonlocal exit did not select dispatcher fallback");
+	Check(SpirvInstructionOpcodeCount(result.spirv, 251) != 0,
+	      "nested-loop nonlocal exit dispatcher SPIR-V lacks OpSwitch");
+	CheckSpirvBinaryValidates(result.spirv);
+}
+
+void TestNewShaderRecompilerCfgNestedLoopLocalExitNoSelection() {
+	const uint32_t shader[] = {
+	    EncodeSopc(0x0a, 0, 129),    // outer loop: s_cmp_lt_u32 s0, 1
+	    EncodeSopp(0x04, 6),         // outer exit -> end
+	    EncodeSopc(0x0a, 1, 129),    // inner loop: s_cmp_lt_u32 s1, 1
+	    EncodeSopp(0x04, 2),         // inner exit -> outer continue
+	    EncodeSMovB32(2, 129),       // inner work
+	    EncodeSopp(0x02, 0xfffcu),   // inner backedge
+	    EncodeSop2(0x00, 0, 0, 129), // outer continue: s_add_u32 s0, s0, 1
+	    EncodeSopp(0x02, 0xfff8u),   // outer backedge
+	    0xbf810000u,
+	};
+
+	ShaderRecompiler::CompileOptions options;
+	options.stage   = ShaderType::Compute;
+	options.dump_ir = true;
+
+	ShaderRecompiler::CompileResult result;
+	std::string                     error;
+	Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str());
+	Check(Common::ContainsStr(result.ir_dump, "mode=structured"),
+	      "nested local loop exit did not stay on structured path");
+	Check(SpirvInstructionOpcodeCount(result.spirv, 246) >= 2,
+	      "nested local loop exit SPIR-V lacks both OpLoopMerge instructions");
+	Check(SpirvInstructionOpcodeCount(result.spirv, 247) == 0,
+	      "nested local loop exit SPIR-V unexpectedly used OpSelectionMerge");
+	Check(SpirvInstructionOpcodeCount(result.spirv, 251) == 0,
+	      "nested local loop exit unexpectedly used dispatcher OpSwitch");
+	CheckSpirvBinaryValidates(result.spirv);
+}
+
+void TestNewShaderRecompilerCfgNestedLoopExitTailMergeSplit() {
+	const uint32_t shader[] = {
+	    EncodeSopc(0x0a, 0, 129),    // outer loop: s_cmp_lt_u32 s0, 1
+	    EncodeSopp(0x04, 11),        // outer exit -> end
+	    EncodeSopc(0x06, 1, 1),      // inner loop first exit condition
+	    EncodeSopp(0x05, 3),         // first inner exit -> tail A
+	    EncodeSopc(0x06, 2, 2),      // inner loop second exit condition
+	    EncodeSopp(0x05, 3),         // second inner exit -> tail B
+	    EncodeSopp(0x02, 0xfffbu),   // inner backedge
+	    EncodeSMovB32(3, 129),       // tail A
+	    EncodeSopp(0x02, 2),         // tail A -> outer continue
+	    EncodeSMovB32(4, 129),       // tail B
+	    EncodeSopp(0x02, 0),         // tail B -> outer continue
+	    EncodeSop2(0x00, 0, 0, 129), // outer continue: s_add_u32 s0, s0, 1
+	    EncodeSopp(0x02, 0xfff3u),   // outer backedge
+	    0xbf810000u,
+	};
+
+	ShaderRecompiler::Decoder::Program program;
+	std::string                        error;
+	Check(ShaderRecompiler::Decoder::DecodeProgram(std::span {shader}, program, &error),
+	      error.c_str());
+
+	ShaderRecompiler::CFG::Graph graph;
+	Check(ShaderRecompiler::CFG::BuildGraph(program, graph, &error), error.c_str());
+	const auto original_block_count = graph.blocks.size();
+	Check(ShaderRecompiler::CFG::Structurize(graph, &error), error.c_str());
+	Check(graph.blocks.size() > original_block_count,
+	      "nested loop exit tails did not create a private inner merge");
+
+	const auto* outer_header = graph.FindBlockByPc(0);
+	const auto* inner_header = graph.FindBlockByPc(8);
+	Check(outer_header != nullptr && inner_header != nullptr &&
+	          outer_header->terminator.loop_header && inner_header->terminator.loop_header,
+	      "nested loop exit-tail fixture did not retain both loop headers");
+	Check(inner_header->terminator.merge_block != outer_header->terminator.continue_block,
+	      "inner loop merge still aliases the outer continue target");
+	const auto* inner_merge = graph.FindBlock(inner_header->terminator.merge_block);
+	Check(inner_merge != nullptr && inner_merge->inst_begin == inner_merge->inst_end &&
+	          inner_merge->terminator.kind == ShaderRecompiler::CFG::TerminatorKind::Branch &&
+	          inner_merge->terminator.true_block == outer_header->terminator.continue_block,
+	      "private inner merge does not forward to the outer continue target");
+}
+
+void TestNewShaderRecompilerCfgMixedContinueNonmergeExitDispatcher() {
+	const uint32_t shader[] = {
+	    EncodeSopc(0x06, 7, 7),      // entry branch bypasses loop -> exit X
+	    EncodeSopp(0x05, 5),         // entry -> X
+	    EncodeSopc(0x0a, 0, 129),    // loop: s_cmp_lt_u32 s0, 1
+	    EncodeSopp(0x04, 5),         // loop exit -> Y
+	    EncodeSopc(0x06, 1, 1),      // inner condition
+	    EncodeSopp(0x05, 1),         // nonmerge exit -> X, else continue
+	    EncodeSopp(0x02, 0xfffbu),   // loop backedge
+	    EncodeSMovB32(2, 129),       // X
+	    EncodeSopp(0x02, 2),         // X -> end
+	    EncodeSMovB32(3, 129),       // Y
+	    EncodeSopp(0x02, 0),         // Y -> end
+	    0xbf810000u,
+	};
+
+	ShaderRecompiler::CompileOptions options;
+	options.stage   = ShaderType::Compute;
+	options.dump_ir = true;
+
+	ShaderRecompiler::CompileResult result;
+	std::string                     error;
+	Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str());
+	Check(Common::ContainsStr(result.ir_dump, "mode=dispatcher"),
+	      "mixed continue/nonmerge exit did not select dispatcher fallback");
+	Check(SpirvInstructionOpcodeCount(result.spirv, 251) != 0,
+	      "mixed continue/nonmerge exit dispatcher SPIR-V lacks OpSwitch");
+	CheckSpirvBinaryValidates(result.spirv);
+}
+
+void TestNewShaderRecompilerCfgConditionalLatchNoSelection() {
+	const uint32_t shader[] = {
+	    EncodeSopp(0x02, 0),       // loop header -> conditional block
+	    EncodeSopc(0x06, 0, 0),    // s_cmp_eq_u32 s0, s0
+	    EncodeSopp(0x05, 1),       // loop exit -> end
+	    EncodeSopp(0x02, 0xfffcu), // separate latch -> loop header
+	    0xbf810000u,
+	};
+
+	ShaderRecompiler::CompileOptions options;
+	options.stage   = ShaderType::Compute;
+	options.dump_ir = true;
+
+	ShaderRecompiler::CompileResult result;
+	std::string                     error;
+	Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str());
+	Check(Common::ContainsStr(result.ir_dump, "mode=structured"),
+	      "conditional latch did not stay on structured path");
+	Check(SpirvInstructionOpcodeCount(result.spirv, 246) != 0,
+	      "conditional latch SPIR-V lacks OpLoopMerge");
+	Check(SpirvInstructionOpcodeCount(result.spirv, 247) == 0,
+	      "conditional latch SPIR-V unexpectedly used OpSelectionMerge");
+	Check(SpirvInstructionOpcodeCount(result.spirv, 251) == 0,
+	      "conditional latch unexpectedly used dispatcher OpSwitch");
+	CheckSpirvBinaryValidates(result.spirv);
+}
+
+void TestNewShaderRecompilerCfgDirectConditionalLatchNoSelection() {
+	const uint32_t shader[] = {
+	    EncodeSopp(0x02, 0),       // loop header -> conditional latch
+	    EncodeSopc(0x06, 0, 0),    // s_cmp_eq_u32 s0, s0
+	    EncodeSopp(0x05, 0xfffdu), // direct latch backedge -> loop header
+	    0xbf810000u,
+	};
+
+	ShaderRecompiler::CompileOptions options;
+	options.stage   = ShaderType::Compute;
+	options.dump_ir = true;
+
+	ShaderRecompiler::CompileResult result;
+	std::string                     error;
+	Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str());
+	Check(Common::ContainsStr(result.ir_dump, "mode=structured"),
+	      "direct conditional latch did not stay on structured path");
+	Check(SpirvInstructionOpcodeCount(result.spirv, 246) != 0,
+	      "direct conditional latch SPIR-V lacks OpLoopMerge");
+	Check(SpirvInstructionOpcodeCount(result.spirv, 247) == 0,
+	      "direct conditional latch SPIR-V unexpectedly used OpSelectionMerge");
+	Check(SpirvInstructionOpcodeCount(result.spirv, 251) == 0,
+	      "direct conditional latch unexpectedly used dispatcher OpSwitch");
+	CheckSpirvBinaryValidates(result.spirv);
+}
+
+void TestNewShaderRecompilerCfgLoopEarlyContinuesNoSelection() {
 	const uint32_t shader[] = {
 	    EncodeSMovB32(0, 128),       // s0 = 0
 	    EncodeSopc(0x0a, 0, 130),    // loop: s_cmp_lt_u32 s0, 2
@@ -5419,15 +5636,111 @@ void TestNewShaderRecompilerCfgLoopSharedContinueSelectionMerges() {
 	std::string                     error;
 	Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str());
 	Check(Common::ContainsStr(result.ir_dump, "mode=structured"),
-	      "shared loop continue selections should stay on structured path");
-	Check(!Common::ContainsStr(result.ir_dump, "duplicate structured merge block"),
-	      "shared loop continue selections were not split before structurization");
-	Check(SpirvContainsOpcode(result.spirv, 246),
-	      "shared loop continue selections SPIR-V lacks OpLoopMerge");
-	Check(SpirvContainsOpcode(result.spirv, 247),
-	      "shared loop continue selections SPIR-V lacks OpSelectionMerge");
-	Check(!SpirvContainsOpcode(result.spirv, 251),
-	      "shared loop continue selections unexpectedly used dispatcher OpSwitch");
+	      "loop early continues should stay on structured path");
+	Check(SpirvInstructionOpcodeCount(result.spirv, 246) != 0,
+	      "loop early continues SPIR-V lacks OpLoopMerge");
+	Check(SpirvInstructionOpcodeCount(result.spirv, 247) == 0,
+	      "loop early continues SPIR-V unexpectedly used OpSelectionMerge");
+	Check(SpirvInstructionOpcodeCount(result.spirv, 251) == 0,
+	      "loop early continues unexpectedly used dispatcher OpSwitch");
+	CheckSpirvBinaryValidates(result.spirv);
+}
+
+void TestNewShaderRecompilerCfgConditionalLoopHeaderSelection() {
+	const uint32_t shader[] = {
+	    EncodeSopc(0x06, 0, 0),    // loop body selection condition
+	    EncodeSopp(0x05, 2),       // select path B
+	    EncodeSMovB32(1, 129),     // path A
+	    EncodeSopp(0x02, 1),       // path A -> join
+	    EncodeSMovB32(2, 129),     // path B
+	    EncodeSMovB32(3, 129),     // join
+	    EncodeSopc(0x06, 4, 4),    // repeat condition
+	    EncodeSopp(0x05, 0xfff8u), // repeat -> guest header
+	    0xbf810000u,
+	};
+
+	ShaderRecompiler::Decoder::Program decoded;
+	std::string                        error;
+	Check(ShaderRecompiler::Decoder::DecodeProgram(std::span {shader}, decoded, &error),
+	      error.c_str());
+	ShaderRecompiler::CFG::Graph graph;
+	Check(ShaderRecompiler::CFG::BuildGraph(decoded, graph, &error), error.c_str());
+	const auto original_block_count = graph.blocks.size();
+	Check(ShaderRecompiler::CFG::Structurize(graph, &error), error.c_str());
+	Check(graph.blocks.size() > original_block_count,
+	      "conditional guest loop header did not create a synthetic header");
+
+	uint32_t loop_headers      = 0;
+	uint32_t selection_headers = 0;
+	for (const auto& block: graph.blocks) {
+		if (block.terminator.loop_header) {
+			loop_headers++;
+			Check(block.inst_begin == block.inst_end &&
+			          block.terminator.kind == ShaderRecompiler::CFG::TerminatorKind::Branch,
+			      "canonical loop header is not an empty unconditional block");
+		} else if (block.terminator.kind ==
+		               ShaderRecompiler::CFG::TerminatorKind::ConditionalBranch &&
+		           block.terminator.merge_block != UINT32_MAX) {
+			selection_headers++;
+		}
+	}
+	Check(loop_headers == 1u && selection_headers == 1u,
+	      "guest conditional was not separated from the loop header");
+
+	ShaderRecompiler::CompileOptions options;
+	options.stage = ShaderType::Compute;
+	ShaderRecompiler::CompileResult result;
+	Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str());
+	Check(SpirvInstructionOpcodeCount(result.spirv, 246) == 1u,
+	      "conditional loop-header SPIR-V has the wrong loop-merge count");
+	Check(SpirvInstructionOpcodeCount(result.spirv, 247) == 1u,
+	      "conditional loop-header SPIR-V has the wrong selection-merge count");
+	Check(SpirvInstructionOpcodeCount(result.spirv, 251) == 0u,
+	      "conditional loop-header unexpectedly used dispatcher OpSwitch");
+	CheckSpirvBinaryValidates(result.spirv);
+}
+
+void TestNewShaderRecompilerCfgMultipleLoopLatches() {
+	const uint32_t shader[] = {
+	    EncodeSopc(0x0a, 0, 129),  // loop condition
+	    EncodeSopp(0x04, 5),       // loop exit -> end
+	    EncodeSopc(0x06, 1, 1),    // early repeat condition
+	    EncodeSopp(0x05, 0xfffcu), // early repeat -> header
+	    EncodeSMovB32(2, 129),     // body
+	    EncodeSMovB32(3, 129),     // body tail
+	    EncodeSopp(0x02, 0xfff9u), // ordinary latch -> header
+	    0xbf810000u,
+	};
+
+	ShaderRecompiler::Decoder::Program decoded;
+	std::string                        error;
+	Check(ShaderRecompiler::Decoder::DecodeProgram(std::span {shader}, decoded, &error),
+	      error.c_str());
+	ShaderRecompiler::CFG::Graph graph;
+	Check(ShaderRecompiler::CFG::BuildGraph(decoded, graph, &error), error.c_str());
+	const auto original_block_count = graph.blocks.size();
+	Check(graph.back_edges.size() == 2u, "multiple-latch fixture lacks two native backedges");
+	Check(ShaderRecompiler::CFG::Structurize(graph, &error), error.c_str());
+	Check(graph.blocks.size() == original_block_count + 1u,
+	      "multiple native latches did not create one synthetic continue");
+	Check(graph.back_edges.size() == 1u && graph.natural_loops.size() == 1u,
+	      "multiple native latches were not coalesced to one SPIR-V backedge");
+	const auto& loop           = graph.natural_loops.front();
+	const auto* continue_block = graph.FindBlock(loop.continue_block);
+	Check(continue_block != nullptr && continue_block->inst_begin == continue_block->inst_end &&
+	          continue_block->predecessors.size() == 2u,
+	      "canonical continue does not join both native latches");
+
+	ShaderRecompiler::CompileOptions options;
+	options.stage = ShaderType::Compute;
+	ShaderRecompiler::CompileResult result;
+	Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str());
+	Check(SpirvInstructionOpcodeCount(result.spirv, 246) == 1u,
+	      "multiple-latch SPIR-V has the wrong loop-merge count");
+	Check(SpirvInstructionOpcodeCount(result.spirv, 247) == 0u,
+	      "multiple-latch SPIR-V unexpectedly used a selection merge");
+	Check(SpirvInstructionOpcodeCount(result.spirv, 251) == 0u,
+	      "multiple-latch SPIR-V unexpectedly used dispatcher OpSwitch");
 	CheckSpirvBinaryValidates(result.spirv);
 }
 
@@ -5457,6 +5770,85 @@ void TestNewShaderRecompilerCfgDuplicateMergeStructuredSplit() {
 	CheckSpirvBinaryValidates(result.spirv);
 }
 
+void TestNewShaderRecompilerCfgOverlappingEarlyExitLadder() {
+	const uint32_t shader[] = {
+	    EncodeSopc(0x06, 0, 0), // block 0
+	    EncodeSopp(0x04, 2),    // block 0 -> 2 or 1
+	    EncodeSopc(0x06, 1, 1), // block 1
+	    EncodeSopp(0x04, 6),    // block 1 -> 5 or 2
+	    EncodeSopc(0x06, 2, 2), // block 2
+	    EncodeSopp(0x04, 4),    // block 2 -> 5 or 3
+	    EncodeSopc(0x06, 3, 3), // block 3
+	    EncodeSopp(0x04, 2),    // block 3 -> 5 or 4
+	    EncodeSMovB32(4, 129),  // block 4
+	    0xbf810000u,             // block 4 -> 6
+	    EncodeSMovB32(5, 129),  // block 5
+	    0xbf810000u,             // block 5 -> 6
+	};
+
+	ShaderRecompiler::Decoder::Program decoded;
+	std::string                        error;
+	Check(ShaderRecompiler::Decoder::DecodeProgram(std::span {shader}, decoded, &error),
+	      error.c_str());
+	ShaderRecompiler::CFG::Graph graph;
+	Check(ShaderRecompiler::CFG::BuildGraph(decoded, graph, &error), error.c_str());
+	Check(graph.blocks.size() == 7u && graph.blocks[0].successors == std::vector<uint32_t>({1, 2}) &&
+	          graph.blocks[0].terminator.true_block == 2u &&
+	          graph.blocks[0].terminator.false_block == 1u &&
+	          graph.blocks[1].successors == std::vector<uint32_t>({2, 5}) &&
+	          graph.blocks[1].terminator.true_block == 5u &&
+	          graph.blocks[1].terminator.false_block == 2u &&
+	          graph.blocks[2].successors == std::vector<uint32_t>({3, 5}) &&
+	          graph.blocks[2].terminator.true_block == 5u &&
+	          graph.blocks[2].terminator.false_block == 3u &&
+	          graph.blocks[3].successors == std::vector<uint32_t>({4, 5}) &&
+	          graph.blocks[3].terminator.true_block == 5u &&
+	          graph.blocks[3].terminator.false_block == 4u &&
+	          graph.blocks[4].successors == std::vector<uint32_t>({6}) &&
+	          graph.blocks[5].successors == std::vector<uint32_t>({6}),
+	      "overlapping early-exit fixture does not match the observed shader CFG");
+	Check(ShaderRecompiler::CFG::Structurize(graph, &error), error.c_str());
+	std::vector<bool>     reachable(graph.blocks.size());
+	std::vector<uint32_t> pending = {graph.entry_block};
+	while (!pending.empty()) {
+		const auto block_id = pending.back();
+		pending.pop_back();
+		if (reachable[block_id]) {
+			continue;
+		}
+		reachable[block_id] = true;
+		pending.insert(pending.end(), graph.blocks[block_id].successors.begin(),
+		               graph.blocks[block_id].successors.end());
+	}
+	Check(std::all_of(reachable.begin(), reachable.end(), [](bool value) { return value; }),
+	      "overlapping early-exit structurization left unreachable blocks");
+	std::vector<uint32_t> merges;
+	for (const auto& block: graph.blocks) {
+		if (block.terminator.kind == ShaderRecompiler::CFG::TerminatorKind::ConditionalBranch) {
+			Check(block.terminator.merge_block != UINT32_MAX &&
+			          std::find(merges.begin(), merges.end(), block.terminator.merge_block) ==
+			              merges.end(),
+			      "overlapping early-exit structurization retained a shared merge");
+			merges.push_back(block.terminator.merge_block);
+		}
+	}
+
+	ShaderRecompiler::CompileOptions options;
+	options.stage   = ShaderType::Pixel;
+	options.dump_ir = true;
+	ShaderRecompiler::CompileResult result;
+	Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str());
+	Check(Common::ContainsStr(result.ir_dump, "mode=structured"),
+	      "overlapping early-exit ladder did not stay on the structured path");
+	Check(!Common::ContainsStr(result.ir_dump, "duplicate structured merge block"),
+	      "overlapping early-exit ladder retained a shared merge");
+	Check(SpirvInstructionOpcodeCount(result.spirv, 247) >= 4u,
+	      "overlapping early-exit ladder lost its selections");
+	Check(SpirvInstructionOpcodeCount(result.spirv, 251) == 0u,
+	      "overlapping early-exit ladder used dispatcher OpSwitch");
+	CheckSpirvBinaryValidates(result.spirv);
+}
+
 void TestNewShaderRecompilerCfgIrreducibleDispatcher() {
 	const uint32_t shader[] = {
 	    EncodeSopp(0x05, 2),       // entry -> B, fallthrough A
@@ -7014,7 +7406,6 @@ int main() {
 	using namespace Libs::Graphics;
 
 	EnsureConfigInitialized();
-
 	TestResourceDescriptorClassification();
 	TestNativeShaderResourceDependencies();
 	TestNormalizedImageContracts();
@@ -7086,8 +7477,18 @@ int main() {
 	TestNewShaderRecompilerCfgLoopHeaderBufferLoadDispatcher();
 	TestNewShaderRecompilerCfgLoopHeaderDsAppendConsumeDispatcher();
 	TestNewShaderRecompilerCfgSharedOuterAndLoopMerge();
-	TestNewShaderRecompilerCfgLoopSharedContinueSelectionMerges();
+	TestNewShaderRecompilerCfgLoopEarlyBreakNoSelection();
+	TestNewShaderRecompilerCfgNestedLoopNonlocalExitDispatcher();
+	TestNewShaderRecompilerCfgNestedLoopLocalExitNoSelection();
+	TestNewShaderRecompilerCfgNestedLoopExitTailMergeSplit();
+	TestNewShaderRecompilerCfgMixedContinueNonmergeExitDispatcher();
+	TestNewShaderRecompilerCfgConditionalLatchNoSelection();
+	TestNewShaderRecompilerCfgDirectConditionalLatchNoSelection();
+	TestNewShaderRecompilerCfgLoopEarlyContinuesNoSelection();
+	TestNewShaderRecompilerCfgConditionalLoopHeaderSelection();
+	TestNewShaderRecompilerCfgMultipleLoopLatches();
 	TestNewShaderRecompilerCfgDuplicateMergeStructuredSplit();
+	TestNewShaderRecompilerCfgOverlappingEarlyExitLadder();
 	TestNewShaderRecompilerCfgIrreducibleDispatcher();
 	TestNewShaderRecompilerExecMaskHelpers();
 	TestComputeShaderInputWaveSize();