add TextToSpeech2 stubs

* shader: specialize cube image descriptors

Track whether image descriptors refer to cube maps during resource specialization, and apply the coordinate offset conversion when sampling cube maps as 2D image arrays in SPIR-V emission.

* shader: fix cube array coordinate lowering

---------

Co-authored-by: nmzik <Nmzik@mail.ru>

.github/workflows/build.yml

@@ -83,7 +83,12 @@ jobs:
       - name: Build
         shell: cmd
         run: |
-          cmake --build _Build/windows --target launcher --parallel
+          cmake --build _Build/windows --target launcher virtual_memory_allocation_tests --parallel
+
+      - name: Test
+        shell: cmd
+        run: |
+          ctest --test-dir _Build/windows --output-on-failure -R "^virtual_memory_allocation$"
 
       - name: Install
         shell: cmd
@@ -153,7 +158,15 @@ jobs:
       - name: Build
         shell: bash
         run: |
-          cmake --build _Build/macos --target launcher --parallel
+          cmake --build _Build/macos \
+            --target launcher virtual_memory_allocation_tests \
+            --parallel
+
+      - name: Test
+        shell: bash
+        run: |
+          ctest --test-dir _Build/macos --output-on-failure \
+            -R '^virtual_memory_allocation$'
 
       - name: Install
         shell: bash
@@ -284,13 +297,14 @@ jobs:
         run: |
           cmake --build _Build/linux \
             --target launcher page_manager_tests memory_tracker_tests \
+              virtual_memory_allocation_tests \
             --parallel
 
       - name: Test
         shell: bash
         run: |
           ctest --test-dir _Build/linux --output-on-failure \
-            -R '^(page_manager|memory_tracker)$'
+            -R '^(page_manager|memory_tracker|virtual_memory_allocation)$'
 
       - name: Install
         shell: bash

README.md

@@ -50,7 +50,7 @@ graphical glitches, low compatibility, and poor performance.
   </tr>
   <tr>
     <td align="center">
-      <strong>Minecraft Legends</strong><br>
+      <strong>Neptunia ReVerse</strong><br>
       <img src="docs/screenshots/ps5-04.png" width="300" alt="Minecraft Legends running in KytyPS5">
     </td>
     <td align="center">
@@ -58,8 +58,20 @@ graphical glitches, low compatibility, and poor performance.
       <img src="docs/screenshots/ps5-05.png" width="300" alt="SILENT HILL: The Short Message running in KytyPS5">
     </td>
   </tr>
+  <tr>
+    <td align="center">
+      <strong>Hellboy</strong><br>
+      <img src="docs/screenshots/ps5-02.png" width="300" alt="Disgaea 6 running in KytyPS5">
+    </td>
+    <td align="center">
+      <strong>Paleo Pines</strong><br>
+      <img src="docs/screenshots/ps5-06.png" width="300" alt="Dreaming Sarah running in KytyPS5">
+    </td>
+  </tr>
 </table>
 
+<p align="center"><em>And many more...</em></p>
+
 ## Contributing
 
 Testing games and submitting detailed bug reports are useful ways to contribute. Search existing

src/CMakeLists.txt

@@ -314,6 +314,16 @@ function(add_kyty_full_emulator_test target source)
 	endif()
 endfunction()
 
+function(configure_macos_guest_address_space target)
+	if(APPLE AND (CMAKE_OSX_ARCHITECTURES STREQUAL "x86_64" OR
+			(NOT CMAKE_OSX_ARCHITECTURES AND CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|AMD64)$")))
+		target_sources(${target} PRIVATE kernel/macosGuestAddressSpace.cpp)
+		target_compile_definitions(${target} PRIVATE KYTY_LINKED_GUEST_ADDRESS_SPACE=1)
+		target_link_options(${target} PRIVATE
+			-Wl,-ld_classic,-no_pie,-no_fixup_chains,-no_huge,-pagezero_size,0x40000,-segaddr,SYSTEM_MANAGED,0x40000,-segaddr,SYSTEM_RESERVED,0x7ffffc000,-segaddr,USER_AREA,0x7000000000,-image_base,0x700000000000)
+	endif()
+endfunction()
+
 add_kyty_full_emulator_test(shader_cfg_tests ../tests/shaderCfgTests.cpp)
 
 add_executable(scalar_provenance_tests EXCLUDE_FROM_ALL
@@ -338,7 +348,6 @@ add_executable(memory_tracker_tests EXCLUDE_FROM_ALL
 )
 target_link_libraries(memory_tracker_tests fmt::fmt common)
 target_include_directories(memory_tracker_tests PRIVATE ${inc_headers})
-target_compile_definitions(memory_tracker_tests PRIVATE KYTY_MEMORY_TRACKER_TESTS=1)
 
 add_executable(shader_vertex_metadata_tests EXCLUDE_FROM_ALL
 	../tests/ShaderVertexMetadataTests.cpp
@@ -421,6 +430,7 @@ target_sources(shader_recompiler_compute_tests PRIVATE
 add_kyty_full_emulator_test(virtual_memory_allocation_tests ../tests/VirtualMemoryAllocationTests.cpp)
 target_compile_definitions(virtual_memory_allocation_tests PRIVATE
 	KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS=1)
+configure_macos_guest_address_space(virtual_memory_allocation_tests)
 
 # These tests use exceptions.
 if(NOT KYTY_CLANG_CL)
@@ -437,6 +447,8 @@ if(BUILD_TESTING)
 	add_test(NAME resource_mutex COMMAND $<TARGET_FILE:resource_mutex_tests>)
 	add_test(NAME event_queue_lifetime COMMAND $<TARGET_FILE:event_queue_lifetime_tests>)
 	add_test(NAME shader_recompiler_compute COMMAND $<TARGET_FILE:shader_recompiler_compute_tests>)
+	add_test(NAME virtual_memory_allocation
+		COMMAND $<TARGET_FILE:virtual_memory_allocation_tests>)
 	add_test(NAME command_scheduler_timeline
 		COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --scheduler-only)
 	add_test(NAME stream_buffer_ring
@@ -470,6 +482,7 @@ endif()
 
 
 add_executable(kyty_emulator main.cpp ${kyty_emulator_src})
+configure_macos_guest_address_space(kyty_emulator)
 
 target_link_libraries(kyty_emulator ${kyty_emulator_link_libraries})
 if (WIN32)

src/common/hostException.cpp

@@ -324,6 +324,10 @@ bool InstallHandler(Handler handler) {
 	sa.sa_sigaction = SignalHandler;
 	sa.sa_flags     = SA_SIGINFO;
 	sigemptyset(&sa.sa_mask);
+	// The guest signal-dispatch path (KernelRaiseException) interrupts threads with
+	// SIGUSR1; block it while a fault is being resolved so a stop-the-world request
+	// cannot preempt the handler between the protection fix and the retry.
+	sigaddset(&sa.sa_mask, SIGUSR1);
 
 	// macOS raises SIGBUS for protection faults on some paths and SIGSEGV on others;
 	// SIGILL covers instructions the host cannot execute (routed to the x64 emulator).

src/common/platform/sysLinuxVirtual.cpp

@@ -432,11 +432,6 @@ uint64_t SysVirtualReserveAligned(uint64_t address, uint64_t size, uint64_t alig
 
 	pthread_mutex_lock(&g_virtual_mutex);
 	record_alloc(ret_addr, size);
-	uintptr_t page_start  = ret_addr >> 12u;
-	uintptr_t page_end    = (ret_addr + size - 1) >> 12u;
-	for (uintptr_t page = page_start; page <= page_end; page++) {
-		(*g_protects)[page] = PROT_NONE;
-	}
 	pthread_mutex_unlock(&g_virtual_mutex);
 
 	return ret_addr;
@@ -470,11 +465,6 @@ bool SysVirtualReserveFixed(uint64_t address, uint64_t size) {
 	if (ptr != MAP_FAILED) {
 		pthread_mutex_lock(&g_virtual_mutex);
 		record_alloc(ret_addr, size);
-		uintptr_t page_start  = ret_addr >> 12u;
-		uintptr_t page_end    = (ret_addr + size - 1) >> 12u;
-		for (uintptr_t page = page_start; page <= page_end; page++) {
-			(*g_protects)[page] = PROT_NONE;
-		}
 		pthread_mutex_unlock(&g_virtual_mutex);
 
 		return true;

src/common/virtualMemory.cpp

@@ -58,25 +58,6 @@ bool FlushInstructionCache(uint64_t address, uint64_t size) {
 	return SysVirtualFlushInstructionCache(address, size);
 }
 
-bool PatchReplace(uint64_t vaddr, uint64_t value) {
-	Mode old_mode {};
-	Protect(vaddr, 8, Mode::ReadWrite, &old_mode);
-
-	auto* ptr = reinterpret_cast<uint64_t*>(vaddr);
-
-	bool ret = (*ptr != value);
-
-	*ptr = value;
-
-	Protect(vaddr, 8, old_mode);
-
-	if (IsExecute(old_mode)) {
-		FlushInstructionCache(vaddr, 8);
-	}
-
-	return ret;
-}
-
 } // namespace VirtualMemory
 
 } // namespace Common

src/common/virtualMemory.h

@@ -37,7 +37,6 @@ bool     Free(uint64_t address);
 bool     FreeRange(uint64_t address, uint64_t size);
 bool     Protect(uint64_t address, uint64_t size, Mode mode, Mode* old_mode = nullptr);
 bool     FlushInstructionCache(uint64_t address, uint64_t size);
-bool     PatchReplace(uint64_t vaddr, uint64_t value);
 
 } // namespace VirtualMemory
 

src/emulator.cpp

@@ -105,7 +105,7 @@ static void ClearDebugTextureFolder() {
 	}
 }
 
-static void Init(const Config::ConfigOptions& cfg) {
+static void Init(const Config::ConfigOptions& cfg, const std::filesystem::path& param_json) {
 	EXIT_IF(!Common::Thread::IsMainThread());
 
 	auto* slist = Common::SubsystemsList::Instance();
@@ -127,12 +127,21 @@ static void Init(const Config::ConfigOptions& cfg) {
 	slist->InitAll(true);
 
 	Config::Load(cfg);
+	slist->Add(log, {core, config});
+	slist->InitAll(true);
+
+	if (Common::File::IsFileExisting(param_json)) {
+		Loader::SystemContentLoadParamSfo(param_json);
+		if (const auto flexible_memory_size = Loader::SystemContentGetFlexibleMemorySize();
+		    flexible_memory_size != 0) {
+			Libs::LibKernel::Memory::SetFlexibleMemorySize(flexible_memory_size);
+		}
+	}
 
 	slist->Add(audio, {core, log, pthread, memory});
 	slist->Add(controller, {core, log, config});
 	slist->Add(file_system, {core, log, pthread});
 	slist->Add(graphics, {core, log, pthread, memory, config, profiler, controller});
-	slist->Add(log, {core, config});
 	slist->Add(memory, {core, log});
 	slist->Add(network, {core, log, pthread});
 	slist->Add(profiler, {core, config});
@@ -180,7 +189,8 @@ void Run(const RunOptions& options) {
 		EXIT("ELF is required\n");
 	}
 
-	Init(options.config);
+	const auto param_json = options.app0_dir / "sce_sys" / "param.json";
+	Init(options.config, param_json);
 
 	ClearDebugTextureFolder();
 
@@ -192,15 +202,6 @@ void Run(const RunOptions& options) {
 	Libs::LibKernel::FileSystem::Mount(options.app0_dir, "/app0");
 	Libs::LibKernel::FileSystem::Mount(options.app0_dir, "/hostapp");
 
-	auto param_json = options.app0_dir / "sce_sys" / "param.json";
-	if (Common::File::IsFileExisting(param_json)) {
-		Loader::SystemContentLoadParamSfo(param_json);
-		if (auto flexible_memory_size = Loader::SystemContentGetFlexibleMemorySize();
-		    flexible_memory_size != 0) {
-			Libs::LibKernel::Memory::SetFlexibleMemorySize(flexible_memory_size);
-		}
-	}
-
 	MountSandboxDirs();
 
 	auto* rt = Common::Singleton<Loader::RuntimeLinker>::Instance();

src/graphics/host_gpu/memoryTracker.cpp

@@ -4,16 +4,6 @@
 
 namespace Libs::Graphics {
 
-#if defined(KYTY_MEMORY_TRACKER_TESTS)
-namespace {
-std::atomic<MemoryTracker::UnmapContentionHook> g_unmap_contention_hook {nullptr};
-}
-
-void MemoryTracker::SetUnmapContentionHook(UnmapContentionHook hook) noexcept {
-	g_unmap_contention_hook.store(hook, std::memory_order_release);
-}
-#endif
-
 static_assert(std::atomic<void*>::is_always_lock_free);
 
 MemoryTracker::MemoryTracker(PageManager& page_manager, PageWatchMode gpu_watch_mode)
@@ -94,7 +84,6 @@ RegionManager* MemoryTracker::GetOrCreateRegion(uint64_t index) {
 bool MemoryTracker::IsRegionCpuModified(uint64_t vaddr, uint64_t size) {
 	CheckNotInUploadCallback();
 	std::lock_guard access(m_access_mutex);
-	RequireMapped(vaddr, size);
 	return Iterate<true>(vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) {
 		std::scoped_lock lock(manager->lock);
 		return manager->IsModified<DirtySource::Cpu>(offset, bytes);
@@ -104,7 +93,6 @@ bool MemoryTracker::IsRegionCpuModified(uint64_t vaddr, uint64_t size) {
 bool MemoryTracker::IsRegionGpuModified(uint64_t vaddr, uint64_t size) {
 	CheckNotInUploadCallback();
 	std::lock_guard access(m_access_mutex);
-	RequireMapped(vaddr, size);
 	return Iterate<false>(vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) {
 		std::scoped_lock lock(manager->lock);
 		return manager->IsModified<DirtySource::Gpu>(offset, bytes);
@@ -114,7 +102,6 @@ bool MemoryTracker::IsRegionGpuModified(uint64_t vaddr, uint64_t size) {
 void MemoryTracker::MarkRegionAsCpuModified(uint64_t vaddr, uint64_t size) {
 	CheckNotInUploadCallback();
 	std::lock_guard access(m_access_mutex);
-	RequireMapped(vaddr, size);
 	Iterate<true>(vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) {
 		std::scoped_lock lock(manager->lock);
 		const auto       changed =
@@ -126,7 +113,6 @@ void MemoryTracker::MarkRegionAsCpuModified(uint64_t vaddr, uint64_t size) {
 void MemoryTracker::MarkRegionAsGpuModified(uint64_t vaddr, uint64_t size) {
 	CheckNotInUploadCallback();
 	std::lock_guard access(m_access_mutex);
-	RequireMapped(vaddr, size);
 	Iterate<true>(vaddr, size, [this](RegionManager* manager, uint64_t offset, uint64_t bytes) {
 		std::scoped_lock lock(manager->lock);
 		const auto       changed =
@@ -138,7 +124,6 @@ void MemoryTracker::MarkRegionAsGpuModified(uint64_t vaddr, uint64_t size) {
 void MemoryTracker::UnmarkRegionAsGpuModified(uint64_t vaddr, uint64_t size) {
 	CheckNotInUploadCallback();
 	std::lock_guard access(m_access_mutex);
-	RequireMapped(vaddr, size);
 	Iterate<true>(vaddr, size, [this](RegionManager* manager, uint64_t offset, uint64_t bytes) {
 		std::scoped_lock lock(manager->lock);
 		if (!manager->IsFullyModified<DirtySource::Gpu>(offset, bytes)) {
@@ -151,8 +136,6 @@ void MemoryTracker::UnmarkRegionAsGpuModified(uint64_t vaddr, uint64_t size) {
 }
 
 void MemoryTracker::UntrackMemoryLocked(uint64_t vaddr, uint64_t size) {
-	RequireMapped(vaddr, size);
-
 	std::vector<RegionManager*> managers;
 	managers.reserve((vaddr % TRACKER_REGION_SIZE + size + TRACKER_REGION_SIZE - 1) /
 	                 TRACKER_REGION_SIZE);
@@ -185,22 +168,6 @@ void MemoryTracker::UntrackMemory(uint64_t vaddr, uint64_t size) {
 	UntrackMemoryLocked(vaddr, size);
 }
 
-void MemoryTracker::UnmapMemory(uint64_t vaddr, uint64_t size) {
-	CheckNotInUploadCallback();
-	std::unique_lock access(m_access_mutex, std::try_to_lock);
-	if (!access.owns_lock()) {
-#if defined(KYTY_MEMORY_TRACKER_TESTS)
-		if (const auto hook = g_unmap_contention_hook.load(std::memory_order_acquire);
-		    hook != nullptr) {
-			hook();
-		}
-#endif
-		access.lock();
-	}
-	UntrackMemoryLocked(vaddr, size);
-	m_page_manager.OnGpuUnmap(vaddr, size);
-}
-
 bool MemoryTracker::InvalidateRegion(uint64_t vaddr, uint64_t size, PageFaultPhase phase) noexcept {
 	switch (phase) {
 		case PageFaultPhase::Release: return true;

src/graphics/host_gpu/memoryTracker.h

@@ -30,7 +30,6 @@ public:
 	void               MarkRegionAsGpuModified(uint64_t vaddr, uint64_t size);
 	void               UnmarkRegionAsGpuModified(uint64_t vaddr, uint64_t size);
 	void               UntrackMemory(uint64_t vaddr, uint64_t size);
-	void               UnmapMemory(uint64_t vaddr, uint64_t size);
 	[[nodiscard]] CpuFaultAction
 	                   BeginCpuFault(uint64_t vaddr, uint64_t size,
 	                                 PageFaultAccess access = PageFaultAccess::Write) noexcept;
@@ -91,8 +90,7 @@ public:
 		static_assert(std::is_nothrow_invocable_v<Preflight&, uint64_t, uint64_t>);
 		static_assert(std::is_nothrow_invocable_v<Func&, uint64_t, uint64_t>);
 		CheckNotInUploadCallback();
-		std::lock_guard access(m_access_mutex);
-		RequireMapped(vaddr, size);
+		std::lock_guard             access(m_access_mutex);
 		std::vector<RegionManager*> managers;
 		Iterate<false>(vaddr, size, [&](RegionManager* manager, uint64_t, uint64_t) {
 			managers.push_back(manager);
@@ -132,11 +130,6 @@ public:
 		    vaddr, size, [](uint64_t, uint64_t) noexcept {}, std::forward<Func>(func));
 	}
 
-#if defined(KYTY_MEMORY_TRACKER_TESTS)
-	using UnmapContentionHook = void (*)() noexcept;
-	static void SetUnmapContentionHook(UnmapContentionHook hook) noexcept;
-#endif
-
 	template <typename RangeFunc, typename UploadFunc>
 	void ForEachUploadRange(uint64_t vaddr, uint64_t size, bool is_written, RangeFunc&& range_func,
 	                        UploadFunc&& upload_func) {
@@ -144,7 +137,6 @@ public:
 		static_assert(std::is_nothrow_invocable_v<UploadFunc&>);
 		CheckNotInUploadCallback();
 		std::unique_lock access(m_access_mutex);
-		RequireMapped(vaddr, size);
 		Iterate<true>(vaddr, size, [](RegionManager*, uint64_t, uint64_t) {});
 		const auto* previous_upload_owner = std::exchange(s_upload_owner, this);
 		Iterate<false>(vaddr, size, [&](RegionManager* manager, uint64_t offset, uint64_t bytes) {
@@ -209,16 +201,8 @@ private:
 		return false;
 	}
 
-	static void ValidateRange(uint64_t vaddr, uint64_t size);
-	void        UntrackMemoryLocked(uint64_t vaddr, uint64_t size);
-	void        RequireMapped(uint64_t vaddr, uint64_t size) const {
-		ValidateRange(vaddr, size);
-		if (!m_page_manager.IsMapped(vaddr, size)) {
-			EXIT("memory tracker range [0x%llx, 0x%llx) is not mapped\n",
-			     static_cast<unsigned long long>(vaddr),
-			     static_cast<unsigned long long>(vaddr + size));
-		}
-	}
+	static void    ValidateRange(uint64_t vaddr, uint64_t size);
+	void           UntrackMemoryLocked(uint64_t vaddr, uint64_t size);
 	RegionManager* GetOrCreateRegion(uint64_t index);
 
 	std::unique_ptr<std::atomic<RegionManager*>[]> m_regions;

src/graphics/host_gpu/pageManager.cpp

@@ -1,6 +1,7 @@
 #include "graphics/host_gpu/pageManager.h"
 
 #include "graphics/host_gpu/regionDefinitions.h"
+#include "kernel/memory.h"
 
 #include <algorithm>
 #include <array>
@@ -21,16 +22,11 @@
 #undef min
 #undef max
 #elif defined(__APPLE__)
-#include <mach/mach.h>
-#include <mach/mach_vm.h>
 #include <pthread.h>
 #include <sys/mman.h>
 #include <unistd.h>
 #else
-#include <cerrno>
-#include <cstring>
 #include <execinfo.h>
-#include <fcntl.h>
 #include <sys/mman.h>
 #include <sys/syscall.h>
 #include <unistd.h>
@@ -57,45 +53,8 @@ constexpr uint64_t REGION_PAGES = REGION_SIZE / PAGE_SIZE;
 constexpr uint32_t NO_ACCESS_PROTECTION  = PAGE_NOACCESS;
 constexpr uint32_t READ_ONLY_PROTECTION  = PAGE_READONLY;
 constexpr uint32_t READ_WRITE_PROTECTION = PAGE_READWRITE;
-
-#if defined(__APPLE__)
-// Map the tracker's Win32-style protection tags to POSIX mprotect flags.
-static int PageProtToPosix(uint32_t protection) {
-	switch (protection) {
-		case PAGE_NOACCESS: return PROT_NONE;
-		case PAGE_READONLY: return PROT_READ;
-		case PAGE_READWRITE: return PROT_READ | PROT_WRITE;
-		default: return PROT_NONE;
-	}
-}
-
-// Query the current protection of the page containing vaddr via the Mach VM map and
-// collapse it to the tracker's read/write tags (execute is irrelevant to write tracking).
-static uint32_t MachQueryPageProt(uint64_t vaddr) {
-	auto                           region_addr = static_cast<mach_vm_address_t>(vaddr);
-	mach_vm_size_t                 region_size = 0;
-	vm_region_basic_info_data_64_t info {};
-	mach_msg_type_number_t         count       = VM_REGION_BASIC_INFO_COUNT_64;
-	mach_port_t                    object_name = MACH_PORT_NULL;
-
-	kern_return_t kr =
-	    mach_vm_region(mach_task_self(), &region_addr, &region_size, VM_REGION_BASIC_INFO_64,
-	                   reinterpret_cast<vm_region_info_t>(&info), &count, &object_name);
-	if (kr != KERN_SUCCESS || region_addr > vaddr) {
-		return PAGE_NOACCESS; // no region covering vaddr
-	}
-	if ((info.protection & VM_PROT_WRITE) != 0) {
-		return PAGE_READWRITE;
-	}
-	if ((info.protection & VM_PROT_READ) != 0) {
-		return PAGE_READONLY;
-	}
-	return PAGE_NOACCESS;
-}
-#elif defined(__linux__)
 // Zero is the unknown protection sentinel.
 constexpr uint32_t UNKNOWN_PROTECTION = 0;
-#endif
 
 thread_local bool g_in_fault_resolution = false;
 
@@ -136,6 +95,15 @@ thread_local bool g_in_fault_resolution = false;
 	std::_Exit(322);
 }
 
+Common::VirtualMemory::Mode ToMemoryMode(uint32_t protection) {
+	switch (protection) {
+		case NO_ACCESS_PROTECTION: return Common::VirtualMemory::Mode::NoAccess;
+		case READ_ONLY_PROTECTION: return Common::VirtualMemory::Mode::Read;
+		case READ_WRITE_PROTECTION: return Common::VirtualMemory::Mode::ReadWrite;
+		default: Fatal("unmappable protection 0x%08" PRIx32, protection);
+	}
+}
+
 uint32_t CurrentThread() noexcept {
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
 	return GetCurrentThreadId();
@@ -155,130 +123,6 @@ uint32_t CurrentThread() noexcept {
 #endif
 }
 
-#if defined(__linux__)
-int ToHostProtection(uint32_t protection) {
-	switch (protection) {
-		case NO_ACCESS_PROTECTION: return PROT_NONE;
-		case READ_ONLY_PROTECTION: return PROT_READ;
-		case READ_WRITE_PROTECTION: return PROT_READ | PROT_WRITE;
-		default: Fatal("unmappable protection 0x%08" PRIx32, protection);
-	}
-}
-
-struct HostMapping {
-	uint64_t end        = 0;
-	uint32_t protection = UNKNOWN_PROTECTION;
-};
-
-// Async-signal-safe lookup in the address-ordered /proc/self/maps.
-HostMapping QueryHostMapping(uint64_t vaddr) noexcept {
-	int fd = ::open("/proc/self/maps", O_RDONLY | O_CLOEXEC); // NOLINT
-	if (fd < 0) {
-		return {};
-	}
-
-	enum class Field { Start, End, Perms, Rest };
-
-	HostMapping result {};
-	auto        field      = Field::Start;
-	uint64_t    start      = 0;
-	uint64_t    end        = 0;
-	char        perms[4]   = {};
-	uint32_t    perms_len  = 0;
-	bool        line_valid = true;
-
-	char buffer[8192];
-
-	for (bool done = false; !done;) {
-		const auto got = ::read(fd, buffer, sizeof(buffer));
-		if (got < 0) {
-			if (errno == EINTR) {
-				continue;
-			}
-			break;
-		}
-		if (got == 0) {
-			break;
-		}
-
-		for (ssize_t i = 0; i < got && !done; i++) {
-			const char c = buffer[i];
-
-			if (c == '\n') {
-				field      = Field::Start;
-				start      = 0;
-				end        = 0;
-				perms_len  = 0;
-				line_valid = true;
-				continue;
-			}
-
-			if (!line_valid) {
-				continue;
-			}
-
-			switch (field) {
-				case Field::Start:
-				case Field::End: {
-					uint64_t digit = 0;
-					if (c >= '0' && c <= '9') {
-						digit = static_cast<uint64_t>(c - '0');
-					} else if (c >= 'a' && c <= 'f') {
-						digit = static_cast<uint64_t>(c - 'a') + 10;
-					} else if (c == '-' && field == Field::Start) {
-						field = Field::End;
-						break;
-					} else if (c == ' ' && field == Field::End) {
-						field     = Field::Perms;
-						perms_len = 0;
-						break;
-					} else {
-						line_valid = false;
-						break;
-					}
-
-					auto& value = (field == Field::Start ? start : end);
-					value       = (value << 4u) | digit;
-					break;
-				}
-
-				case Field::Perms: {
-					if (c != ' ') {
-						if (perms_len < sizeof(perms)) {
-							perms[perms_len] = c;
-						}
-						perms_len++;
-						break;
-					}
-
-					if (vaddr < start) {
-						done = true;
-					} else if (vaddr < end && perms_len >= 2) {
-						result.end        = end;
-						result.protection = perms[1] == 'w'   ? READ_WRITE_PROTECTION
-						                    : perms[0] == 'r' ? READ_ONLY_PROTECTION
-						                                      : NO_ACCESS_PROTECTION;
-						done              = true;
-					} else {
-						field = Field::Rest;
-					}
-					break;
-				}
-
-				case Field::Rest: break;
-			}
-		}
-	}
-
-	::close(fd);
-	return result;
-}
-
-uint32_t QueryHostProtection(uint64_t vaddr) noexcept {
-	return QueryHostMapping(vaddr).protection;
-}
-#endif
-
 class SpinGuard final {
 public:
 	explicit SpinGuard(std::atomic_flag& lock): m_lock(lock) {
@@ -313,21 +157,16 @@ uint64_t PageEnd(uint64_t vaddr, uint64_t size) {
 struct PageManager::Impl {
 	struct PageState {
 		std::atomic_flag lock                = ATOMIC_FLAG_INIT;
-		uint32_t         mappings            = 0;
-		uint32_t         gpu_read_mappings   = 0;
-		uint32_t         gpu_write_mappings  = 0;
 		uint32_t         write_watchers      = 0;
 		uint32_t         access_watchers     = 0;
 		uint32_t         original_protection = 0;
 		uint32_t         backing_writer      = 0;
-#if defined(__linux__)
 		// Shadow the protection applied through Protect().
-		uint32_t current_protection = UNKNOWN_PROTECTION;
-#endif
-		bool resolving            = false;
-		bool resolving_read_write = false;
-		bool late_read_pending    = false;
-		bool late_write_pending   = false;
+		uint32_t current_protection   = UNKNOWN_PROTECTION;
+		bool     resolving            = false;
+		bool     resolving_read_write = false;
+		bool     late_read_pending    = false;
+		bool     late_write_pending   = false;
 	};
 
 	struct Region {
@@ -356,7 +195,7 @@ struct PageManager::Impl {
 
 	Impl(PageFaultHandler handler, void* context): fault_handler(handler), fault_context(context) {
 		if (fault_handler == nullptr) {
-			Fatal("null fault handler");
+			Fatal("null page-manager fault callback");
 		}
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
 		SYSTEM_INFO info {};
@@ -386,9 +225,8 @@ struct PageManager::Impl {
 		for (const auto& region: region_storage) {
 			for (auto& page: region->pages) {
 				SpinGuard lock(page.lock);
-				if (page.mappings != 0 || page.gpu_read_mappings != 0 ||
-				    page.gpu_write_mappings != 0 || page.write_watchers != 0 ||
-				    page.access_watchers != 0 || page.backing_writer != 0 || page.resolving) {
+				if (page.write_watchers != 0 || page.access_watchers != 0 ||
+				    page.backing_writer != 0 || page.resolving) {
 					FailFast("PageManager destroyed with live page state");
 				}
 			}
@@ -441,179 +279,30 @@ struct PageManager::Impl {
 		}
 	}
 
-	static void ValidateInitialProtection(std::span<PageState*> pages, uint64_t vaddr) {
-		const auto end = vaddr + pages.size() * PAGE_SIZE;
-#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
-		for (auto address = vaddr; address < end;) {
-			MEMORY_BASIC_INFORMATION info {};
-			if (VirtualQuery(reinterpret_cast<const void*>(static_cast<uintptr_t>(address)), &info,
-			                 sizeof(info)) == 0 ||
-			    info.State != MEM_COMMIT || info.Protect != PAGE_READWRITE) {
-				Fatal("basic path requires PAGE_READWRITE at 0x%016" PRIx64 " (state=0x%08" PRIx32
-				      ", protection=0x%08" PRIx32 ")",
-				      address, static_cast<uint32_t>(info.State),
-				      static_cast<uint32_t>(info.Protect));
-			}
-			const auto region_end = reinterpret_cast<uint64_t>(info.BaseAddress) + info.RegionSize;
-			if (region_end <= address) {
-				Fatal("VirtualQuery returned an invalid region at 0x%016" PRIx64, address);
-			}
-			address = std::min(end, region_end);
-		}
-#elif defined(__APPLE__)
-		for (auto address = vaddr; address < end; address += PAGE_SIZE) {
-			const uint32_t protection = MachQueryPageProt(address);
-			if (protection != PAGE_READWRITE) {
-				Fatal("basic path requires PAGE_READWRITE at 0x%016" PRIx64
-				      " (protection=0x%08" PRIx32 ")",
-				      address, protection);
-			}
-		}
-#else
-		for (auto address = vaddr; address < end;) {
-			const auto mapping = QueryHostMapping(address);
-			if (mapping.protection != READ_WRITE_PROTECTION || mapping.end <= address) {
-				Fatal("basic path requires a read/write mapping at 0x%016" PRIx64
-				      " (protection=0x%08" PRIx32 ")",
-				      address, mapping.protection);
-			}
-			address = std::min(end, mapping.end);
-		}
-		for (auto* page: pages) {
-			page->current_protection = READ_WRITE_PROTECTION;
-		}
-#endif
+	static void InitializeProtection(std::span<PageState*> pages) {
 		for (auto* page: pages) {
 			page->original_protection = READ_WRITE_PROTECTION;
+			page->current_protection  = READ_WRITE_PROTECTION;
 		}
 	}
 
-	static bool AllowsAccess([[maybe_unused]] const PageState& page, uint64_t vaddr,
+	static bool AllowsAccess(const PageState& page, [[maybe_unused]] uint64_t vaddr,
 	                         PageFaultAccess access) noexcept {
-#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
-		MEMORY_BASIC_INFORMATION info {};
-		if (VirtualQuery(reinterpret_cast<const void*>(static_cast<uintptr_t>(vaddr)), &info,
-		                 sizeof(info)) == 0 ||
-		    info.State != MEM_COMMIT) {
-			return false;
-		}
 		switch (access) {
 			case PageFaultAccess::Read:
-				return info.Protect == PAGE_READONLY || info.Protect == PAGE_READWRITE;
-			case PageFaultAccess::Write: return info.Protect == PAGE_READWRITE;
+				return page.current_protection == READ_ONLY_PROTECTION ||
+				       page.current_protection == READ_WRITE_PROTECTION;
+			case PageFaultAccess::Write: return page.current_protection == READ_WRITE_PROTECTION;
 			default: return false;
 		}
-#elif defined(__APPLE__)
-		const uint32_t protection = MachQueryPageProt(vaddr);
-		switch (access) {
-			case PageFaultAccess::Read:
-				return protection == PAGE_READONLY || protection == PAGE_READWRITE;
-			case PageFaultAccess::Write: return protection == PAGE_READWRITE;
-			default: return false;
-		}
-#else
-		const auto permitted = [](uint32_t protection, PageFaultAccess wanted) {
-			switch (wanted) {
-				case PageFaultAccess::Read:
-					return protection == READ_ONLY_PROTECTION ||
-					       protection == READ_WRITE_PROTECTION;
-				case PageFaultAccess::Write: return protection == READ_WRITE_PROTECTION;
-				default: return false;
-			}
-		};
-
-		if (!permitted(page.current_protection, access)) {
-			return false;
-		}
-		return permitted(QueryHostProtection(vaddr), access);
-#endif
 	}
 
-	static void ProtectRange(std::span<PageState*> pages, uint64_t vaddr, uint32_t protection,
-	                         std::span<const uint32_t> expected_old, bool fault_path) noexcept {
+	void ProtectRange(std::span<PageState*> pages, uint64_t vaddr, uint32_t protection,
+	                  std::span<const uint32_t> expected_old, bool fault_path) noexcept {
 		const auto size = pages.size() * PAGE_SIZE;
 		if (pages.size() != expected_old.size()) {
 			FailFast("protection range state size mismatch");
 		}
-#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
-		struct HostRange {
-			uint64_t begin = 0;
-			uint64_t end   = 0;
-		};
-		std::vector<HostRange> host_ranges;
-		const auto             end = vaddr + size;
-		for (auto address = vaddr; address < end;) {
-			MEMORY_BASIC_INFORMATION info {};
-			if (VirtualQuery(reinterpret_cast<const void*>(static_cast<uintptr_t>(address)), &info,
-			                 sizeof(info)) == 0 ||
-			    info.State != MEM_COMMIT) {
-				if (fault_path) {
-					FailFast("VirtualProtect fault transition did not match expected protection");
-				}
-				Fatal("invalid protection transition at 0x%016" PRIx64 ", state=0x%08" PRIx32
-				      ", new=0x%08" PRIx32,
-				      address, static_cast<uint32_t>(info.State), protection);
-			}
-			const auto region_end = reinterpret_cast<uint64_t>(info.BaseAddress) + info.RegionSize;
-			const auto query_end  = std::min(end, region_end);
-			if (query_end <= address) {
-				if (fault_path) {
-					FailFast("VirtualQuery returned an invalid fault transition region");
-				}
-				Fatal("VirtualQuery returned an invalid region at 0x%016" PRIx64, address);
-			}
-			const auto first_page = static_cast<size_t>((address - vaddr) / PAGE_SIZE);
-			const auto last_page =
-			    static_cast<size_t>((query_end - vaddr + PAGE_SIZE - 1) / PAGE_SIZE);
-			for (auto page = first_page; page < last_page; page++) {
-				if (info.Protect != expected_old[page]) {
-					if (fault_path) {
-						FailFast(
-						    "VirtualProtect fault transition did not match expected protection");
-					}
-					Fatal("invalid protection transition at 0x%016" PRIx64 ", actual=0x%08" PRIx32
-					      ", expected=0x%08" PRIx32 ", new=0x%08" PRIx32,
-					      vaddr + page * PAGE_SIZE, static_cast<uint32_t>(info.Protect),
-					      expected_old[page], protection);
-				}
-			}
-			const auto allocation = reinterpret_cast<uint64_t>(info.AllocationBase);
-			if (host_ranges.empty() || allocation != host_ranges.back().begin) {
-				host_ranges.push_back({allocation, query_end});
-			} else {
-				host_ranges.back().end = query_end;
-			}
-			address = query_end;
-		}
-		for (auto range: host_ranges) {
-			range.begin               = std::max(range.begin, vaddr);
-			DWORD      old_protection = 0;
-			const auto first_page     = static_cast<size_t>((range.begin - vaddr) / PAGE_SIZE);
-			if (VirtualProtect(reinterpret_cast<void*>(static_cast<uintptr_t>(range.begin)),
-			                   range.end - range.begin, protection, &old_protection) == 0 ||
-			    old_protection != expected_old[first_page]) {
-				if (fault_path) {
-					FailFast("VirtualProtect fault transition did not match expected protection");
-				}
-				Fatal("invalid protection transition at 0x%016" PRIx64 ", old=0x%08" PRIx32
-				      ", expected=0x%08" PRIx32 ", new=0x%08" PRIx32,
-				      range.begin, static_cast<uint32_t>(old_protection), expected_old[first_page],
-				      protection);
-			}
-		}
-#elif defined(__APPLE__)
-		// mprotect cannot report the previous protection, so the expected_old comparison
-		// is dropped; the tracker is the sole mutator of these pages and drives the
-		// transition from its own shadow state.
-		(void)expected_old;
-		if (mprotect(reinterpret_cast<void*>(static_cast<uintptr_t>(vaddr)), size,
-		             PageProtToPosix(protection)) != 0) {
-			if (fault_path) {
-				FailFast("mprotect fault transition failed");
-			}
-			Fatal("mprotect failed at 0x%016" PRIx64 ", new=0x%08" PRIx32, vaddr, protection);
-		}
-#else
 		for (size_t i = 0; i < pages.size(); i++) {
 			const auto actual = pages[i]->current_protection;
 			if (actual != UNKNOWN_PROTECTION && actual != expected_old[i]) {
@@ -625,22 +314,21 @@ struct PageManager::Impl {
 				      vaddr + i * PAGE_SIZE, actual, expected_old[i], protection);
 			}
 		}
-		if (::mprotect(reinterpret_cast<void*>(static_cast<uintptr_t>(vaddr)), size,
-		               ToHostProtection(protection)) != 0) {
+		if (!Libs::LibKernel::Memory::ProtectGuestHostMemory(vaddr, size,
+		                                                     ToMemoryMode(protection))) {
 			if (fault_path) {
-				FailFast("mprotect failed on the fault path");
+				FailFast("address-space fault protection transition failed");
 			}
-			Fatal("mprotect failed at 0x%016" PRIx64 ", new=0x%08" PRIx32 " (%s)", vaddr,
-			      protection, std::strerror(errno));
+			Fatal("address-space protection failed at 0x%016" PRIx64 ", new=0x%08" PRIx32, vaddr,
+			      protection);
 		}
 		for (auto* page: pages) {
 			page->current_protection = protection;
 		}
-#endif
 	}
 
-	static void Protect(PageState& page, uint64_t vaddr, uint32_t protection, uint32_t expected_old,
-	                    bool fault_path) noexcept {
+	void Protect(PageState& page, uint64_t vaddr, uint32_t protection, uint32_t expected_old,
+	             bool fault_path) noexcept {
 		PageState* pages[]    = {&page};
 		uint32_t   expected[] = {expected_old};
 		ProtectRange(pages, vaddr, protection, expected, fault_path);
@@ -680,50 +368,6 @@ bool PageManager::IsTracked(uint64_t vaddr) const noexcept {
 	return page.write_watchers != 0 || page.access_watchers != 0;
 }
 
-bool PageManager::IsMapped(uint64_t vaddr, uint64_t size) const noexcept {
-	if (vaddr == 0 || size == 0 || vaddr >= ADDRESS_SIZE || size > ADDRESS_SIZE - vaddr) {
-		return false;
-	}
-	const auto end = PageStart(vaddr + size - 1) + PAGE_SIZE;
-	for (auto page_vaddr = PageStart(vaddr); page_vaddr < end; page_vaddr += PAGE_SIZE) {
-		auto* region = m_impl->FindRegion(page_vaddr);
-		if (region == nullptr) {
-			return false;
-		}
-		auto&     page = m_impl->GetPage(*region, page_vaddr);
-		SpinGuard lock(page.lock);
-		if (page.mappings == 0) {
-			return false;
-		}
-	}
-	return true;
-}
-
-bool PageManager::HasGpuAccess(uint64_t vaddr, uint64_t size, GpuAccess access) const noexcept {
-	if (access != GpuAccess::Read && access != GpuAccess::Write && access != GpuAccess::ReadWrite) {
-		FailFast("HasGpuAccess received an invalid GPU access mode");
-	}
-	const bool need_read  = access == GpuAccess::Read || access == GpuAccess::ReadWrite;
-	const bool need_write = access == GpuAccess::Write || access == GpuAccess::ReadWrite;
-	if (vaddr == 0 || size == 0 || vaddr >= ADDRESS_SIZE || size > ADDRESS_SIZE - vaddr) {
-		return false;
-	}
-	const auto end = PageEnd(vaddr, size);
-	for (auto addr = PageStart(vaddr); addr < end; addr += PAGE_SIZE) {
-		auto* region = m_impl->FindRegion(addr);
-		if (region == nullptr) {
-			return false;
-		}
-		auto&     page = m_impl->GetPage(*region, addr);
-		SpinGuard lock(page.lock);
-		if ((need_read && page.gpu_read_mappings == 0) ||
-		    (need_write && page.gpu_write_mappings == 0)) {
-			return false;
-		}
-	}
-	return true;
-}
-
 void PageManager::UpdatePageWatchers(bool track, uint64_t vaddr, uint64_t size,
                                      PageWatchMode mode) {
 	if (mode != PageWatchMode::Write && mode != PageWatchMode::ReadWrite) {
@@ -754,9 +398,6 @@ void PageManager::UpdatePageWatchers(bool track, uint64_t vaddr, uint64_t size,
 			if (page.resolving && track) {
 				FailFast("new page watcher raced active fault resolution");
 			}
-			if (page.mappings == 0) {
-				Fatal("watching unmapped page 0x%016" PRIx64, address);
-			}
 			auto& watchers =
 			    (mode == PageWatchMode::ReadWrite ? page.access_watchers : page.write_watchers);
 			if (track) {
@@ -784,8 +425,7 @@ void PageManager::UpdatePageWatchers(bool track, uint64_t vaddr, uint64_t size,
 					last++;
 				}
 				if (first != last) {
-					Impl::ValidateInitialProtection(std::span {pages}.subspan(first, last - first),
-					                                chunk_begin + first * PAGE_SIZE);
+					Impl::InitializeProtection(std::span {pages}.subspan(first, last - first));
 				}
 				first = last;
 			}
@@ -831,9 +471,9 @@ void PageManager::UpdatePageWatchers(bool track, uint64_t vaddr, uint64_t size,
 					last = current + 1;
 				}
 			}
-			Impl::ProtectRange(std::span {pages}.subspan(first, last - first),
-			                   chunk_begin + first * PAGE_SIZE, protection,
-			                   std::span {old_protections}.subspan(first, last - first), false);
+			m_impl->ProtectRange(std::span {pages}.subspan(first, last - first),
+			                     chunk_begin + first * PAGE_SIZE, protection,
+			                     std::span {old_protections}.subspan(first, last - first), false);
 			first = current;
 		}
 
@@ -860,70 +500,9 @@ void PageManager::UpdatePageWatchers(bool track, uint64_t vaddr, uint64_t size,
 	}
 }
 
-void PageManager::OnGpuMap(uint64_t vaddr, uint64_t size, GpuAccess access) {
-	if (g_in_fault_resolution) {
-		FailFast("GPU mapping changed during fault resolution");
-	}
-	if (access != GpuAccess::Read && access != GpuAccess::Write && access != GpuAccess::ReadWrite) {
-		FailFast("GPU map received an invalid access mode");
-	}
-	const bool gpu_read  = access == GpuAccess::Read || access == GpuAccess::ReadWrite;
-	const bool gpu_write = access == GpuAccess::Write || access == GpuAccess::ReadWrite;
-	const auto end       = PageEnd(vaddr, size);
-	for (auto addr = PageStart(vaddr); addr < end; addr += PAGE_SIZE) {
-		auto&     page = m_impl->GetPage(*m_impl->GetOrCreateRegion(addr), addr);
-		SpinGuard lock(page.lock);
-		if (page.resolving || page.mappings == std::numeric_limits<uint32_t>::max() ||
-		    (gpu_read && page.gpu_read_mappings == std::numeric_limits<uint32_t>::max()) ||
-		    (gpu_write && page.gpu_write_mappings == std::numeric_limits<uint32_t>::max())) {
-			Fatal("invalid map state at 0x%016" PRIx64, addr);
-		}
-		page.mappings++;
-		page.gpu_read_mappings += gpu_read ? 1u : 0u;
-		page.gpu_write_mappings += gpu_write ? 1u : 0u;
-#if defined(__linux__)
-		// New guest mappings start read/write.
-		if (page.current_protection == UNKNOWN_PROTECTION) {
-			page.current_protection = READ_WRITE_PROTECTION;
-		}
-#endif
-	}
-}
+void PageManager::OnGpuMap(uint64_t, uint64_t) {}
 
-void PageManager::OnGpuUnmap(uint64_t vaddr, uint64_t size, GpuAccess access) {
-	if (g_in_fault_resolution) {
-		FailFast("GPU unmapping changed during fault resolution");
-	}
-	if (access != GpuAccess::Read && access != GpuAccess::Write && access != GpuAccess::ReadWrite) {
-		FailFast("GPU unmap received an invalid access mode");
-	}
-	const bool gpu_read  = access == GpuAccess::Read || access == GpuAccess::ReadWrite;
-	const bool gpu_write = access == GpuAccess::Write || access == GpuAccess::ReadWrite;
-	const auto end       = PageEnd(vaddr, size);
-	for (auto page_vaddr = PageStart(vaddr); page_vaddr < end; page_vaddr += PAGE_SIZE) {
-		auto* region = m_impl->FindRegion(page_vaddr);
-		if (region == nullptr) {
-			Fatal("unmapping unknown page 0x%016" PRIx64, page_vaddr);
-		}
-		auto&     page = m_impl->GetPage(*region, page_vaddr);
-		SpinGuard lock(page.lock);
-		if (page.resolving || page.mappings == 0 || (gpu_read && page.gpu_read_mappings == 0) ||
-		    (gpu_write && page.gpu_write_mappings == 0) ||
-		    (page.mappings == 1 && (page.write_watchers != 0 || page.access_watchers != 0))) {
-			Fatal("invalid unmap state at 0x%016" PRIx64, page_vaddr);
-		}
-		page.mappings--;
-		page.gpu_read_mappings -= gpu_read ? 1u : 0u;
-		page.gpu_write_mappings -= gpu_write ? 1u : 0u;
-		if (page.mappings == 0) {
-			if (page.gpu_read_mappings != 0 || page.gpu_write_mappings != 0) {
-				FailFast("GPU unmap left nonzero GPU mapping counts");
-			}
-			page.late_read_pending  = false;
-			page.late_write_pending = false;
-		}
-	}
-}
+void PageManager::OnGpuUnmap(uint64_t, uint64_t) {}
 
 PageManager::BackingWrite::BackingWrite(PageManager& manager, uint64_t vaddr,
                                         uint64_t size) noexcept
@@ -979,8 +558,7 @@ void PageManager::BeginBackingWrite(uint64_t vaddr, uint64_t size) noexcept {
 		}
 		auto&     page = m_impl->GetPage(*region, address);
 		SpinGuard lock(page.lock);
-		if (page.mappings == 0 || page.resolving || page.backing_writer != 0 ||
-		    page.access_watchers == 0) {
+		if (page.resolving || page.backing_writer != 0 || page.access_watchers == 0) {
 			Fatal("backing write races page resolution at 0x%016" PRIx64, address);
 		}
 		page.resolving            = true;
@@ -1008,7 +586,7 @@ void PageManager::EndBackingWrite(uint64_t vaddr, uint64_t size) noexcept {
 		const auto old_protection = NO_ACCESS_PROTECTION;
 		const auto new_protection = Impl::WatcherProtection(page);
 		if (new_protection != old_protection) {
-			Impl::Protect(page, address, new_protection, old_protection, false);
+			m_impl->Protect(page, address, new_protection, old_protection, false);
 		}
 		Impl::PublishDelayedFaults(page, old_protection, new_protection);
 		if (page.write_watchers == 0 && page.access_watchers == 0) {
@@ -1109,7 +687,8 @@ bool PageManager::HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noex
 				page.write_watchers = 0;
 			}
 			const auto restored_protection = Impl::WatcherProtection(page);
-			Impl::Protect(page, PageStart(fault_vaddr), restored_protection, old_protection, true);
+			m_impl->Protect(page, PageStart(fault_vaddr), restored_protection, old_protection,
+			                true);
 			if (page.write_watchers == 0) {
 				page.original_protection = 0;
 			}

src/graphics/host_gpu/pageManager.h

@@ -13,11 +13,9 @@ namespace Libs::Graphics {
 enum class PageFaultAccess { Read, Write, Execute, Unknown };
 enum class PageFaultPhase { Invalidate, Complete, Release };
 enum class PageWatchMode { Write, ReadWrite };
-enum class GpuAccess { Read, Write, ReadWrite };
 
 using PageFaultHandler = bool (*)(void* context, PageFaultAccess access, uint64_t vaddr,
                                   uint64_t size, PageFaultPhase phase) noexcept;
-
 class PageManager final {
 public:
 	class BackingWrite final {
@@ -40,13 +38,11 @@ public:
 
 	[[nodiscard]] uint64_t GetPageSize() const;
 	[[nodiscard]] bool     IsTracked(uint64_t vaddr) const noexcept;
-	[[nodiscard]] bool     IsMapped(uint64_t vaddr, uint64_t size) const noexcept;
-	[[nodiscard]] bool HasGpuAccess(uint64_t vaddr, uint64_t size, GpuAccess access) const noexcept;
 
 	void UpdatePageWatchers(bool track, uint64_t vaddr, uint64_t size,
 	                        PageWatchMode mode = PageWatchMode::Write);
-	void OnGpuMap(uint64_t vaddr, uint64_t size, GpuAccess access = GpuAccess::ReadWrite);
-	void OnGpuUnmap(uint64_t vaddr, uint64_t size, GpuAccess access = GpuAccess::ReadWrite);
+	void OnGpuMap(uint64_t vaddr, uint64_t size);
+	void OnGpuUnmap(uint64_t vaddr, uint64_t size);
 
 	[[nodiscard]] bool HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noexcept;
 	[[nodiscard]] std::vector<std::unique_ptr<BackingWrite>>

src/graphics/host_gpu/renderer/cache/bufferCache.cpp

@@ -394,6 +394,7 @@ void BufferCache::InvalidateMemory(uint64_t vaddr, uint64_t size) {
 }
 
 void BufferCache::ReadMemory(uint64_t vaddr, uint64_t size) {
+	(void)SynchronizeBacking(vaddr, size);
 	std::vector<DownloadCopy> copies;
 	{
 		FaultSafeCacheLock lock(this, m_mutex);
@@ -426,7 +427,7 @@ void BufferCache::ReadMemory(uint64_t vaddr, uint64_t size) {
 		    });
 	}
 	if (copies.empty()) {
-		EXIT("BufferCache: GPU-owned invalidation has no dirty byte ranges\n");
+		return;
 	}
 	auto downloads = RecordDownloads(copies);
 	m_scheduler.FinishCurrent();
@@ -713,7 +714,6 @@ BufferBinding BufferCache::ObtainBuffer(CommandBuffer& command, uint64_t vaddr,
 	if (command.IsInvalid() || command.IsExecute()) {
 		EXIT("BufferCache: buffer request requires a recording command buffer\n");
 	}
-	ValidateGpuAccess(vaddr, size, is_read, is_written);
 	std::lock_guard transaction(m_resource_mutex);
 	(void)SynchronizeBacking(vaddr, size);
 
@@ -998,7 +998,6 @@ void BufferCache::FillBuffer(uint64_t vaddr, uint64_t size, uint32_t value, bool
 	if (vaddr == 0) {
 		EXIT("BufferCache: invalid fill memory address\n");
 	}
-	ValidateGpuAccess(vaddr, size, false, true);
 	(void)m_texture_cache.ClearMeta(vaddr);
 	{
 		std::lock_guard transaction(m_resource_mutex);
@@ -1040,12 +1039,6 @@ void BufferCache::CopyBuffer(uint64_t dst_vaddr, uint64_t src_vaddr, uint64_t si
 	    (src_gds && (src_vaddr > m_gds_buffer.Size() || size > m_gds_buffer.Size() - src_vaddr))) {
 		EXIT("BufferCache: invalid or overlapping copy range\n");
 	}
-	if (src_memory) {
-		ValidateGpuAccess(src_vaddr, size, true, false);
-	}
-	if (dst_memory) {
-		ValidateGpuAccess(dst_vaddr, size, false, true);
-	}
 	if (src_memory || dst_memory) {
 		std::lock_guard transaction(m_resource_mutex);
 		if (src_memory) {
@@ -1202,19 +1195,6 @@ void BufferCache::PublishImageBuffer(uint64_t vaddr, uint64_t size) {
 	owner->second->tick_accessed_last = m_gc_tick;
 }
 
-void BufferCache::ValidateGpuAccess(uint64_t vaddr, uint64_t size, bool is_read,
-                                    bool is_written) const {
-	if ((!is_read && !is_written) || vaddr == 0 || size == 0 || size > UINT64_MAX - vaddr) {
-		EXIT("BufferCache: invalid GPU access request\n");
-	}
-	if (is_read && !m_page_manager.HasGpuAccess(vaddr, size, GpuAccess::Read)) {
-		EXIT("BufferCache: GPU-read access denied\n");
-	}
-	if (is_written && !m_page_manager.HasGpuAccess(vaddr, size, GpuAccess::Write)) {
-		EXIT("BufferCache: GPU-write access denied\n");
-	}
-}
-
 void BufferCache::RunGarbageCollector() {
 	std::lock_guard transaction(m_resource_mutex);
 	const auto      tick = m_gc_tick++;

src/graphics/host_gpu/renderer/cache/bufferCache.h

@@ -50,6 +50,7 @@ public:
 	[[nodiscard]] bool InvalidateMemory(PageFaultAccess access, uint64_t vaddr, uint64_t size,
 	                                    PageFaultPhase phase) noexcept;
 	void               InvalidateMemory(uint64_t vaddr, uint64_t size);
+	void               ReadMemory(uint64_t vaddr, uint64_t size);
 	void               UnmapMemory(uint64_t vaddr, uint64_t size);
 	[[nodiscard]] BufferBinding ObtainBuffer(CommandBuffer& command, uint64_t vaddr, uint64_t size,
 	                                         bool is_written = false, bool is_read = true,
@@ -76,8 +77,7 @@ public:
 	void               CompleteBackingPublication(uint64_t vaddr, uint64_t size, uint64_t tick);
 	[[nodiscard]] bool SynchronizeBacking(uint64_t vaddr, uint64_t size);
 	void               PublishImageBuffer(uint64_t vaddr, uint64_t size);
-	void ValidateGpuAccess(uint64_t vaddr, uint64_t size, bool is_read, bool is_written) const;
-	void RunGarbageCollector();
+	void               RunGarbageCollector();
 
 private:
 	friend struct BufferCacheTestAccess;
@@ -111,7 +111,6 @@ private:
 	void QueueGarbageDownload(std::span<const DownloadCopy> copies, RetiredBuffer retire);
 	void RefreshInvalidatedRanges(CommandBuffer& command, CachedBuffer& cached, uint64_t vaddr,
 	                              uint64_t size, bool upload);
-	void ReadMemory(uint64_t vaddr, uint64_t size);
 	void DiscardGpuDirtyBytesLocked(uint64_t vaddr, uint64_t size, const char* operation);
 	void WriteHostMemory(uint64_t vaddr, std::span<const uint8_t> data);
 

src/graphics/host_gpu/renderer/cache/gpuResourceManager.cpp

@@ -4,7 +4,6 @@
 #include "graphics/guest_gpu/command_processor/commandProcessor.h"
 #include "graphics/guest_gpu/graphicsRun.h"
 #include "graphics/host_gpu/renderer/commandScheduler.h"
-
 namespace Libs::Graphics {
 
 GpuResourceManager::GpuResourceManager(GraphicContext& graphics, CommandScheduler& scheduler)
@@ -36,7 +35,8 @@ bool GpuResourceManager::InvalidateMemory(PageFaultAccess access, uint64_t vaddr
 }
 
 bool GpuResourceManager::HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noexcept {
-	if (!m_page_manager.IsMapped(fault_vaddr, 1)) {
+	constexpr uint64_t fault_size = 8;
+	if (!IsMapped(fault_vaddr, fault_size)) {
 		return false;
 	}
 	if (CommandScheduler::InDeferredOperation()) {
@@ -47,10 +47,15 @@ bool GpuResourceManager::HandleFault(PageFaultAccess access, uint64_t fault_vadd
 	bool       handled = false;
 	const auto resolve = [this, access, fault_vaddr, &handled](CommandProcessor& cp) {
 		cp.BeginReadbackTransaction();
-		(void)m_buffer_cache.SynchronizeBacking(fault_vaddr, 1);
 		{
 			ResourceMutex::FaultScope fault(m_resource_mutex);
-			handled = m_page_manager.HandleFault(access, fault_vaddr);
+			if (access == PageFaultAccess::Write) {
+				m_buffer_cache.InvalidateMemory(fault_vaddr, fault_size);
+				m_texture_cache.InvalidateMemory(fault_vaddr, fault_size);
+			} else {
+				m_buffer_cache.ReadMemory(fault_vaddr, fault_size);
+			}
+			handled = true;
 		}
 		cp.EndReadbackTransaction();
 	};
@@ -109,22 +114,19 @@ bool GpuResourceManager::IsMapped(uint64_t vaddr, uint64_t size) const noexcept
 	return m_mapped_ranges.Contains(vaddr, size);
 }
 
-void GpuResourceManager::MapMemory(uint64_t vaddr, uint64_t size, GpuAccess access) {
+void GpuResourceManager::MapMemory(uint64_t vaddr, uint64_t size) {
 	{
 		std::lock_guard lock(m_mapped_ranges_mutex);
 		m_mapped_ranges.Add(vaddr, size);
 	}
-	m_page_manager.OnGpuMap(vaddr, size, access);
+	m_page_manager.OnGpuMap(vaddr, size);
 }
 
-void GpuResourceManager::UnmapMemory(uint64_t vaddr, uint64_t size, GpuAccess access) {
-	if (!IsMapped(vaddr, size)) {
-		EXIT("cannot unmap an unmapped GPU resource range\n");
-	}
-	const auto unmap = [this, vaddr, size, access] {
-		m_texture_cache.UnmapMemory(vaddr, size);
+void GpuResourceManager::UnmapMemory(uint64_t vaddr, uint64_t size) {
+	const auto unmap = [this, vaddr, size] {
 		m_buffer_cache.UnmapMemory(vaddr, size);
-		m_page_manager.OnGpuUnmap(vaddr, size, access);
+		m_texture_cache.UnmapMemory(vaddr, size);
+		m_page_manager.OnGpuUnmap(vaddr, size);
 		std::lock_guard lock(m_mapped_ranges_mutex);
 		m_mapped_ranges.Subtract(vaddr, size);
 	};

src/graphics/host_gpu/renderer/cache/gpuResourceManager.h

@@ -29,8 +29,8 @@ public:
 	[[nodiscard]] bool HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noexcept;
 	[[nodiscard]] bool InvalidateMemory(uint64_t vaddr, uint64_t size);
 	[[nodiscard]] bool IsMapped(uint64_t vaddr, uint64_t size) const noexcept;
-	void               MapMemory(uint64_t vaddr, uint64_t size, GpuAccess access);
-	void               UnmapMemory(uint64_t vaddr, uint64_t size, GpuAccess access);
+	void               MapMemory(uint64_t vaddr, uint64_t size);
+	void               UnmapMemory(uint64_t vaddr, uint64_t size);
 	void               RunGarbageCollector();
 
 private:

src/graphics/host_gpu/renderer/cache/textureCache.cpp

@@ -88,15 +88,34 @@ TextureCache::~TextureCache() {
 
 bool TextureCache::SameBacking(const ImageInfo& cached, const ImageInfo& requested,
                                bool exact_format) {
-	const bool unit_extent =
-	    requested.extent.width == 1 && requested.extent.height == 1 && requested.extent.depth == 1;
-	return cached.data == requested.data && cached.extent == requested.extent &&
-	       cached.samples == requested.samples &&
-	       cached.bytes_per_block == requested.bytes_per_block &&
-	       (cached.type == requested.type || unit_extent) &&
-	       (exact_format
-	            ? cached.pixel_format == requested.pixel_format
-	            : ImageViewOps::FormatsCompatible(cached.pixel_format, requested.pixel_format));
+	if (cached.data.address != requested.data.address) {
+		return false;
+	}
+	if (cached.data.size != requested.data.size) {
+		return false;
+	}
+	if (cached.extent != requested.extent) {
+		return false;
+	}
+	if (cached.samples != requested.samples) {
+		return false;
+	}
+	if (cached.bytes_per_block != requested.bytes_per_block) {
+		return false;
+	}
+	if (cached.tile_mode != requested.tile_mode) {
+		return false;
+	}
+	if (!ImageViewOps::FormatsCompatible(cached.pixel_format, requested.pixel_format)) {
+		return false;
+	}
+	if (cached.type != requested.type && requested.extent != vk::Extent3D {1, 1, 1}) {
+		return false;
+	}
+	if (exact_format && cached.pixel_format != requested.pixel_format) {
+		return false;
+	}
+	return true;
 }
 
 TextureCache::BindingType TextureCache::UploadBinding(const Image& image) {
@@ -735,6 +754,12 @@ TextureCache::OverlapResult TextureCache::ResolveOverlap(const ImageInfo& reques
 		    (requested.IsVolume() || cached.info.IsVolume())) {
 			return {ExpandImage(requested, cached_id)};
 		}
+		if (requested.tile_mode != cached.info.tile_mode) {
+			if (safe_to_delete) {
+				DeleteImages(std::array {cached_id}, cached_id);
+			}
+			return {merged_id};
+		}
 		if (requested.pixel_format != cached.info.pixel_format ||
 		    requested.data.size <= cached.info.data.size) {
 			const auto result_id = merged_id ? merged_id : cached_id;
@@ -747,12 +772,6 @@ TextureCache::OverlapResult TextureCache::ResolveOverlap(const ImageInfo& reques
 		if (requested.type == cached.info.type && requested.resources > cached.info.resources) {
 			return {ExpandImage(requested, cached_id)};
 		}
-		if (requested.tile_mode != cached.info.tile_mode) {
-			if (safe_to_delete) {
-				DeleteImages(std::array {cached_id}, cached_id);
-			}
-			return {merged_id};
-		}
 		EXIT("TextureCache: unresolvable equal-address image overlap, address=0x%016" PRIx64
 		     " requested=%ux%u "
 		     "cached=%ux%u requested_size=0x%016" PRIx64 " cached_size=0x%016" PRIx64
@@ -1122,7 +1141,7 @@ ImageId TextureCache::FindImage(ImageDesc& desc, bool exact_format) {
 
 		for (const auto id: candidates) {
 			const auto owner = ResolveOwner(id);
-			if (owner == nullptr || owner->info.data != desc.info.data) {
+			if (owner == nullptr) {
 				continue;
 			}
 			if (SameBacking(owner->info, desc.info, exact_format)) {
@@ -1358,7 +1377,6 @@ bool TextureCache::ClearImageFromBuffer(CommandBuffer& command, uint64_t address
 	if (command.IsInvalid() || !GuestRange {address, size}.Valid()) {
 		EXIT("TextureCache: invalid image clear\n");
 	}
-	m_buffer_cache.ValidateGpuAccess(address, size, false, true);
 	std::lock_guard      transaction(m_resource_mutex);
 	CacheLock            lock(*this, m_lock);
 	ImageId              selected {};

src/graphics/host_gpu/renderer/image/imageView.h

@@ -103,10 +103,7 @@ IsSupportedSampledDepthUintResource(const ShaderRecompiler::IR::ImageResource& r
 
 inline void ValidateStorageColorView(vk::Format image_format, vk::Format view_format,
                                      uint32_t swizzle) noexcept {
-	const auto srgb_view = SrgbStorageViewFormat(image_format);
-	const bool srgb_storage_view =
-	    srgb_view != vk::Format::eUndefined && view_format == srgb_view;
-	if ((image_format != view_format && !srgb_storage_view) ||
+	if (!ImageViewOps::FormatsCompatible(image_format, view_format) ||
 	    !IsValidImageSwizzle(swizzle)) {
 		UnsupportedColorView("storage", image_format, view_format, swizzle);
 	}
@@ -122,7 +119,10 @@ IsSupportedStorageImageResource(const ShaderRecompiler::IR::ImageResource& resou
 	        resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim3D ||
 	        resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2DArray) &&
 	       resource.mip_mode == ShaderRecompiler::IR::ImageMipMode::None && resource.written &&
-	       !resource.atomic && !resource.depth_compare;
+	       (!resource.atomic ||
+	        (resource.kind == ShaderRecompiler::IR::ResourceKind::StorageImageUint &&
+	         resource.read)) &&
+	       !resource.depth_compare;
 }
 
 inline void

src/graphics/host_gpu/renderer/pipeline/descriptors.cpp

@@ -14,13 +14,13 @@
 #include "graphics/guest_gpu/tile.h"
 #include "graphics/host_gpu/graphicContext.h"
 #include "graphics/host_gpu/hostMemory.h"
-#include "graphics/host_gpu/renderer/image/textureCommon.h"
 #include "graphics/host_gpu/renderer/debug.h"
-#include "graphics/host_gpu/renderer/pipeline/descriptorCache.h"
 #include "graphics/host_gpu/renderer/image/imageView.h"
+#include "graphics/host_gpu/renderer/image/textureCommon.h"
+#include "graphics/host_gpu/renderer/pipeline/descriptorCache.h"
+#include "graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.h"
 #include "graphics/host_gpu/renderer/render.h"
 #include "graphics/host_gpu/renderer/renderContext.h"
-#include "graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.h"
 #include "graphics/host_gpu/vma.h"
 #include "graphics/host_gpu/vulkanCommon.h"
 #include "graphics/shader/recompiler/ir/BindingLayout.h"
@@ -239,11 +239,11 @@ bool IsSupportedDepthTextureEncoding(const ShaderTextureResource& descriptor, co
 	const uint32_t     field3_expected =
 	    (descriptor.Type() << 28u) | field3_common | descriptor.DstSelXYZW();
 	const uint32_t field4_expected = descriptor.Depth() | (descriptor.BaseArray5() << 16u);
-	const bool common = (descriptor.fields[1] & field1_reserved_mask) == 0 &&
-	                    (descriptor.fields[2] & field2_reserved_mask) == 0 &&
-	                    descriptor.fields[3] == field3_expected &&
-	                    descriptor.fields[4] == field4_expected &&
-	                    descriptor.fields[5] == field5_expected;
+	const bool     common          = (descriptor.fields[1] & field1_reserved_mask) == 0 &&
+	                                 (descriptor.fields[2] & field2_reserved_mask) == 0 &&
+	                                 descriptor.fields[3] == field3_expected &&
+	                                 descriptor.fields[4] == field4_expected &&
+	                                 descriptor.fields[5] == field5_expected;
 	if (!common || (descriptor.fields[6] == 0 && descriptor.fields[7] != 0)) {
 		return false;
 	}
@@ -318,8 +318,8 @@ static bool IsSupportedStorageTextureDescriptor(const ShaderRecompiler::IR::Imag
 	const bool valid_2d_slice =
 	    (is_color_2d && descriptor.Depth() == 0 && descriptor.BaseArray5() == 0) ||
 	    (is_color_2d_array && descriptor.BaseArray5() <= descriptor.Depth());
-	const bool is_2d = resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2D &&
-	                   valid_2d_slice;
+	const bool is_2d =
+	    resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2D && valid_2d_slice;
 	const bool is_2d_array =
 	    resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2DArray &&
 	    is_color_2d_array && descriptor.BaseArray5() <= descriptor.Depth();
@@ -340,13 +340,13 @@ static bool IsSupportedStorageTextureDescriptor(const ShaderRecompiler::IR::Imag
 	const bool supported_tile = tile == Prospero::GpuEnumValue(Prospero::TileMode::kLinear) ||
 	                            tile == Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget) ||
 	                            supported_depth_tile || supported_standard_tile;
+	const auto swizzle        = descriptor.DstSelXYZW();
 	const bool supported_swizzle =
-	    IsValidImageSwizzle(descriptor.DstSelXYZW()) &&
-	    (descriptor.DstSelXYZW() == DstSel(4, 5, 6, 7) || !resource.read);
+	    IsValidImageSwizzle(swizzle) &&
+	    (swizzle == DstSel(4, 5, 6, 7) || !resource.read || resource.atomic);
 	const bool supported_mip_view = descriptor.BaseLevel() == 0 || is_1d || is_2d;
 	return (is_1d || is_1d_array || is_2d || is_2d_array || is_3d) && supported_tile &&
-	       supported_mip_view &&
-	       descriptor.BaseLevel() == descriptor.LastLevel() &&
+	       supported_mip_view && descriptor.BaseLevel() == descriptor.LastLevel() &&
 	       descriptor.LastLevel() <= descriptor.MaxMip() && descriptor.MinLod() == 0 &&
 	       supported_swizzle && descriptor.BCSwizzle() == 0 && !descriptor.MsaaDepth();
 }
@@ -377,8 +377,10 @@ void ValidateStorageTexture(const ShaderRecompiler::IR::ImageResource& resource,
 	const bool encoding_ok   = IsSupportedStorageTextureEncoding(descriptor);
 	const bool uint_resource =
 	    resource.kind == ShaderRecompiler::IR::ResourceKind::StorageImageUint;
-	const bool format_ok = Prospero::IsSupportedTextureFormat(format) &&
-	                       uint_resource == Prospero::IsUintTextureFormat(format);
+	const bool format_ok =
+	    Prospero::IsSupportedTextureFormat(format) &&
+	    uint_resource == Prospero::IsUintTextureFormat(format) &&
+	    (!resource.atomic || format == Prospero::GpuEnumValue(Prospero::BufferFormat::k32UInt));
 	if (resource_ok && descriptor_ok && encoding_ok && format_ok && size != 0) {
 		return;
 	}
@@ -614,16 +616,14 @@ RenderExecutor::ResolveTexture(const ShaderRecompiler::IR::ImageResource&   reso
 	                     (address & (static_cast<uint64_t>(size.align) - 1u)) != 0);
 	if (storage) {
 		ValidateStorageTexture(resource, descriptor, size.size);
-		m_context.GetBufferCache().ValidateGpuAccess(address, size.size, resource.read,
-		                                             resource.written);
 	}
 
-	const auto              pixel_format = TextureGetFormat(format);
+	const auto              pixel_format        = TextureGetFormat(format);
 	const auto              storage_view_format = SrgbStorageViewFormat(pixel_format);
-	const auto              view_format =
-	    storage && storage_view_format != vk::Format::eUndefined ? storage_view_format
-	                                                            : pixel_format;
-	const auto              block_bytes  = Prospero::BlockCompressedBytesPerBlock(format);
+	const auto              view_format = storage && storage_view_format != vk::Format::eUndefined
+	                                          ? storage_view_format
+	                                          : pixel_format;
+	const auto              block_bytes = Prospero::BlockCompressedBytesPerBlock(format);
 	TextureCache::ImageDesc desc {};
 	desc.info.data         = {address, size.size};
 	desc.info.pixel_format = pixel_format;

src/graphics/host_gpu/renderer/renderDraw.cpp

@@ -18,15 +18,16 @@
 #include "graphics/host_gpu/renderer/depthRenderTarget.h"
 #include "graphics/host_gpu/renderer/pipeline/descriptorCache.h"
 #include "graphics/host_gpu/renderer/pipeline/pipelineCache.h"
-#include "graphics/host_gpu/renderer/render.h"
-#include "graphics/host_gpu/renderer/renderContext.h"
 #include "graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.h"
 #include "graphics/host_gpu/renderer/pipeline/shaderSubgroup.h"
+#include "graphics/host_gpu/renderer/render.h"
+#include "graphics/host_gpu/renderer/renderContext.h"
 #include "graphics/host_gpu/vulkanCommon.h"
 #include "graphics/shader/recompiler/ir/ResourceMaterialization.h"
 #include "graphics/shader/recompiler/ir/ShaderIR.h"
 #include "graphics/shader/shader.h"
 #include "kernel/eventQueue.h"
+#include "kernel/memory.h"
 #include "kernel/pthread.h"
 #include "libs/errno.h"
 
@@ -221,8 +222,7 @@ static void LogDrawTargetState(const char* draw_name, const RenderColorInfo& col
 	LogMrtState(draw_name, buffer, ps_input_info);
 }
 
-static void LogDrawInputState(const RenderCommandBuffer&   buffer,
-                               const RenderColorInfo&       color,
+static void LogDrawInputState(const RenderCommandBuffer& buffer, const RenderColorInfo& color,
                               const ShaderVertexInputInfo& vs_input_info,
                               uint32_t index_type_and_size, uint32_t index_count,
                               const void* index_addr) {
@@ -499,9 +499,9 @@ struct DrawCallInfo {
 };
 
 RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderColorInfo* colors,
-                                             uint32_t color_count, RenderDepthInfo& depth) {
+                                                 uint32_t color_count, RenderDepthInfo& depth) {
 	EXIT_IF(colors == nullptr || color_count > RENDER_COLOR_ATTACHMENTS_MAX);
-	auto& cache = m_context.GetTextureCache();
+	auto&       cache = m_context.GetTextureCache();
 	RenderState state {};
 	state.width                 = std::numeric_limits<uint32_t>::max();
 	state.height                = std::numeric_limits<uint32_t>::max();
@@ -512,8 +512,7 @@ RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderCo
 		auto& target = colors[i];
 		EXIT_IF(!target.image_id);
 		const auto old_image = cache.ResolveOwner(target.image_id);
-		if (old_image == nullptr ||
-		    (!old_image->registered && !old_image->info.data.Empty()) ||
+		if (old_image == nullptr || (!old_image->registered && !old_image->info.data.Empty()) ||
 		    old_image->binding.needs_rebind) {
 			if (old_image != nullptr) {
 				old_image->binding = {};
@@ -522,7 +521,7 @@ RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderCo
 			BindRenderTarget(target.image_id);
 		}
 		target.image_view = cache.FindRenderTarget(target.image_id, target.desc);
-		auto& image = cache.GetImage(target.image_id);
+		auto& image       = cache.GetImage(target.image_id);
 		EXIT_IF(image.backing.samples != target.samples || target.image_view == nullptr);
 		if (attachment_samples == 0) {
 			attachment_samples = target.samples;
@@ -530,20 +529,19 @@ RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderCo
 			EXIT("mixed color attachment sample counts are unsupported: %u and %u\n",
 			     attachment_samples, target.samples);
 		}
-		const auto& view = target.desc.view_info;
-		const auto layout =
-		    image.binding.is_bound ? vk::ImageLayout::eGeneral
-		                           : vk::ImageLayout::eColorAttachmentOptimal;
+		const auto& view   = target.desc.view_info;
+		const auto  layout = image.binding.is_bound ? vk::ImageLayout::eGeneral
+		                                            : vk::ImageLayout::eColorAttachmentOptimal;
 		image.Transit(layout,
 		              vk::AccessFlagBits2::eColorAttachmentRead |
 		                  vk::AccessFlagBits2::eColorAttachmentWrite,
 		              ImageSubresourceRange {view.base_level, view.level_count, view.base_layer,
 		                                     view.layer_count},
 		              buffer.Handle());
-		state.width      = std::min(state.width, target.extent.width);
-		state.height     = std::min(state.height, target.extent.height);
-		state.num_layers = std::min(state.num_layers, view.layer_count);
-		auto& attachment = state.color_attachments[i];
+		state.width             = std::min(state.width, target.extent.width);
+		state.height            = std::min(state.height, target.extent.height);
+		state.num_layers        = std::min(state.num_layers, view.layer_count);
+		auto& attachment        = state.color_attachments[i];
 		attachment.image_view   = target.image_view;
 		attachment.image_layout = layout;
 		attachment.clear_value  = target.color_clear_value.uint32;
@@ -561,8 +559,7 @@ RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderCo
 		depth.depth_meta_clear_enable =
 		    depth.htile &&
 		    cache.IsMetaCleared(depth.htile_buffer_vaddr, depth.desc.view_info.base_layer);
-		depth.depth_load_clear_enable =
-		    depth.depth_clear_enable || depth.depth_meta_clear_enable;
+		depth.depth_load_clear_enable = depth.depth_clear_enable || depth.depth_meta_clear_enable;
 		if (depth.depth_meta_clear_enable &&
 		    !cache.TouchMeta(depth.htile_buffer_vaddr, depth.desc.view_info.base_layer, false)) {
 			EXIT("failed to consume HTile clear state\n");
@@ -572,12 +569,12 @@ RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderCo
 		if (attachment_samples == 0) {
 			attachment_samples = depth.samples;
 		} else if (attachment_samples != depth.samples) {
-			EXIT("mixed color/depth sample counts are unsupported: %u and %u\n",
-			     attachment_samples, depth.samples);
+			EXIT("mixed color/depth sample counts are unsupported: %u and %u\n", attachment_samples,
+			     depth.samples);
 		}
 		const auto layout = depth_attachment_layout(depth);
 		const auto writes = depth.AttachmentWriteAspects();
-		auto access = vk::AccessFlags2 {vk::AccessFlagBits2::eDepthStencilAttachmentRead};
+		auto       access = vk::AccessFlags2 {vk::AccessFlagBits2::eDepthStencilAttachmentRead};
 		if (writes) {
 			access |= vk::AccessFlagBits2::eDepthStencilAttachmentWrite;
 		}
@@ -586,21 +583,19 @@ RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderCo
 		              ImageSubresourceRange {view.base_level, view.level_count, view.base_layer,
 		                                     view.layer_count},
 		              buffer.Handle());
-		state.width      = std::min(state.width, depth.width);
-		state.height     = std::min(state.height, depth.height);
-		state.num_layers = std::min(state.num_layers, view.layer_count);
-		const auto aspects = ImageViewOps::DepthAspectMask(depth.format);
-		auto& attachment = state.depth_stencil_attachment;
+		state.width               = std::min(state.width, depth.width);
+		state.height              = std::min(state.height, depth.height);
+		state.num_layers          = std::min(state.num_layers, view.layer_count);
+		const auto aspects        = ImageViewOps::DepthAspectMask(depth.format);
+		auto&      attachment     = state.depth_stencil_attachment;
 		attachment.image_view     = depth.image_view;
 		attachment.image_layout   = layout;
 		attachment.clear_value[0] = std::bit_cast<uint32_t>(depth.depth_clear_value);
 		attachment.clear_value[1] = depth.stencil_clear_value;
-		attachment.has_depth =
-		    static_cast<bool>(aspects & vk::ImageAspectFlagBits::eDepth);
-		attachment.depth_clear = depth.depth_load_clear_enable;
-		attachment.has_stencil =
-		    static_cast<bool>(aspects & vk::ImageAspectFlagBits::eStencil);
-		attachment.stencil_clear = depth.stencil_clear_enable;
+		attachment.has_depth      = static_cast<bool>(aspects & vk::ImageAspectFlagBits::eDepth);
+		attachment.depth_clear    = depth.depth_load_clear_enable;
+		attachment.has_stencil    = static_cast<bool>(aspects & vk::ImageAspectFlagBits::eStencil);
+		attachment.stencil_clear  = depth.stencil_clear_enable;
 	}
 	if (attachment_samples == 0 ||
 	    vulkan_sample_count(attachment_samples) == vk::SampleCountFlagBits {}) {
@@ -685,6 +680,85 @@ static uint64_t VertexBufferDescriptorSize(const ShaderVertexInputBuffer& buffer
 	                           : buffer.num_records);
 }
 
+struct VertexBufferRange {
+	uint64_t      base_address  = 0;
+	uint64_t      requested_end = 0;
+	uint64_t      acquired_end  = 0;
+	BufferBinding binding;
+
+	[[nodiscard]] uint64_t RequestedSize() const { return requested_end - base_address; }
+};
+
+static std::vector<BufferBinding> AcquireVertexBuffers(RenderCommandBuffer&         buffer,
+                                                       const ShaderVertexInputInfo& vs_input_info) {
+	// Collect the non-empty guest vertex ranges.
+	std::vector<VertexBufferRange> ranges;
+	ranges.reserve(vs_input_info.buffers_num);
+	for (int i = 0; i < vs_input_info.buffers_num; i++) {
+		const auto& vertex = vs_input_info.buffers[i];
+		const auto  size   = VertexBufferDescriptorSize(vertex);
+		if (size == 0) {
+			continue;
+		}
+		if (vertex.addr == 0 || size > UINT64_MAX - vertex.addr) {
+			EXIT("invalid vertex buffer range: addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n",
+			     vertex.addr, size);
+		}
+		ranges.push_back({vertex.addr, vertex.addr + size});
+	}
+
+	std::ranges::sort(ranges, [](const VertexBufferRange& left, const VertexBufferRange& right) {
+		return left.base_address < right.base_address;
+	});
+
+	// Merge overlapping or touching ranges before acquiring host buffers.
+	std::vector<VertexBufferRange> merged_ranges;
+	merged_ranges.reserve(ranges.size());
+	for (const auto& range: ranges) {
+		if (!merged_ranges.empty() && merged_ranges.back().requested_end >= range.base_address) {
+			merged_ranges.back().requested_end =
+			    std::max(merged_ranges.back().requested_end, range.requested_end);
+			continue;
+		}
+		merged_ranges.push_back(range);
+	}
+
+	auto& cache = buffer.GetContext().GetBufferCache();
+	for (auto& range: merged_ranges) {
+		// PPSA20298
+		const auto size =
+		    Libs::LibKernel::Memory::ClampRangeSize(range.base_address, range.RequestedSize());
+		range.acquired_end = range.base_address + size;
+		range.binding      = cache.ObtainBuffer(buffer, range.base_address, size);
+	}
+
+	// Rebuild slot bindings, offsetting non-empty slots into their acquired merged range.
+	std::vector<BufferBinding> bindings;
+	bindings.reserve(vs_input_info.buffers_num);
+	for (int i = 0; i < vs_input_info.buffers_num; i++) {
+		const auto& vertex = vs_input_info.buffers[i];
+		const auto  size   = VertexBufferDescriptorSize(vertex);
+		if (size == 0) {
+			auto owner = cache.ObtainNullBuffer();
+			bindings.push_back({owner, owner->Handle(), 0});
+			continue;
+		}
+
+		const auto range = std::ranges::find_if(merged_ranges, [&](const VertexBufferRange& value) {
+			return vertex.addr >= value.base_address && vertex.addr < value.acquired_end;
+		});
+		if (range == merged_ranges.end()) {
+			EXIT("vertex buffer address is outside the acquired range: addr=0x%016" PRIx64 "\n",
+			     vertex.addr);
+		}
+
+		auto binding = range->binding;
+		binding.offset += vertex.addr - range->base_address;
+		bindings.push_back(std::move(binding));
+	}
+	return bindings;
+}
+
 static void SetDrawDebugPhase(RenderCommandBuffer& buffer, uint64_t submit_id,
                               const DrawCallInfo& draw, uint32_t phase) {
 	EXIT_IF(draw.name == nullptr);
@@ -736,9 +810,9 @@ static bool GetDrawTopology(const HW::UserConfig& ucfg, bool auto_draw, bool use
 }
 
 bool RenderExecutor::PrepareDrawRenderState(uint64_t submit_id, RenderCommandBuffer& buffer,
-                                        const DrawCallInfo& draw,
-                                        uint32_t render_target_slice_offset,
-                                        bool log_setup_phases, DrawRenderState& state) {
+                                            const DrawCallInfo& draw,
+                                            uint32_t            render_target_slice_offset,
+                                            bool log_setup_phases, DrawRenderState& state) {
 	EXIT_IF(draw.name == nullptr);
 	auto& ctx = buffer.GetRegisters();
 
@@ -823,37 +897,13 @@ static std::vector<BufferBinding> PrepareVertexBuffers(uint64_t
 	(void)submit_id;
 
 	LogDrawPhase(draw.name, "PrepareVertexBuffers");
-	std::vector<BufferBinding> bindings;
-	bindings.reserve(vs_input_info.buffers_num);
-	for (int i = 0; i < vs_input_info.buffers_num; i++) {
-		const auto& b    = vs_input_info.buffers[i];
-		const auto  size = VertexBufferDescriptorSize(b);
-		if (size == 0) {
-			auto owner = buffer.GetContext().GetBufferCache().ObtainNullBuffer();
-			bindings.push_back({owner, owner->Handle(), 0});
-		} else {
-			bindings.push_back(
-			    buffer.GetContext().GetBufferCache().ObtainBuffer(buffer, b.addr, size));
-		}
-	}
-	return bindings;
+	return AcquireVertexBuffers(buffer, vs_input_info);
 }
 
 static void RebindVertexBuffers(RenderCommandBuffer&         buffer,
                                 const ShaderVertexInputInfo& vs_input_info,
                                 std::vector<BufferBinding>&  bindings) {
-	EXIT_IF(bindings.size() != static_cast<size_t>(vs_input_info.buffers_num));
-	for (int i = 0; i < vs_input_info.buffers_num; i++) {
-		const auto& vertex = vs_input_info.buffers[i];
-		const auto  size   = VertexBufferDescriptorSize(vertex);
-		if (size == 0) {
-			auto owner  = buffer.GetContext().GetBufferCache().ObtainNullBuffer();
-			bindings[i] = {owner, owner->Handle(), 0};
-		} else {
-			bindings[i] =
-			    buffer.GetContext().GetBufferCache().ObtainBuffer(buffer, vertex.addr, size);
-		}
-	}
+	bindings = AcquireVertexBuffers(buffer, vs_input_info);
 }
 
 static PreparedIndexBuffer PrepareIndexBuffer(RenderCommandBuffer&         buffer,
@@ -1011,17 +1061,17 @@ static void EmitDrawPrimitives(const HW::UserConfig& ucfg, vk::CommandBuffer vk_
 }
 
 void RenderExecutor::ExecutePreparedDraw(uint64_t submit_id, RenderCommandBuffer& buffer,
-                                     const DrawCallInfo& draw, DrawRenderState& state,
-                                     vk::PrimitiveTopology topology, const DrawEmitInfo& emit,
-                                     const DrawIndexBufferSource& index_source,
-                                     bool log_pipeline_phase, bool set_bind_debug,
-                                     bool set_auto_debug) {
+                                         const DrawCallInfo& draw, DrawRenderState& state,
+                                         vk::PrimitiveTopology topology, const DrawEmitInfo& emit,
+                                         const DrawIndexBufferSource& index_source,
+                                         bool log_pipeline_phase, bool set_bind_debug,
+                                         bool set_auto_debug) {
 	EXIT_IF(draw.name == nullptr);
 	auto& ucfg = buffer.GetUserConfig();
 
 	LogDrawPhase(draw.name, "PrepareBindings");
-	auto bindings = PrepareGraphicsBindings(buffer, state.vs_input_info.stage,
-	                                        state.ps_input_info.stage, state.ps_active);
+	auto bindings        = PrepareGraphicsBindings(buffer, state.vs_input_info.stage,
+	                                               state.ps_input_info.stage, state.ps_active);
 	auto vertex_bindings = PrepareVertexBuffers(submit_id, buffer, draw, state.vs_input_info);
 	auto index_binding   = PrepareIndexBuffer(buffer, index_source);
 	RebindVertexBuffers(buffer, state.vs_input_info, vertex_bindings);
@@ -1094,10 +1144,10 @@ void RenderExecutor::ExecutePreparedDraw(uint64_t submit_id, RenderCommandBuffer
 }
 
 void RenderExecutor::DrawIndex(uint64_t submit_id, RenderCommandBuffer& buffer,
-                           uint32_t index_type_and_size, uint32_t index_count,
-                           const void* index_addr, uint32_t flags, uint32_t type,
-                           uint32_t instance_count, uint32_t render_target_slice_offset,
-                           int32_t vertex_offset_add, uint32_t first_instance) {
+                               uint32_t index_type_and_size, uint32_t index_count,
+                               const void* index_addr, uint32_t flags, uint32_t type,
+                               uint32_t instance_count, uint32_t render_target_slice_offset,
+                               int32_t vertex_offset_add, uint32_t first_instance) {
 	KYTY_PROFILER_FUNCTION();
 
 	EXIT_IF(buffer.IsInvalid());
@@ -1228,11 +1278,10 @@ void RenderExecutor::DrawIndex(uint64_t submit_id, RenderCommandBuffer& buffer,
 }
 
 // NOLINTNEXTLINE(readability-function-cognitive-complexity)
-void RenderExecutor::DrawAuto(uint64_t submit_id, RenderCommandBuffer& buffer,
-                              uint32_t index_count,
-                          uint32_t flags, uint32_t render_target_slice_offset,
-                          uint32_t instance_count, uint32_t first_vertex,
-                          uint32_t first_instance) {
+void RenderExecutor::DrawAuto(uint64_t submit_id, RenderCommandBuffer& buffer, uint32_t index_count,
+                              uint32_t flags, uint32_t render_target_slice_offset,
+                              uint32_t instance_count, uint32_t first_vertex,
+                              uint32_t first_instance) {
 	KYTY_PROFILER_FUNCTION();
 
 	EXIT_IF(buffer.IsInvalid());
@@ -1290,7 +1339,8 @@ void RenderExecutor::DrawAuto(uint64_t submit_id, RenderCommandBuffer& buffer,
 	                         instance_count,  first_instance};
 
 	DrawRenderState state {};
-	if (!PrepareDrawRenderState(submit_id, buffer, draw, render_target_slice_offset, false, state)) {
+	if (!PrepareDrawRenderState(submit_id, buffer, draw, render_target_slice_offset, false,
+	                            state)) {
 		ResetBindings();
 		return;
 	}
@@ -1340,7 +1390,7 @@ void RenderExecutor::DrawAuto(uint64_t submit_id, RenderCommandBuffer& buffer,
 }
 
 bool RenderExecutor::ResolveColorTargets(uint64_t submit_id, RenderCommandBuffer& buffer,
-                                     uint32_t render_target_slice_offset) {
+                                         uint32_t render_target_slice_offset) {
 	const auto& hw = buffer.GetRegisters();
 	if (hw.GetColorControl().mode != 3) {
 		return false;
@@ -1369,8 +1419,7 @@ bool RenderExecutor::ResolveColorTargets(uint64_t submit_id, RenderCommandBuffer
 	cache.MarkGpuWritten(dst.image_id);
 	auto& source      = cache.GetImage(src.image_id);
 	auto& destination = cache.GetImage(dst.image_id);
-	destination.Resolve(source,
-	                    {src.base_mip_level, 1, src.base_array_layer, 1},
+	destination.Resolve(source, {src.base_mip_level, 1, src.base_array_layer, 1},
 	                    {dst.base_mip_level, 1, dst.base_array_layer, 1});
 	return true;
 }

src/graphics/shader/recompiler/decompiler/MemoryOps.cpp

@@ -421,13 +421,13 @@ bool DecodeDs(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
 	const uint32_t data0   = (word1 >> 8u) & 0xffu;
 	const uint32_t addr    = word1 & 0xffu;
 
-	inst.pc         = pc;
-	inst.word       = word0;
-	inst.word_count = 2;
-	inst.offset     = offset0 | (offset1 << 8u);
-	inst.gds        = ((word0 >> 17u) & 1u) != 0u;
-	inst.family     = Family::DS;
-	inst.opcode_id  = opcode;
+	inst.pc          = pc;
+	inst.word        = word0;
+	inst.word_count  = 2;
+	inst.offset      = offset0 | (offset1 << 8u);
+	inst.gds         = ((word0 >> 17u) & 1u) != 0u;
+	inst.family      = Family::DS;
+	inst.opcode_id   = opcode;
 	const auto* info = LookupMemoryOpcode(DS_OPS, static_cast<uint32_t>(std::size(DS_OPS)), opcode);
 	ApplyMemoryInfo(inst, info);
 	SetRawWords(inst, code, word_index, 2);
@@ -442,11 +442,6 @@ bool DecodeDs(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
 	     inst.opcode == Opcode::DsReadAddtidB32)) {
 		SetUnsupported(inst, Family::DS, opcode, "DS swizzle/addtid is available only for LDS");
 	}
-	if (inst.gds && (inst.opcode == Opcode::DsAppend || inst.opcode == Opcode::DsConsume) &&
-	    inst.offset != 0u) {
-		SetUnsupported(inst, Family::DS, opcode,
-		               "GDS append/consume requires a zero instruction offset");
-	}
 	if (inst.opcode == Opcode::DsWriteAddtidB32 && data1 != 0u) {
 		SetUnsupported(inst, Family::DS, opcode,
 		               "DS write addtid data1 operand is not implemented");

src/graphics/shader/recompiler/emitter/SpirvEmitter.cpp

@@ -1,7 +1,7 @@
 #include "graphics/shader/recompiler/emitter/SpirvEmitter.h"
 
-#include "graphics/shader/recompiler/ir/SrtWalker.h"
 #include "graphics/shader/recompiler/emitter/spirvEmitterInternal.h"
+#include "graphics/shader/recompiler/ir/SrtWalker.h"
 
 #include <algorithm>
 #include <array>
@@ -160,8 +160,7 @@ bool ValidateInstructionContract(const IR::Instruction& inst, std::string* error
 	                            inst.dst.kind != IR::OperandKind::Null)) ||
 	    ((inst.op == IR::Opcode::DsAppend || inst.op == IR::Opcode::DsConsume) &&
 	     (!ds_kind || !ds_resource || inst.src_count != 1 ||
-	      inst.dst.kind != IR::OperandKind::Register ||
-	      (kind == IR::ResourceKind::Gds && inst.memory.offset != 0))) ||
+	      inst.dst.kind != IR::OperandKind::Register)) ||
 	    ((inst.op == IR::Opcode::DsMinF32 || inst.op == IR::Opcode::DsMaxF32) &&
 	     (!ds_kind || !ds_resource || inst.src_count != 3 ||
 	      inst.dst.kind != IR::OperandKind::Null)) ||

src/graphics/shader/recompiler/emitter/spirvEmitterControlFlow.cpp

@@ -1025,15 +1025,47 @@ void EmitDispatcherSwitch(EmitterState& state, const IR::Program& program) {
 	EmitDispatcherExit(state);
 }
 
+size_t BufferLoadGroupSize(const IR::BasicBlock& block, size_t first_index) {
+	const auto& first = block.instructions[first_index];
+	if (first.op != IR::Opcode::BufferLoadDword || first.memory.component_index != 0u ||
+	    first.memory.component_count <= 1u) {
+		return 1u;
+	}
+
+	size_t count = 1u;
+	while (first_index + count < block.instructions.size() &&
+	       count < first.memory.component_count) {
+		const auto& next = block.instructions[first_index + count];
+		if (next.op != IR::Opcode::BufferLoadDword || next.pc != first.pc ||
+		    next.memory.component_index != count ||
+		    next.memory.component_count != first.memory.component_count) {
+			break;
+		}
+		count++;
+	}
+	return count;
+}
+
+void EmitBlockInstructions(EmitterState& state, const IR::BasicBlock& block) {
+	for (size_t i = 0; i < block.instructions.size();) {
+		const auto count = BufferLoadGroupSize(block, i);
+		if (count > 1u) {
+			EmitBufferLoadDwordGroup(state, block.instructions.data() + i,
+			                         static_cast<uint32_t>(count));
+		} else {
+			EmitInstruction(state, block.instructions[i]);
+		}
+		i += count;
+	}
+}
+
 void EmitDispatcherBlocks(EmitterState& state, const IR::Program& program) {
 	for (const auto& block: program.blocks) {
 		if (block.id >= state.reachable_blocks.size() || !state.reachable_blocks[block.id]) {
 			continue;
 		}
 		state.builder.AddFunction({OpLabel, BlockLabel(state, block.id)});
-		for (const auto& inst: block.instructions) {
-			EmitInstruction(state, inst);
-		}
+		EmitBlockInstructions(state, block);
 		EmitDispatcherTerminator(state, block.terminator);
 	}
 }
@@ -1083,9 +1115,7 @@ void EmitFunction(EmitterState& state, const IR::Program& program) {
 			continue;
 		}
 		state.builder.AddFunction({OpLabel, BlockLabel(state, block.id)});
-		for (const auto& inst: block.instructions) {
-			EmitInstruction(state, inst);
-		}
+		EmitBlockInstructions(state, block);
 		EmitTerminator(state, block.terminator);
 	}
 

src/graphics/shader/recompiler/emitter/spirvEmitterImageHelpers.cpp

@@ -2,6 +2,66 @@
 #include "graphics/shader/recompiler/emitter/spirvEmitterInternal.h"
 
 namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
+namespace {
+
+uint32_t EmitCubeAxisF32(EmitterState& state, uint32_t value) {
+	const auto normalized = state.builder.AllocateId();
+	state.builder.AddFunction(
+	    {OpFSub, state.float_type, normalized, value, ConstantF32(state, 0x3f800000u)});
+	return normalized;
+}
+
+uint32_t EmitCubeLayerF32(EmitterState& state, uint32_t face_id) {
+	// Sampled RDNA2 cubemaps encode face_id as slice * 8 + face. The native
+	// 2D-array view stores six contiguous faces per slice, so remove the two
+	// reserved face IDs from every preceding slice.
+	const auto guest_layer = state.builder.AllocateId();
+	const auto slice       = state.builder.AllocateId();
+	const auto padding     = state.builder.AllocateId();
+	const auto host_layer  = state.builder.AllocateId();
+	const auto result      = state.builder.AllocateId();
+	state.builder.AddFunction({OpConvertFToU, state.uint_type, guest_layer, face_id});
+	state.builder.AddFunction(
+	    {OpShiftRightLogical, state.uint_type, slice, guest_layer, ConstantU32(state, 3)});
+	state.builder.AddFunction(
+	    {OpShiftLeftLogical, state.uint_type, padding, slice, ConstantU32(state, 1)});
+	state.builder.AddFunction({OpISub, state.uint_type, host_layer, guest_layer, padding});
+	state.builder.AddFunction({OpConvertUToF, state.float_type, result, host_layer});
+	return result;
+}
+
+uint32_t EmitImageCoordF32Impl(EmitterState& state, const IR::Instruction& inst,
+                               const IR::Operand& address, uint32_t first_component,
+                               uint32_t components) {
+	auto x = EmitImageAddressFloatLoad(state, inst, address, first_component);
+	if (components == 1u) {
+		return x;
+	}
+	auto y = inst.memory.image_address_components > first_component + 1u
+	             ? EmitImageAddressFloatLoad(state, inst, address, first_component + 1u)
+	             : EmitZeroF32(state);
+	if (inst.memory.image_cube) {
+		// RDNA2 sampled cubemap S/T coordinates are biased by +1 relative to
+		// normalized 2D-array coordinates.
+		x = EmitCubeAxisF32(state, x);
+		y = EmitCubeAxisF32(state, y);
+	}
+	const auto coord = state.builder.AllocateId();
+	if (components == 3u) {
+		auto z = inst.memory.image_address_components > first_component + 2u
+		             ? EmitImageAddressFloatLoad(state, inst, address, first_component + 2u)
+		             : EmitZeroF32(state);
+		if (inst.memory.image_cube) {
+			z = EmitCubeLayerF32(state, z);
+		}
+		state.builder.AddFunction({OpCompositeConstruct, state.vec3_float_type, coord, x, y, z});
+	} else {
+		state.builder.AddFunction({OpCompositeConstruct, state.vec2_float_type, coord, x, y});
+	}
+	return coord;
+}
+
+} // namespace
 
 bool HasImageSampleFlag(const IR::Instruction& inst, uint32_t flag) {
 	return (inst.memory.image_sample_flags & flag) != 0;
@@ -21,7 +81,7 @@ ImageSampleLayout MakeImageSampleLayout(const IR::Instruction& inst, ImageViewKi
 	}
 	if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagDerivative)) {
 		const auto components = ImageViewSpatialComponents(view);
-		layout.grad_x = cursor;
+		layout.grad_x         = cursor;
 		cursor += components;
 		layout.grad_y = cursor;
 		cursor += components;
@@ -36,24 +96,8 @@ ImageSampleLayout MakeImageSampleLayout(const IR::Instruction& inst, ImageViewKi
 
 uint32_t EmitImageCoordF32(EmitterState& state, const IR::Instruction& inst,
                            const ImageSampleLayout& layout, ImageViewKind view) {
-	const auto x          = EmitImageAddressFloatLoad(state, inst, inst.src[0], layout.coord);
-	const auto components = ImageViewCoordinateComponents(view);
-	if (components == 1u) {
-		return x;
-	}
-	const auto y = inst.memory.image_address_components > layout.coord + 1u
-	                       ? EmitImageAddressFloatLoad(state, inst, inst.src[0], layout.coord + 1u)
-	                       : EmitZeroF32(state);
-	const auto coord = state.builder.AllocateId();
-	if (components == 3u) {
-		const auto z = inst.memory.image_address_components > layout.coord + 2u
-		                   ? EmitImageAddressFloatLoad(state, inst, inst.src[0], layout.coord + 2u)
-		                   : EmitZeroF32(state);
-		state.builder.AddFunction({OpCompositeConstruct, state.vec3_float_type, coord, x, y, z});
-	} else {
-		state.builder.AddFunction({OpCompositeConstruct, state.vec2_float_type, coord, x, y});
-	}
-	return coord;
+	return EmitImageCoordF32Impl(state, inst, inst.src[0], layout.coord,
+	                             ImageViewCoordinateComponents(view));
 }
 
 uint32_t EmitImageLodF32(EmitterState& state, const IR::Instruction& inst,
@@ -95,10 +139,10 @@ uint32_t EmitImageGradientF32(EmitterState& state, const IR::Instruction& inst,
 	                   : EmitZeroF32(state);
 	const auto grad = state.builder.AllocateId();
 	if (components == 3u) {
-		const auto z = inst.memory.image_address_components > first_component + 2u
-		                   ? EmitImageAddressFloatLoad(state, inst, inst.src[0],
-		                                               first_component + 2u)
-		                   : EmitZeroF32(state);
+		const auto z =
+		    inst.memory.image_address_components > first_component + 2u
+		        ? EmitImageAddressFloatLoad(state, inst, inst.src[0], first_component + 2u)
+		        : EmitZeroF32(state);
 		state.builder.AddFunction({OpCompositeConstruct, state.vec3_float_type, grad, x, y, z});
 	} else {
 		state.builder.AddFunction({OpCompositeConstruct, state.vec2_float_type, grad, x, y});
@@ -120,8 +164,7 @@ uint32_t EmitImagePackedOffsetI32(EmitterState& state, const IR::Instruction& in
 			state.builder.AddFunction(
 			    {OpCompositeConstruct, state.vec3_int_type, ret, zero, zero, zero});
 		} else {
-			state.builder.AddFunction(
-			    {OpCompositeConstruct, state.vec2_int_type, ret, zero, zero});
+			state.builder.AddFunction({OpCompositeConstruct, state.vec2_int_type, ret, zero, zero});
 		}
 		return ret;
 	}
@@ -131,18 +174,18 @@ uint32_t EmitImagePackedOffsetI32(EmitterState& state, const IR::Instruction& in
 	const auto offset_x    = state.builder.AllocateId();
 	state.builder.AddFunction({OpBitcast, state.int_type, packed_i32, packed_bits});
 	state.builder.AddFunction({OpBitFieldSExtract, state.int_type, offset_x, packed_i32,
-	                            ConstantI32(state, 0), ConstantI32(state, 6)});
+	                           ConstantI32(state, 0), ConstantI32(state, 6)});
 	if (components == 1u) {
 		return offset_x;
 	}
 	const auto offset_y = state.builder.AllocateId();
 	const auto offset   = state.builder.AllocateId();
 	state.builder.AddFunction({OpBitFieldSExtract, state.int_type, offset_y, packed_i32,
-	                            ConstantI32(state, 8), ConstantI32(state, 6)});
+	                           ConstantI32(state, 8), ConstantI32(state, 6)});
 	if (components == 3u) {
 		const auto offset_z = state.builder.AllocateId();
 		state.builder.AddFunction({OpBitFieldSExtract, state.int_type, offset_z, packed_i32,
-		                            ConstantI32(state, 16), ConstantI32(state, 6)});
+		                           ConstantI32(state, 16), ConstantI32(state, 6)});
 		state.builder.AddFunction(
 		    {OpCompositeConstruct, state.vec3_int_type, offset, offset_x, offset_y, offset_z});
 	} else {
@@ -158,7 +201,7 @@ uint32_t EmitImageCoordU32(EmitterState& state, const IR::Instruction& inst, Ima
 	if (components == 1u) {
 		return x;
 	}
-	const auto y = inst.memory.image_address_components > 1u
+	const auto y     = inst.memory.image_address_components > 1u
 	                       ? EmitImageAddressValueLoad(state, inst, inst.src[1], 1)
 	                       : ConstantU32(state, 0);
 	const auto coord = state.builder.AllocateId();
@@ -180,7 +223,7 @@ uint32_t EmitImageLoadCoordU32(EmitterState& state, const IR::Instruction& inst,
 	if (components == 1u) {
 		return x;
 	}
-	const auto y = inst.memory.image_address_components > 1u
+	const auto y     = inst.memory.image_address_components > 1u
 	                       ? EmitImageAddressValueLoad(state, inst, inst.src[0], 1)
 	                       : ConstantU32(state, 0);
 	const auto coord = state.builder.AllocateId();
@@ -209,24 +252,8 @@ uint32_t EmitImageMipLodU32(EmitterState& state, const IR::Instruction& inst,
 
 uint32_t EmitImageQueryCoordF32(EmitterState& state, const IR::Instruction& inst,
                                 ImageViewKind view) {
-	const auto x          = EmitImageAddressFloatLoad(state, inst, inst.src[0], 0);
-	const auto components = ImageViewCoordinateComponents(view);
-	if (components == 1u) {
-		return x;
-	}
-	const auto y = inst.memory.image_address_components > 1u
-	                       ? EmitImageAddressFloatLoad(state, inst, inst.src[0], 1)
-	                       : EmitZeroF32(state);
-	const auto coord = state.builder.AllocateId();
-	if (components == 3u) {
-		const auto z = inst.memory.image_address_components > 2u
-		                   ? EmitImageAddressFloatLoad(state, inst, inst.src[0], 2)
-		                   : EmitZeroF32(state);
-		state.builder.AddFunction({OpCompositeConstruct, state.vec3_float_type, coord, x, y, z});
-	} else {
-		state.builder.AddFunction({OpCompositeConstruct, state.vec2_float_type, coord, x, y});
-	}
-	return coord;
+	// OpImageQueryLod takes only the spatial coordinates, even for arrayed images.
+	return EmitImageCoordF32Impl(state, inst, inst.src[0], 0, ImageViewSpatialComponents(view));
 }
 
 uint32_t DmaskComponentIndex(uint32_t dmask, uint32_t component) {

src/graphics/shader/recompiler/emitter/spirvEmitterInternal.h

@@ -3,11 +3,11 @@
 
 #include "common/common.h"
 #include "common/stringUtils.h"
-#include "graphics/shader/recompiler/ir/BindingLayout.h"
 #include "graphics/shader/recompiler/BufferFormat.h"
+#include "graphics/shader/recompiler/emitter/SpirvBuilder.h"
+#include "graphics/shader/recompiler/ir/BindingLayout.h"
 #include "graphics/shader/recompiler/ir/ResourceMaterialization.h"
 #include "graphics/shader/recompiler/ir/ShaderIR.h"
-#include "graphics/shader/recompiler/emitter/SpirvBuilder.h"
 
 #include <algorithm>
 #include <array>
@@ -312,117 +312,117 @@ struct EmitterState {
 	EmitterState(const IR::Program& program_, const IR::ResourceSnapshot& resources_)
 	    : program(program_), resources(resources_) {}
 
-	Builder                                builder;
-	const IR::Program&                     program;
-	const IR::ResourceSnapshot&            resources;
-	const ShaderVertexInputInfo*           vertex_input_info            = nullptr;
-	const ShaderPixelInputInfo*            pixel_input_info             = nullptr;
-	const ShaderComputeInputInfo*          compute_input_info           = nullptr;
-	ShaderType                             stage                        = ShaderType::Unknown;
-	uint32_t                               wave_size                    = 64;
-	bool                                   exact_subgroup_operations    = false;
-	bool                                   per_invocation_masks         = false;
-	uint32_t                               void_type                    = 0;
-	uint32_t                               bool_type                    = 0;
-	uint32_t                               uint_type                    = 0;
-	uint32_t                               uint_pair_type               = 0;
-	uint32_t                               int_pair_type                = 0;
-	uint32_t                               int_type                     = 0;
-	uint32_t                               float_type                   = 0;
-	uint32_t                               vec2_uint_type               = 0;
-	uint32_t                               vec3_uint_type               = 0;
-	uint32_t                               vec4_uint_type               = 0;
-	uint32_t                               vec2_int_type                = 0;
-	uint32_t                               vec3_int_type                = 0;
-	uint32_t                               vec4_int_type                = 0;
-	uint32_t                               vec2_float_type              = 0;
-	uint32_t                               vec3_float_type              = 0;
-	uint32_t                               vec4_float_type              = 0;
-	uint32_t                               ptr_func_uint                = 0;
-	uint32_t                               ptr_input_float              = 0;
-	uint32_t                               ptr_input_bool               = 0;
-	uint32_t                               ptr_input_int                = 0;
-	uint32_t                               ptr_input_uint               = 0;
-	uint32_t                               ptr_input_vec2_float         = 0;
-	uint32_t                               ptr_input_vec3_float         = 0;
-	uint32_t                               ptr_input_vec2_int           = 0;
-	uint32_t                               ptr_input_vec3_int           = 0;
-	uint32_t                               ptr_input_vec4_int           = 0;
-	uint32_t                               ptr_input_vec2_uint          = 0;
-	uint32_t                               ptr_input_vec3_uint          = 0;
-	uint32_t                               ptr_input_vec4_uint          = 0;
-	uint32_t                               ptr_input_vec4_float         = 0;
-	uint32_t                               sample_mask_array_type       = 0;
-	uint32_t                               ptr_output_int               = 0;
-	uint32_t                               ptr_output_sample_mask_array = 0;
-	uint32_t                               ptr_output_float             = 0;
-	uint32_t                               ptr_output_vec4_float        = 0;
-	uint32_t                               per_vertex_type              = 0;
-	uint32_t                               ptr_output_per_vertex        = 0;
-	uint32_t                               storage_runtime_array_type   = 0;
-	uint32_t                               storage_buffer_type          = 0;
-	uint32_t                               ptr_storage_buffer           = 0;
-	uint32_t                               ptr_storage_buffer_uint      = 0;
-	uint32_t                               storage_buffer_array_type    = 0;
-	uint32_t                               ptr_storage_buffer_array     = 0;
-	uint32_t                               storage_buffer_variable      = 0;
+	Builder                                          builder;
+	const IR::Program&                               program;
+	const IR::ResourceSnapshot&                      resources;
+	const ShaderVertexInputInfo*                     vertex_input_info  = nullptr;
+	const ShaderPixelInputInfo*                      pixel_input_info   = nullptr;
+	const ShaderComputeInputInfo*                    compute_input_info = nullptr;
+	ShaderType                                       stage              = ShaderType::Unknown;
+	uint32_t                                         wave_size          = 64;
+	bool                                             exact_subgroup_operations    = false;
+	bool                                             per_invocation_masks         = false;
+	uint32_t                                         void_type                    = 0;
+	uint32_t                                         bool_type                    = 0;
+	uint32_t                                         uint_type                    = 0;
+	uint32_t                                         uint_pair_type               = 0;
+	uint32_t                                         int_pair_type                = 0;
+	uint32_t                                         int_type                     = 0;
+	uint32_t                                         float_type                   = 0;
+	uint32_t                                         vec2_uint_type               = 0;
+	uint32_t                                         vec3_uint_type               = 0;
+	uint32_t                                         vec4_uint_type               = 0;
+	uint32_t                                         vec2_int_type                = 0;
+	uint32_t                                         vec3_int_type                = 0;
+	uint32_t                                         vec4_int_type                = 0;
+	uint32_t                                         vec2_float_type              = 0;
+	uint32_t                                         vec3_float_type              = 0;
+	uint32_t                                         vec4_float_type              = 0;
+	uint32_t                                         ptr_func_uint                = 0;
+	uint32_t                                         ptr_input_float              = 0;
+	uint32_t                                         ptr_input_bool               = 0;
+	uint32_t                                         ptr_input_int                = 0;
+	uint32_t                                         ptr_input_uint               = 0;
+	uint32_t                                         ptr_input_vec2_float         = 0;
+	uint32_t                                         ptr_input_vec3_float         = 0;
+	uint32_t                                         ptr_input_vec2_int           = 0;
+	uint32_t                                         ptr_input_vec3_int           = 0;
+	uint32_t                                         ptr_input_vec4_int           = 0;
+	uint32_t                                         ptr_input_vec2_uint          = 0;
+	uint32_t                                         ptr_input_vec3_uint          = 0;
+	uint32_t                                         ptr_input_vec4_uint          = 0;
+	uint32_t                                         ptr_input_vec4_float         = 0;
+	uint32_t                                         sample_mask_array_type       = 0;
+	uint32_t                                         ptr_output_int               = 0;
+	uint32_t                                         ptr_output_sample_mask_array = 0;
+	uint32_t                                         ptr_output_float             = 0;
+	uint32_t                                         ptr_output_vec4_float        = 0;
+	uint32_t                                         per_vertex_type              = 0;
+	uint32_t                                         ptr_output_per_vertex        = 0;
+	uint32_t                                         storage_runtime_array_type   = 0;
+	uint32_t                                         storage_buffer_type          = 0;
+	uint32_t                                         ptr_storage_buffer           = 0;
+	uint32_t                                         ptr_storage_buffer_uint      = 0;
+	uint32_t                                         storage_buffer_array_type    = 0;
+	uint32_t                                         ptr_storage_buffer_array     = 0;
+	uint32_t                                         storage_buffer_variable      = 0;
 	std::array<uint32_t, IR::ShaderInfo::MaxBuffers> storage_buffer_offsets {};
-	uint32_t                               address_memory_array_type    = 0;
-	uint32_t                               ptr_address_memory_array     = 0;
-	uint32_t                               address_memory_variable      = 0;
-	uint32_t                               gds_variable                 = 0;
-	uint32_t                               push_constant_array_type     = 0;
-	uint32_t                               push_constant_block_type     = 0;
-	uint32_t                               ptr_push_constant_block      = 0;
-	uint32_t                               ptr_push_constant_uint       = 0;
-	uint32_t                               push_constant_variable       = 0;
-	uint32_t                               vsharp_storage_variable      = 0;
-	uint32_t                               flattened_srt_variable       = 0;
-	uint32_t                               lds_array_type               = 0;
-	uint32_t                               ptr_workgroup_array          = 0;
-	uint32_t                               ptr_workgroup_uint           = 0;
-	uint32_t                               lds_variable                 = 0;
-	std::array<SampledImageDescriptors, 10> sampled_images;
-	std::array<StorageImageDescriptors, 10> storage_images;
-	uint32_t                               sampler_type                                  = 0;
-	uint32_t                               sampler_array_type                            = 0;
-	uint32_t                               ptr_uniform_sampler                           = 0;
-	uint32_t                               ptr_uniform_sampler_array                     = 0;
-	uint32_t                               sampler_variable                              = 0;
-	uint32_t                               ptr_image_uint                                = 0;
-	uint32_t                               func_type                                     = 0;
-	uint32_t                               main_func                                     = 0;
-	uint32_t                               entry_label                                   = 0;
-	uint32_t                               pixel_valid_mask_variable                     = 0;
-	bool                                   dispatcher_fallback                           = false;
-	uint32_t                               dispatch_pc_variable                          = 0;
-	uint32_t                               dispatch_header_label                         = 0;
-	uint32_t                               dispatch_select_label                         = 0;
-	uint32_t                               dispatch_default_label                        = 0;
-	uint32_t                               dispatch_after_switch_label                   = 0;
-	uint32_t                               dispatch_continue_label                       = 0;
-	uint32_t                               dispatch_merge_label                          = 0;
-	uint32_t                               glsl_std450                                   = 0;
-	uint32_t                               subgroup_local_invocation_id_variable         = 0;
-	uint32_t                               per_vertex_variable                           = 0;
-	uint32_t                               depth_variable                                = 0;
-	uint32_t                               sample_mask_variable                          = 0;
-	bool                                   needs_subgroup_ballot                         = false;
-	bool                                   needs_subgroup_shuffle                        = false;
-	bool                                   needs_subgroup_local_invocation_id            = false;
-	bool                                   needs_compute_derivatives                     = false;
-	bool                                   needs_image_gather_extended                   = false;
-	bool                                   needs_function_lds                            = false;
-	bool                                   needs_pixel_valid_mask                        = false;
-	std::vector<RegisterBinding>           registers;
-	std::vector<InputBinding>              inputs;
-	std::vector<OutputBinding>             outputs;
-	std::vector<uint32_t>                  interface_variables;
-	std::vector<bool>                      reachable_blocks;
-	std::map<uint32_t, uint32_t>           block_labels;
-	std::map<uint32_t, uint32_t>           constants;
-	std::map<uint32_t, uint32_t>           signed_constants;
-	std::map<uint32_t, uint32_t>           float_constants;
+	uint32_t                                         address_memory_array_type = 0;
+	uint32_t                                         ptr_address_memory_array  = 0;
+	uint32_t                                         address_memory_variable   = 0;
+	uint32_t                                         gds_variable              = 0;
+	uint32_t                                         push_constant_array_type  = 0;
+	uint32_t                                         push_constant_block_type  = 0;
+	uint32_t                                         ptr_push_constant_block   = 0;
+	uint32_t                                         ptr_push_constant_uint    = 0;
+	uint32_t                                         push_constant_variable    = 0;
+	uint32_t                                         vsharp_storage_variable   = 0;
+	uint32_t                                         flattened_srt_variable    = 0;
+	uint32_t                                         lds_array_type            = 0;
+	uint32_t                                         ptr_workgroup_array       = 0;
+	uint32_t                                         ptr_workgroup_uint        = 0;
+	uint32_t                                         lds_variable              = 0;
+	std::array<SampledImageDescriptors, 10>          sampled_images;
+	std::array<StorageImageDescriptors, 10>          storage_images;
+	uint32_t                                         sampler_type                          = 0;
+	uint32_t                                         sampler_array_type                    = 0;
+	uint32_t                                         ptr_uniform_sampler                   = 0;
+	uint32_t                                         ptr_uniform_sampler_array             = 0;
+	uint32_t                                         sampler_variable                      = 0;
+	uint32_t                                         ptr_image_uint                        = 0;
+	uint32_t                                         func_type                             = 0;
+	uint32_t                                         main_func                             = 0;
+	uint32_t                                         entry_label                           = 0;
+	uint32_t                                         pixel_valid_mask_variable             = 0;
+	bool                                             dispatcher_fallback                   = false;
+	uint32_t                                         dispatch_pc_variable                  = 0;
+	uint32_t                                         dispatch_header_label                 = 0;
+	uint32_t                                         dispatch_select_label                 = 0;
+	uint32_t                                         dispatch_default_label                = 0;
+	uint32_t                                         dispatch_after_switch_label           = 0;
+	uint32_t                                         dispatch_continue_label               = 0;
+	uint32_t                                         dispatch_merge_label                  = 0;
+	uint32_t                                         glsl_std450                           = 0;
+	uint32_t                                         subgroup_local_invocation_id_variable = 0;
+	uint32_t                                         per_vertex_variable                   = 0;
+	uint32_t                                         depth_variable                        = 0;
+	uint32_t                                         sample_mask_variable                  = 0;
+	bool                                             needs_subgroup_ballot                 = false;
+	bool                                             needs_subgroup_shuffle                = false;
+	bool                                             needs_subgroup_local_invocation_id    = false;
+	bool                                             needs_compute_derivatives             = false;
+	bool                                             needs_image_gather_extended           = false;
+	bool                                             needs_function_lds                    = false;
+	bool                                             needs_pixel_valid_mask                = false;
+	std::vector<RegisterBinding>                     registers;
+	std::vector<InputBinding>                        inputs;
+	std::vector<OutputBinding>                       outputs;
+	std::vector<uint32_t>                            interface_variables;
+	std::vector<bool>                                reachable_blocks;
+	std::map<uint32_t, uint32_t>                     block_labels;
+	std::map<uint32_t, uint32_t>                     constants;
+	std::map<uint32_t, uint32_t>                     signed_constants;
+	std::map<uint32_t, uint32_t>                     float_constants;
 };
 
 constexpr uint32_t PsInputOffsetMask = 0x0000001fu;
@@ -990,11 +990,6 @@ uint32_t NormalizeFormatComponent(EmitterState& state, const Format::BufferForma
 uint32_t UnpackTBufferFormat(EmitterState& state, const IR::Instruction& inst,
                              const Format::BufferFormatInfo& info);
 
-bool EmitTypedTBufferLoad(EmitterState& state, const IR::Instruction& inst,
-                          const Format::BufferFormatInfo& info);
-
-bool EmitFormattedBufferLoad(EmitterState& state, const IR::Instruction& inst);
-
 uint32_t FormattedBufferDwordStoreComponentCount(Prospero::BufferFormat format,
                                                  uint32_t               opcode_components);
 
@@ -1020,6 +1015,9 @@ void EmitBufferLoadSshort(EmitterState& state, const IR::Instruction& inst);
 
 void EmitBufferLoadDword(EmitterState& state, const IR::Instruction& inst);
 
+void EmitBufferLoadDwordGroup(EmitterState& state, const IR::Instruction* instructions,
+                              uint32_t count);
+
 void EmitBufferStoreDword(EmitterState& state, const IR::Instruction& inst);
 
 void EmitFlatLoadUbyte(EmitterState& state, const IR::Instruction& inst);

src/graphics/shader/recompiler/emitter/spirvEmitterMemory.cpp

@@ -34,16 +34,15 @@ uint32_t EmitDppWriteActiveBool(EmitterState& state, const IR::Operand& dst) {
 	    {OpShiftLeftLogical, state.uint_type, bank_bit, ConstantU32(state, 1), bank});
 	state.builder.AddFunction(
 	    {OpShiftLeftLogical, state.uint_type, row_bit, ConstantU32(state, 1), row});
-	state.builder.AddFunction({OpBitwiseAnd, state.uint_type, bank_hit,
-	                            ConstantU32(state, dst.dpp_bank_mask), bank_bit});
+	state.builder.AddFunction(
+	    {OpBitwiseAnd, state.uint_type, bank_hit, ConstantU32(state, dst.dpp_bank_mask), bank_bit});
 	state.builder.AddFunction(
 	    {OpBitwiseAnd, state.uint_type, row_hit, ConstantU32(state, dst.dpp_row_mask), row_bit});
 	state.builder.AddFunction(
 	    {OpINotEqual, state.bool_type, bank_active, bank_hit, ConstantU32(state, 0)});
 	state.builder.AddFunction(
 	    {OpINotEqual, state.bool_type, row_active, row_hit, ConstantU32(state, 0)});
-	state.builder.AddFunction(
-	    {OpLogicalAnd, state.bool_type, dpp_active, bank_active, row_active});
+	state.builder.AddFunction({OpLogicalAnd, state.bool_type, dpp_active, bank_active, row_active});
 	uint32_t write_active = dpp_active;
 	if (!dst.dpp_bound_ctrl) {
 		const auto target  = EmitDppTargetLane(state, dst.dpp_ctrl);
@@ -102,7 +101,7 @@ void EmitStoreU32(EmitterState& state, const IR::Operand& dst, uint32_t value) {
 			const auto selected  = state.builder.AllocateId();
 			state.builder.AddFunction({OpLoad, state.uint_type, old_value, pointer});
 			state.builder.AddFunction({OpSelect, state.uint_type, selected,
-			                            EmitDppWriteActiveBool(state, dst), wave_value, old_value});
+			                           EmitDppWriteActiveBool(state, dst), wave_value, old_value});
 			state.builder.AddFunction({OpStore, pointer, selected});
 			return;
 		}
@@ -131,8 +130,7 @@ uint32_t EmitNotEqualZeroBool(EmitterState& state, uint32_t value) {
 uint32_t EmitSelectU32Value(EmitterState& state, uint32_t condition, uint32_t true_value,
                             uint32_t false_value) {
 	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpSelect, state.uint_type, ret, condition, true_value, false_value});
+	state.builder.AddFunction({OpSelect, state.uint_type, ret, condition, true_value, false_value});
 	return ret;
 }
 
@@ -212,12 +210,12 @@ bool IsStorageBufferMemoryKind(IR::ResourceKind kind) {
 
 void EmitStorageBufferOffsets(EmitterState& state) {
 	for (uint32_t i = 0; i < state.program.bindings.buffer_offset_count; i++) {
-		const auto word = EmitShaderDataDwordLoad(
-		    state, state.program.bindings.buffer_offset_dword + i / 4u);
-		const auto shift = ConstantU32(state, (i % 4u) * 8u + 2u);
+		const auto word =
+		    EmitShaderDataDwordLoad(state, state.program.bindings.buffer_offset_dword + i / 4u);
+		const auto shift                = ConstantU32(state, (i % 4u) * 8u + 2u);
 		state.storage_buffer_offsets[i] = EmitBinaryU32(
-		    state, OpBitwiseAnd,
-		    EmitBinaryU32(state, OpShiftRightLogical, word, shift), ConstantU32(state, 0x3fu));
+		    state, OpBitwiseAnd, EmitBinaryU32(state, OpShiftRightLogical, word, shift),
+		    ConstantU32(state, 0x3fu));
 	}
 }
 
@@ -329,8 +327,7 @@ uint32_t EmitRelativeAddress(EmitterState& state, const IR::Instruction& inst, u
 
 uint32_t EmitFlatVirtualAddress(EmitterState& state, const IR::Instruction& inst,
                                 uint32_t first_src, uint32_t src_count) {
-	if (inst.memory.resource >= state.resources.addresses.size() ||
-	    src_count < 2) {
+	if (inst.memory.resource >= state.resources.addresses.size() || src_count < 2) {
 		ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::AddressMemory,
 		                             inst.memory.resource, "flat address snapshot is missing");
 	}
@@ -429,20 +426,20 @@ uint32_t EmitStorageBufferObjectPointer(EmitterState& state, const IR::MemoryInf
 		    ResourceForDescriptor(state, IR::DescriptorBindingKind::AddressMemory, mem.resource);
 		const auto pointer = state.builder.AllocateId();
 		state.builder.AddFunction({OpAccessChain, state.ptr_storage_buffer, pointer,
-		                            state.address_memory_variable,
-		                            ConstantU32(state, binding.array_index)});
+		                           state.address_memory_variable,
+		                           ConstantU32(state, binding.array_index)});
 		return pointer;
 	}
 	const auto binding = StorageBufferBindingForMemory(state, mem, use_pc);
 	const auto pointer = state.builder.AllocateId();
 	state.builder.AddFunction({OpAccessChain, state.ptr_storage_buffer, pointer,
-	                            state.storage_buffer_variable,
-	                            ConstantU32(state, binding.array_index)});
+	                           state.storage_buffer_variable,
+	                           ConstantU32(state, binding.array_index)});
 	return pointer;
 }
 
 uint32_t EmitStorageBufferElementInBounds(EmitterState& state, const IR::MemoryInfo& mem,
-                                           uint32_t index, uint32_t use_pc) {
+                                          uint32_t index, uint32_t use_pc) {
 	index                = EmitStorageBufferIndex(state, mem, index, use_pc);
 	const auto object    = EmitStorageBufferObjectPointer(state, mem, use_pc);
 	const auto length    = state.builder.AllocateId();
@@ -453,7 +450,7 @@ uint32_t EmitStorageBufferElementInBounds(EmitterState& state, const IR::MemoryI
 }
 
 uint32_t EmitStorageBufferElementPointer(EmitterState& state, const IR::MemoryInfo& mem,
-                                          uint32_t index, uint32_t use_pc) {
+                                         uint32_t index, uint32_t use_pc) {
 	index = EmitStorageBufferIndex(state, mem, index, use_pc);
 	if (IsFlatMemoryKind(mem.kind)) {
 		if (state.address_memory_variable == 0) {
@@ -589,9 +586,9 @@ uint32_t EmitMemoryLoadSubDwordValueU32(EmitterState& state, const IR::Instructi
 			const auto left       = state.builder.AllocateId();
 			const auto sign_shift = 32u - data_bits;
 			state.builder.AddFunction({OpShiftLeftLogical, state.uint_type, left, masked,
-			                            ConstantU32(state, sign_shift)});
+			                           ConstantU32(state, sign_shift)});
 			state.builder.AddFunction({OpShiftRightArithmetic, state.uint_type, value, left,
-			                            ConstantU32(state, sign_shift)});
+			                           ConstantU32(state, sign_shift)});
 		}
 		return value;
 	};
@@ -659,16 +656,15 @@ void EmitAtomicUpdateU32(EmitterState& state, uint32_t pointer, IR::ResourceKind
 	state.builder.AddFunction({OpBranch, preheader});
 	state.builder.AddFunction({OpLabel, preheader});
 	state.builder.AddFunction({OpAtomicLoad, state.uint_type, initial, pointer,
-	                            ConstantU32(state, scope),
-	                            ConstantU32(state, MemorySemanticsNone)});
+	                           ConstantU32(state, scope), ConstantU32(state, MemorySemanticsNone)});
 	state.builder.AddFunction({OpBranch, header});
 	state.builder.AddFunction({OpLabel, header});
 	state.builder.AddFunction(
 	    {OpPhi, state.uint_type, observed, initial, preheader, exchanged, continue_label});
 	const auto desired = desired_value(observed);
 	state.builder.AddFunction({OpAtomicCompareExchange, state.uint_type, exchanged, pointer,
-	                            ConstantU32(state, scope), ConstantU32(state, MemorySemanticsNone),
-	                            ConstantU32(state, MemorySemanticsNone), desired, observed});
+	                           ConstantU32(state, scope), ConstantU32(state, MemorySemanticsNone),
+	                           ConstantU32(state, MemorySemanticsNone), desired, observed});
 	const auto success = state.builder.AllocateId();
 	state.builder.AddFunction({OpIEqual, state.bool_type, success, exchanged, observed});
 	state.builder.AddFunction({OpLoopMerge, merge, continue_label, LoopControlNone});
@@ -793,8 +789,7 @@ uint32_t EmitTBufferBitcastU32ToI32(EmitterState& state, uint32_t value) {
 uint32_t EmitTBufferCompareU32Constant(EmitterState& state, uint32_t opcode, uint32_t value,
                                        uint32_t constant) {
 	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {opcode, state.bool_type, ret, value, ConstantU32(state, constant)});
+	state.builder.AddFunction({opcode, state.bool_type, ret, value, ConstantU32(state, constant)});
 	return ret;
 }
 
@@ -821,7 +816,7 @@ uint32_t EmitExtractFormatFieldU32(EmitterState& state, uint32_t raw_word, uint3
 		const auto signed_word = EmitTBufferBitcastU32ToI32(state, raw_word);
 		const auto extracted   = state.builder.AllocateId();
 		state.builder.AddFunction({OpBitFieldSExtract, state.int_type, extracted, signed_word,
-		                            ConstantU32(state, offset), ConstantU32(state, bits)});
+		                           ConstantU32(state, offset), ConstantU32(state, bits)});
 		const auto ret = state.builder.AllocateId();
 		state.builder.AddFunction({OpBitcast, state.uint_type, ret, extracted});
 		return ret;
@@ -829,7 +824,7 @@ uint32_t EmitExtractFormatFieldU32(EmitterState& state, uint32_t raw_word, uint3
 
 	const auto extracted = state.builder.AllocateId();
 	state.builder.AddFunction({OpBitFieldUExtract, state.uint_type, extracted, raw_word,
-	                            ConstantU32(state, offset), ConstantU32(state, bits)});
+	                           ConstantU32(state, offset), ConstantU32(state, bits)});
 	return extracted;
 }
 
@@ -940,7 +935,7 @@ uint32_t NormalizeFormatComponent(EmitterState& state, const Format::BufferForma
 			state.builder.AddFunction(
 			    {OpFDiv, state.float_type, normalized, value, ConstantF32Value(state, max_value)});
 			state.builder.AddFunction({OpExtInst, state.float_type, clamped, state.glsl_std450,
-			                            GlslFMax, normalized, ConstantF32Value(state, -1.0f)});
+			                           GlslFMax, normalized, ConstantF32Value(state, -1.0f)});
 			return EmitTBufferBitcastF32ToU32(state, clamped);
 		}
 		case Format::ComponentType::Float:
@@ -961,18 +956,8 @@ uint32_t UnpackTBufferFormat(EmitterState& state, const IR::Instruction& inst,
 	return NormalizeFormatComponent(state, info, inst.memory.component_index, raw);
 }
 
-bool EmitTypedTBufferLoad(EmitterState& state, const IR::Instruction& inst,
-                          const Format::BufferFormatInfo& info) {
-	if (!Format::CanUseTypedBufferLoad(info.format)) {
-		return false;
-	}
-	const auto value = EmitMemoryLoadDwordValueU32(state, inst, IR::ResourceKind::Buffer, 0,
-	                                               AddressSourceCount(inst, 0));
-	EmitStoreU32(state, inst.dst, value);
-	return true;
-}
-
-bool EmitFormattedBufferLoad(EmitterState& state, const IR::Instruction& inst) {
+bool EmitFormattedBufferLoadValueU32(EmitterState& state, const IR::Instruction& inst,
+                                     uint32_t& value) {
 	if (!IsFormattedBufferComponent(inst)) {
 		return false;
 	}
@@ -984,18 +969,29 @@ bool EmitFormattedBufferLoad(EmitterState& state, const IR::Instruction& inst) {
 
 	const auto info = Format::GetFormatInfo(format);
 	if (inst.memory.component_index >= info.component_count) {
-		EmitStoreU32(state, inst.dst, ConstantU32(state, 0));
+		value = ConstantU32(state, 0);
 		return true;
 	}
 
-	if (EmitTypedTBufferLoad(state, inst, info)) {
+	if (Format::CanUseTypedBufferLoad(info.format)) {
+		value = EmitMemoryLoadDwordValueU32(state, inst, IR::ResourceKind::Buffer, 0,
+		                                    AddressSourceCount(inst, 0));
 		return true;
 	}
 
-	EmitStoreU32(state, inst.dst, UnpackTBufferFormat(state, inst, info));
+	value = UnpackTBufferFormat(state, inst, info);
 	return true;
 }
 
+uint32_t EmitBufferLoadDwordValueU32(EmitterState& state, const IR::Instruction& inst) {
+	uint32_t value = 0;
+	if (EmitFormattedBufferLoadValueU32(state, inst, value)) {
+		return value;
+	}
+	return EmitMemoryLoadDwordValueU32(state, inst, IR::ResourceKind::Buffer, 0,
+	                                   AddressSourceCount(inst, 0));
+}
+
 uint32_t FormattedBufferDwordStoreComponentCount(Prospero::BufferFormat format,
                                                  uint32_t               opcode_components) {
 	switch (format) {
@@ -1117,8 +1113,8 @@ uint32_t EmitAtomicPointer(EmitterState& state, const IR::Instruction& inst) {
 			    StorageImageDescriptorPointer(state, inst.memory.resource, true, inst.pc, view);
 			const auto pointer = state.builder.AllocateId();
 			state.builder.AddFunction({OpImageTexelPointer, state.ptr_image_uint, pointer,
-			                            image_pointer, EmitImageCoordU32(state, inst, view),
-			                            ConstantU32(state, 0)});
+			                           image_pointer, EmitImageCoordU32(state, inst, view),
+			                           ConstantU32(state, 0)});
 			return pointer;
 		}
 		default: return 0;
@@ -1142,8 +1138,8 @@ void EmitAtomicU32(EmitterState& state, const IR::Instruction& inst, uint32_t op
 			    EmitStorageBufferElementPointer(state, inst.memory, index, inst.pc);
 			const auto result = state.builder.AllocateId();
 			state.builder.AddFunction({opcode, state.uint_type, result, pointer,
-			                            ConstantU32(state, ScopeDevice),
-			                            ConstantU32(state, MemorySemanticsNone), value});
+			                           ConstantU32(state, ScopeDevice),
+			                           ConstantU32(state, MemorySemanticsNone), value});
 			EmitDeviceAtomicMemoryBarrier(state);
 			return result;
 		});
@@ -1158,8 +1154,8 @@ void EmitAtomicU32(EmitterState& state, const IR::Instruction& inst, uint32_t op
 			const auto pointer = EmitGdsElementPointer(state, index);
 			const auto result  = state.builder.AllocateId();
 			state.builder.AddFunction({opcode, state.uint_type, result, pointer,
-			                            ConstantU32(state, ScopeDevice),
-			                            ConstantU32(state, MemorySemanticsNone), value});
+			                           ConstantU32(state, ScopeDevice),
+			                           ConstantU32(state, MemorySemanticsNone), value});
 			EmitDeviceAtomicMemoryBarrier(state);
 			return result;
 		});
@@ -1177,7 +1173,7 @@ void EmitAtomicU32(EmitterState& state, const IR::Instruction& inst, uint32_t op
 	const auto old   = state.builder.AllocateId();
 	const auto scope = inst.memory.kind == IR::ResourceKind::Lds ? ScopeWorkgroup : ScopeDevice;
 	state.builder.AddFunction({opcode, state.uint_type, old, pointer, ConstantU32(state, scope),
-	                            ConstantU32(state, MemorySemanticsNone), value});
+	                           ConstantU32(state, MemorySemanticsNone), value});
 	if (inst.memory.kind == IR::ResourceKind::StorageImageUint ||
 	    inst.memory.kind == IR::ResourceKind::Gds) {
 		EmitDeviceAtomicMemoryBarrier(state);
@@ -1199,8 +1195,8 @@ void EmitSLoadDword(EmitterState& state, const IR::Instruction& inst) {
 	    {OpShiftRightLogical, state.uint_type, index, address, ConstantU32(state, 2)});
 	const auto object = state.builder.AllocateId();
 	state.builder.AddFunction({OpAccessChain, state.ptr_storage_buffer, object,
-	                            state.address_memory_variable,
-	                            ConstantU32(state, binding.array_index)});
+	                           state.address_memory_variable,
+	                           ConstantU32(state, binding.array_index)});
 	const auto length    = state.builder.AllocateId();
 	const auto in_bounds = state.builder.AllocateId();
 	state.builder.AddFunction({OpArrayLength, state.uint_type, length, object, 0});
@@ -1227,8 +1223,8 @@ void EmitLoadSrtDword(EmitterState& state, const IR::Instruction& inst) {
 	const auto pointer = state.builder.AllocateId();
 	const auto value   = state.builder.AllocateId();
 	state.builder.AddFunction({OpAccessChain, state.ptr_storage_buffer_uint, pointer,
-	                            state.flattened_srt_variable, ConstantU32(state, 0),
-	                            ConstantU32(state, inst.src[0].imm)});
+	                           state.flattened_srt_variable, ConstantU32(state, 0),
+	                           ConstantU32(state, inst.src[0].imm)});
 	state.builder.AddFunction({OpLoad, state.uint_type, value, pointer});
 	EmitStoreU32(state, inst.dst, value);
 }
@@ -1262,11 +1258,27 @@ void EmitBufferLoadSshort(EmitterState& state, const IR::Instruction& inst) {
 }
 
 void EmitBufferLoadDword(EmitterState& state, const IR::Instruction& inst) {
+	EmitGuardedByExec(
+	    state, [&]() { EmitStoreU32(state, inst.dst, EmitBufferLoadDwordValueU32(state, inst)); });
+}
+
+void EmitBufferLoadDwordGroup(EmitterState& state, const IR::Instruction* instructions,
+                              uint32_t count) {
+	if (instructions == nullptr || count == 0u) {
+		return;
+	}
+
 	EmitGuardedByExec(state, [&]() {
-		if (EmitFormattedBufferLoad(state, inst)) {
-			return;
+		// RDNA VMEM captures every VADDR component before making overlapping VDATA writes
+		// visible. Keep the split IR components instruction-atomic by deferring all stores.
+		std::vector<uint32_t> values;
+		values.reserve(count);
+		for (uint32_t i = 0; i < count; i++) {
+			values.push_back(EmitBufferLoadDwordValueU32(state, instructions[i]));
+		}
+		for (uint32_t i = 0; i < count; i++) {
+			EmitStoreU32(state, instructions[i].dst, values[i]);
 		}
-		EmitMemoryLoadU32(state, inst, IR::ResourceKind::Buffer, 0, AddressSourceCount(inst, 0));
 	});
 }
 
@@ -1385,7 +1397,7 @@ DsCounterAddress EmitAppendConsumeAddress(EmitterState& state, const IR::Instruc
 	state.builder.AddFunction(
 	    {OpShiftRightLogical, state.uint_type, index, address, ConstantU32(state, 2)});
 	state.builder.AddFunction({OpULessThan, state.bool_type, in_bounds,
-	                            ConstantU32(state, inst.memory.offset + 3u), size});
+	                           ConstantU32(state, inst.memory.offset + 3u), size});
 	return {index, size, in_bounds};
 }
 
@@ -1400,7 +1412,7 @@ uint32_t EmitGdsElementInBounds(EmitterState& state, uint32_t index) {
 uint32_t EmitGdsElementPointer(EmitterState& state, uint32_t index) {
 	const auto pointer = state.builder.AllocateId();
 	state.builder.AddFunction({OpAccessChain, state.ptr_storage_buffer_uint, pointer,
-	                            state.gds_variable, ConstantU32(state, 0), index});
+	                           state.gds_variable, ConstantU32(state, 0), index});
 	return pointer;
 }
 
@@ -1431,7 +1443,7 @@ ExecMaskInfo EmitExecMaskInfo(EmitterState& state) {
 	if (state.per_invocation_masks) {
 		const auto ballot = state.builder.AllocateId();
 		state.builder.AddFunction({OpGroupNonUniformBallot, state.vec4_uint_type, ballot,
-		                            ConstantU32(state, ScopeSubgroup), EmitExecActiveBool(state)});
+		                           ConstantU32(state, ScopeSubgroup), EmitExecActiveBool(state)});
 		exec_lo = state.builder.AllocateId();
 		state.builder.AddFunction({OpCompositeExtract, state.uint_type, exec_lo, ballot, 0});
 		if (state.wave_size == 64u) {
@@ -1482,28 +1494,27 @@ void EmitDsAppendConsume(EmitterState& state, const IR::Instruction& inst, uint3
 	const auto do_atomic  = state.builder.AllocateId();
 	state.builder.AddFunction({OpIEqual, state.bool_type, first_lane, subid, exec.first_lane});
 	const auto first_active = EmitLogicalAndBool(state, first_lane, exec.any_active);
-	state.builder.AddFunction(
-	    {OpLogicalAnd, state.bool_type, do_atomic, first_active, in_bounds});
+	state.builder.AddFunction({OpLogicalAnd, state.bool_type, do_atomic, first_active, in_bounds});
 
 	const auto atomic_value = EmitValueOrZeroIfCondition(state, do_atomic, [&]() {
 		const auto pointer = gds ? EmitGdsElementPointer(state, address.index)
 		                         : EmitLdsElementPointer(state, address.index);
 		const auto result  = state.builder.AllocateId();
 		state.builder.AddFunction({atomic_opcode, state.uint_type, result, pointer,
-		                            ConstantU32(state, gds ? ScopeDevice : ScopeWorkgroup),
-		                            ConstantU32(state, MemorySemanticsNone), exec.active_count});
+		                           ConstantU32(state, gds ? ScopeDevice : ScopeWorkgroup),
+		                           ConstantU32(state, MemorySemanticsNone), exec.active_count});
 		if (gds) {
 			EmitDeviceAtomicMemoryBarrier(state);
 		} else {
 			const auto semantics = MemorySemanticsAcquireRelease | MemorySemanticsWorkgroupMemory;
 			state.builder.AddFunction({OpMemoryBarrier, ConstantU32(state, ScopeWorkgroup),
-			                            ConstantU32(state, semantics)});
+			                           ConstantU32(state, semantics)});
 		}
 		return result;
 	});
 	const auto broadcast    = state.builder.AllocateId();
 	state.builder.AddFunction({OpGroupNonUniformShuffle, state.uint_type, broadcast,
-	                            ConstantU32(state, ScopeSubgroup), atomic_value, exec.first_lane});
+	                           ConstantU32(state, ScopeSubgroup), atomic_value, exec.first_lane});
 	const auto value = EmitSelectU32Value(state, exec.any_active, broadcast, ConstantU32(state, 0));
 	EmitStoreU32(state, inst.dst, value);
 }
@@ -1523,8 +1534,8 @@ void EmitDsFloatMinMaxF32(EmitterState& state, const IR::Instruction& inst, bool
 			const auto value_u32 = state.builder.AllocateId();
 			state.builder.AddFunction({OpBitcast, state.float_type, old_f32, old_u32});
 			state.builder.AddFunction({max_value ? OpFOrdGreaterThan : OpFOrdLessThan,
-			                            state.bool_type, store_src, max_value ? old_f32 : cmp_f32,
-			                            max_value ? cmp_f32 : old_f32});
+			                           state.bool_type, store_src, max_value ? old_f32 : cmp_f32,
+			                           max_value ? cmp_f32 : old_f32});
 			state.builder.AddFunction(
 			    {OpSelect, state.float_type, value_f32, store_src, data_f32, old_f32});
 			state.builder.AddFunction({OpBitcast, state.uint_type, value_u32, value_f32});
@@ -1575,14 +1586,13 @@ uint32_t EmitDsSwizzleTargetLane(EmitterState& state, uint32_t subid, uint32_t c
 	const auto xored  = state.builder.AllocateId();
 	const auto base   = state.builder.AllocateId();
 	const auto target = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, 31)});
+	state.builder.AddFunction({OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, 31)});
 	state.builder.AddFunction(
 	    {OpBitwiseAnd, state.uint_type, masked, lane, ConstantU32(state, control & 0x1fu)});
 	state.builder.AddFunction(
 	    {OpBitwiseOr, state.uint_type, ored, masked, ConstantU32(state, (control >> 5u) & 0x1fu)});
-	state.builder.AddFunction({OpBitwiseXor, state.uint_type, xored, ored,
-	                            ConstantU32(state, (control >> 10u) & 0x1fu)});
+	state.builder.AddFunction(
+	    {OpBitwiseXor, state.uint_type, xored, ored, ConstantU32(state, (control >> 10u) & 0x1fu)});
 	state.builder.AddFunction(
 	    {OpBitwiseAnd, state.uint_type, base, subid, ConstantU32(state, 0xffffffe0u)});
 	state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, base, xored});
@@ -1596,7 +1606,7 @@ void EmitDsSwizzleB32(EmitterState& state, const IR::Instruction& inst) {
 	const auto target  = EmitDsSwizzleTargetLane(state, subid, control);
 	const auto value   = state.builder.AllocateId();
 	state.builder.AddFunction({OpGroupNonUniformShuffle, state.uint_type, value,
-	                            ConstantU32(state, ScopeSubgroup), source, target});
+	                           ConstantU32(state, ScopeSubgroup), source, target});
 	const auto exec_active     = EmitLaneIndexActiveBool(state, target);
 	const auto subgroup_active = EmitSubgroupLaneActiveBool(state, target);
 	const auto source_active   = state.builder.AllocateId();

src/graphics/shader/recompiler/ir/ResourceMaterialization.cpp

@@ -12,7 +12,7 @@ namespace {
 
 constexpr uint64_t AddressMask = 0x0000ffffffffffffull;
 
-Decoder::ImageDimension DescriptorDimension(const DescriptorValue&       descriptor,
+Decoder::ImageDimension DescriptorDimension(const DescriptorValue&  descriptor,
                                             Decoder::ImageDimension requested) {
 	const bool is_array = requested == Decoder::ImageDimension::Dim1DArray ||
 	                      requested == Decoder::ImageDimension::Dim2DArray;
@@ -51,8 +51,7 @@ bool ValidImageDescriptor(const DescriptorValue& descriptor) {
 		const auto base_level = (descriptor.dwords[3] >> 12u) & 0xfu;
 		const auto fragments  = (descriptor.dwords[3] >> 16u) & 0xfu;
 		const auto max_mip    = (descriptor.dwords[5] >> 4u) & 0xfu;
-		return base_level == 0 && fragments >= 1 && fragments <= 3 &&
-		       max_mip == fragments;
+		return base_level == 0 && fragments >= 1 && fragments <= 3 && max_mip == fragments;
 	}
 	return true;
 }
@@ -61,6 +60,11 @@ uint32_t DescriptorImageSwizzle(const DescriptorValue& descriptor) {
 	return descriptor.dwords[3] & 0xfffu;
 }
 
+bool DescriptorIsCube(const DescriptorValue& descriptor) {
+	return static_cast<Prospero::ImageType>((descriptor.dwords[3] >> 28u) & 0xfu) ==
+	       Prospero::ImageType::kCube;
+}
+
 bool DecodeBufferDescriptor(const DescriptorValue& descriptor, ShaderBufferResource& result) {
 	if (descriptor.dword_count != std::size(result.fields)) {
 		return false;
@@ -171,12 +175,13 @@ bool ValidateResourceSpecialization(const Program& program, const ResourceSnapsh
 		const auto& image      = program.info.images[i];
 		const auto& descriptor = snapshot.images[i];
 		if (NullImageDescriptor(descriptor)) {
-			bool canonical_kind = image.kind == ResourceKind::Image ||
-			                      image.kind == ResourceKind::StorageImage;
+			bool canonical_kind =
+			    image.kind == ResourceKind::Image || image.kind == ResourceKind::StorageImage;
 			if (image.atomic) {
 				canonical_kind = image.kind == ResourceKind::StorageImageUint;
 			}
-			if (image.dimension != Decoder::ImageDimension::Dim2D || !canonical_kind) {
+			if (image.dimension != Decoder::ImageDimension::Dim2D || image.cube ||
+			    !canonical_kind) {
 				if (error != nullptr) {
 					*error = fmt::format(
 					    "image descriptor {} no longer matches canonical null specialization", i);
@@ -186,7 +191,8 @@ bool ValidateResourceSpecialization(const Program& program, const ResourceSnapsh
 			continue;
 		}
 		const auto dimension = DescriptorDimension(descriptor, image.dimension);
-		if (dimension == Decoder::ImageDimension::Unknown || dimension != image.dimension) {
+		if (dimension == Decoder::ImageDimension::Unknown || dimension != image.dimension ||
+		    DescriptorIsCube(descriptor) != image.cube) {
 			if (error != nullptr) {
 				*error =
 				    fmt::format("image descriptor {} no longer matches specialized dimension", i);
@@ -361,6 +367,7 @@ bool SpecializeResources(Program& program, const ResourceSnapshot& snapshot, std
 		auto&       image      = next.images[i];
 		if (NullImageDescriptor(descriptor)) {
 			image.dimension = Decoder::ImageDimension::Dim2D;
+			image.cube      = false;
 			switch (image.kind) {
 				case ResourceKind::ImageUint: image.kind = ResourceKind::Image; break;
 				case ResourceKind::StorageImageUint:
@@ -386,6 +393,7 @@ bool SpecializeResources(Program& program, const ResourceSnapshot& snapshot, std
 			return false;
 		}
 		image.dimension = descriptor_dimension;
+		image.cube      = DescriptorIsCube(descriptor);
 		if (image.kind == ResourceKind::StorageImage ||
 		    image.kind == ResourceKind::StorageImageUint) {
 			image.storage_swizzle = DescriptorImageSwizzle(descriptor);
@@ -402,6 +410,7 @@ bool SpecializeResources(Program& program, const ResourceSnapshot& snapshot, std
 		std::reference_wrapper<Instruction> inst;
 		ResourceKind                        kind;
 		Decoder::ImageDimension             dimension;
+		bool                                cube;
 	};
 	std::vector<ImagePatch> patches;
 	for (auto& block: program.blocks) {
@@ -420,13 +429,14 @@ bool SpecializeResources(Program& program, const ResourceSnapshot& snapshot, std
 				return false;
 			}
 			const auto& image = next.images[inst.memory.resource];
-			patches.push_back({std::ref(inst), image.kind, image.dimension});
+			patches.push_back({std::ref(inst), image.kind, image.dimension, image.cube});
 		}
 	}
 	program.info = std::move(next);
 	for (const auto& patch: patches) {
 		patch.inst.get().memory.kind            = patch.kind;
 		patch.inst.get().memory.image_dimension = patch.dimension;
+		patch.inst.get().memory.image_cube      = patch.cube;
 	}
 	return true;
 }

src/graphics/shader/recompiler/ir/ScalarProvenance.cpp

@@ -784,10 +784,10 @@ private:
 			if (incoming.empty()) {
 				return ScalarProvenance::Undefined;
 			}
-			if (incoming.size() == 1) {
-				return incoming[0];
-			}
 			if (*phi == ScalarProvenance::Undefined) {
+				if (incoming.size() == 1) {
+					return incoming[0];
+				}
 				*phi = AddValue({ScalarValueOp::Phi, block.start_pc});
 			}
 			m_graph.values[*phi].phi_args = std::move(incoming);

src/graphics/shader/recompiler/ir/ShaderIR.h

@@ -432,6 +432,7 @@ struct MemoryInfo {
 	bool     typed           = false;
 	bool     formatted       = false;
 	bool     image_has_mip   = false;
+	bool     image_cube      = false;
 	bool     glc             = false;
 	bool     slc             = false;
 	bool     idxen           = false;
@@ -607,6 +608,7 @@ struct ImageResource {
 	bool                    written         = false;
 	bool                    atomic          = false;
 	bool                    depth_compare   = false;
+	bool                    cube            = false;
 
 	bool operator==(const ImageResource& other) const = default;
 };

src/kernel/macosGuestAddressSpace.cpp

@@ -0,0 +1,8 @@
+#if defined(__APPLE__) && defined(__x86_64__)
+
+// Make the process own the guest ranges before any runtime initialization.
+asm(".zerofill SYSTEM_MANAGED,SYSTEM_MANAGED,__kyty_system_managed,0x7fffbc000");
+asm(".zerofill SYSTEM_RESERVED,SYSTEM_RESERVED,__kyty_system_reserved,0x7c0004000");
+asm(".zerofill USER_AREA,USER_AREA,__kyty_user_area,0x8c00000000");
+
+#endif

src/kernel/memory.h

@@ -99,13 +99,14 @@ struct KernelMemoryPoolBlockStats {
 static_assert(sizeof(KernelMemoryPoolBlockStats) == 16,
               "KernelMemoryPoolBlockStats struct size is incorrect");
 
-void               RegisterCallbacks(callback_func_t alloc_func, callback_func_t free_func);
-void               SetFlexibleMemorySize(uint64_t size);
-bool               TryWriteBacking(uint64_t vaddr, const void* data, uint64_t size);
-bool               TryReadBacking(uint64_t vaddr, void* data, uint64_t size);
-void               WriteBacking(uint64_t vaddr, const void* data, uint64_t size) noexcept;
-void               InvalidateMemory(uint64_t vaddr, uint64_t size);
-void               InstallGpuResources(Graphics::GpuResourceManager* resources) noexcept;
+void                   RegisterCallbacks(callback_func_t alloc_func, callback_func_t free_func);
+void                   SetFlexibleMemorySize(uint64_t size);
+bool                   TryWriteBacking(uint64_t vaddr, const void* data, uint64_t size);
+bool                   TryReadBacking(uint64_t vaddr, void* data, uint64_t size);
+[[nodiscard]] uint64_t ClampRangeSize(uint64_t vaddr, uint64_t size);
+void                   WriteBacking(uint64_t vaddr, const void* data, uint64_t size) noexcept;
+void                   InvalidateMemory(uint64_t vaddr, uint64_t size);
+void                   InstallGpuResources(Graphics::GpuResourceManager* resources) noexcept;
 [[nodiscard]] bool HandleGpuFault(Graphics::PageFaultAccess access, uint64_t fault_vaddr) noexcept;
 
 int KYTY_SYSV_ABI KernelMapNamedFlexibleMemory(void** addr_in_out, size_t len, int prot, int flags,
@@ -144,7 +145,7 @@ int KYTY_SYSV_ABI KernelIsStack(void* addr, void** start, void** end);
 int KYTY_SYSV_ABI KernelReserveVirtualRange(void** addr, size_t len, int flags, size_t alignment);
 bool              KernelHandleReservedRangeAccessViolation(uint64_t vaddr);
 int KYTY_SYSV_ABI KernelAvailableFlexibleMemorySize(size_t* size);
-int KYTY_SYSV_ABI KernelConfiguredFlexibleMemorySize(uint64_t* size);
+int KYTY_SYSV_ABI KernelConfiguredFlexibleMemorySize(size_t* size);
 int KYTY_SYSV_ABI KernelMprotect(const void* addr, size_t len, int prot);
 int KYTY_SYSV_ABI KernelMtypeprotect(const void* addr, size_t len, int type, int prot);
 int KYTY_SYSV_ABI KernelBatchMap(KernelBatchMapEntry* entries, int num_entries,
@@ -162,17 +163,30 @@ int KYTY_SYSV_ABI KernelMemoryPoolBatch(const KernelMemoryPoolBatchEntry* entrie
 int KYTY_SYSV_ABI KernelMemoryPoolGetBlockStats(KernelMemoryPoolBlockStats* output,
                                                 size_t                      output_size);
 
-void RegisterProgramMemory(uint64_t vaddr, uint64_t size, Common::VirtualMemory::Mode mode,
-                           const char* name);
-void UpdateProgramMemoryProtection(uint64_t vaddr, uint64_t size, Common::VirtualMemory::Mode mode);
-void UnregisterProgramMemory(uint64_t vaddr, uint64_t size);
+uint64_t AllocateProgramMemory(uint64_t search_addr, uint64_t size,
+                               Common::VirtualMemory::Mode mode, const char* name);
+void SetProgramMemoryProtection(uint64_t vaddr, uint64_t size, Common::VirtualMemory::Mode mode);
+uint64_t AllocateRuntimeMemory(uint64_t search_addr, uint64_t size,
+                               Common::VirtualMemory::Mode mode, const char* name,
+                               bool fixed = false);
+uint64_t AllocateGuestStackMemory(uint64_t search_addr, uint64_t size,
+                                  Common::VirtualMemory::Mode mode, const char* name);
+bool     ProtectGuestMemory(uint64_t vaddr, uint64_t size, Common::VirtualMemory::Mode mode,
+                            Common::VirtualMemory::Mode* old_mode = nullptr);
+// Transient PageManager watch state; does not change the guest mapping's semantic protection.
+bool ProtectGuestHostMemory(uint64_t vaddr, uint64_t size, Common::VirtualMemory::Mode mode);
+bool FreeGuestMemory(uint64_t vaddr, uint64_t size);
 
 #if defined(KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS)
-void TestFailNextPhysicalMemoryUnmap();
-void TestFailPhysicalMemoryUnmapAfter(uint32_t successful_unmaps);
-void TestFailHostReservationAfter(uint32_t successful_pages);
-void TestFailNextFixedReserveRangeRegistration();
-bool TestPlaceholderRangeIsFree(uint64_t vaddr, uint64_t size);
+void     TestFailNextPhysicalMemoryUnmap();
+void     TestFailPhysicalMemoryUnmapAfter(uint32_t successful_unmaps);
+void     TestFailGuestBackingStoreUnmapAfter(uint32_t successful_unmaps);
+void     TestFailNextFixedReserveRangeRegistration();
+bool     TestPlaceholderRangeIsFree(uint64_t vaddr, uint64_t size);
+bool     TestGuestAddressRangeIsOwned(uint64_t vaddr, uint64_t size);
+bool     TestGuestBackingOutsideAddressSpace();
+uint64_t TestGuestBackingSize();
+bool     TestGuestFreeRangeBounds();
 #endif
 
 } // namespace Libs::LibKernel::Memory

src/kernel/pthread.cpp

@@ -75,16 +75,16 @@ LIB_NAME("libkernel", "libkernel");
 #undef PTHREAD_STACK_MIN
 #endif
 
-constexpr int      KEYS_MAX                  = 256;
-constexpr int      DESTRUCTOR_ITERATIONS     = 4;
-constexpr size_t   PTHREAD_STACK_DEFAULT     = 0x100000;
-constexpr size_t   GUEST_PTHREAD_STACK_MIN   = 0x4000;
-constexpr size_t   PTHREAD_STACK_PAGE        = 0x4000;
-constexpr size_t   PTHREAD_STACK_GRANULARITY = 0x10000;
-constexpr size_t   PTHREAD_STACK_INITIAL     = 0x200000;
-constexpr size_t   PTHREAD_STACK_EXTRA       = 0x100000;
-constexpr uint64_t PTHREAD_STACK_TOP         = 0x7efff8000ull;
-constexpr uint32_t SIGNAL_APC_POLL_MICROS    = 10000;
+constexpr int      KEYS_MAX                = 256;
+constexpr int      DESTRUCTOR_ITERATIONS   = 4;
+constexpr size_t   PTHREAD_STACK_DEFAULT   = 0x100000;
+constexpr size_t   GUEST_PTHREAD_STACK_MIN = 0x4000;
+constexpr size_t   PTHREAD_STACK_PAGE      = 0x4000;
+constexpr size_t   PTHREAD_STACK_INITIAL   = 0x200000;
+constexpr size_t   PTHREAD_STACK_EXTRA     = 0x100000;
+constexpr uint64_t PTHREAD_STACK_TOP       = 0x7efff8000ull;
+constexpr uint64_t PTHREAD_STACK_BOTTOM    = 0x0000040000ull;
+constexpr uint32_t SIGNAL_APC_POLL_MICROS  = 10000;
 
 static constexpr KernelClockid KERNEL_CLOCK_REALTIME          = 0;
 static constexpr KernelClockid KERNEL_CLOCK_VIRTUAL           = 1;
@@ -697,16 +697,17 @@ static std::atomic<int32_t> g_pthread_thread_id      = 0;
 
 static Common::Mutex g_guest_stack_mutex;
 static uint64_t      g_guest_stack_last = 0;
+struct CachedGuestStack {
+	uint64_t address;
+	size_t   map_size;
+	size_t   guard_size;
+};
+static std::vector<CachedGuestStack> g_guest_stack_cache;
 
 static size_t RoundStackSize(size_t size) {
 	return ((size + PTHREAD_STACK_PAGE - 1) / PTHREAD_STACK_PAGE) * PTHREAD_STACK_PAGE;
 }
 
-static size_t RoundStackMappingSize(size_t size) {
-	return ((size + PTHREAD_STACK_GRANULARITY - 1) / PTHREAD_STACK_GRANULARITY) *
-	       PTHREAD_STACK_GRANULARITY;
-}
-
 static int CreateGuestStack(PthreadAttr attr) {
 	if (attr == nullptr) {
 		return KERNEL_ERROR_EINVAL;
@@ -722,34 +723,41 @@ static int CreateGuestStack(PthreadAttr attr) {
 
 	const auto stack_size = RoundStackSize(attr->stack_size);
 	const auto guard_size = RoundStackSize(attr->guard_size);
-	const auto map_size   = RoundStackMappingSize(stack_size + guard_size);
+	const auto map_size   = stack_size + guard_size;
 
 	uint64_t stack_addr = 0;
+	bool     cached     = false;
 	{
 		Common::LockGuard lock(g_guest_stack_mutex);
 
-		if (g_guest_stack_last == 0) {
-			g_guest_stack_last = (PTHREAD_STACK_TOP - PTHREAD_STACK_INITIAL - PTHREAD_STACK_PAGE) &
-			                     ~(static_cast<uint64_t>(PTHREAD_STACK_GRANULARITY) - 1);
+		auto cached_stack =
+		    std::find_if(g_guest_stack_cache.begin(), g_guest_stack_cache.end(),
+		                 [map_size, guard_size](const auto& stack) {
+			                 return stack.map_size == map_size && stack.guard_size == guard_size;
+		                 });
+		if (cached_stack != g_guest_stack_cache.end()) {
+			stack_addr = cached_stack->address;
+			g_guest_stack_cache.erase(cached_stack);
+			cached = true;
+		} else {
+			if (g_guest_stack_last == 0) {
+				g_guest_stack_last = PTHREAD_STACK_TOP - PTHREAD_STACK_INITIAL - PTHREAD_STACK_PAGE;
+			}
+			if (map_size > g_guest_stack_last - PTHREAD_STACK_BOTTOM) {
+				return KERNEL_ERROR_EAGAIN;
+			}
+			stack_addr = g_guest_stack_last - map_size;
+			g_guest_stack_last -= map_size;
 		}
-
-		stack_addr = g_guest_stack_last - map_size;
-		g_guest_stack_last -= map_size;
 	}
 
-	void* mapped_addr = reinterpret_cast<void*>(stack_addr);
-
-	constexpr int GUEST_PROT_READ_WRITE = 0x03;
-	constexpr int GUEST_MAP_PRIVATE     = 0x02;
-	constexpr int GUEST_MAP_FIXED       = 0x10;
-	constexpr int GUEST_MAP_STACK       = 0x400;
-	constexpr int GUEST_MAP_ANON        = 0x1000;
-
-	int result = Memory::KernelMapNamedFlexibleMemory(
-	    &mapped_addr, map_size, GUEST_PROT_READ_WRITE,
-	    GUEST_MAP_PRIVATE | GUEST_MAP_FIXED | GUEST_MAP_STACK | GUEST_MAP_ANON, "stack");
-	if (result != OK) {
-		return KERNEL_ERROR_EAGAIN;
+	int result = OK;
+	if (!cached) {
+		stack_addr = Memory::AllocateGuestStackMemory(
+		    stack_addr, map_size, Common::VirtualMemory::Mode::ReadWrite, "stack");
+		if (stack_addr == 0) {
+			return KERNEL_ERROR_EAGAIN;
+		}
 	}
 
 	if (guard_size != 0) {
@@ -761,7 +769,7 @@ static int CreateGuestStack(PthreadAttr attr) {
 	}
 
 	attr->stack_addr     = reinterpret_cast<void*>(stack_addr + guard_size);
-	attr->stack_size     = map_size - guard_size;
+	attr->stack_size     = stack_size;
 	attr->stack_user     = false;
 	attr->stack_map_addr = stack_addr;
 	attr->stack_map_size = map_size;
@@ -777,13 +785,90 @@ static void FreeGuestStack(PthreadAttr attr) {
 		return;
 	}
 
-	Memory::KernelMunmap(attr->stack_map_addr, attr->stack_map_size);
+	const auto guard_size = attr->stack_map_size - attr->stack_size;
+	{
+		Common::LockGuard lock(g_guest_stack_mutex);
+		g_guest_stack_cache.push_back({attr->stack_map_addr, attr->stack_map_size, guard_size});
+	}
 
 	attr->stack_addr     = nullptr;
 	attr->stack_map_addr = 0;
 	attr->stack_map_size = 0;
 }
 
+#if defined(KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS)
+bool TestGuestStackOwnerLifecycle(uint64_t* first_address, uint64_t* second_address,
+                                  uint64_t* map_size) {
+	if (first_address == nullptr || second_address == nullptr || map_size == nullptr) {
+		return false;
+	}
+
+	size_t flexible_before = 0;
+	if (Memory::KernelAvailableFlexibleMemorySize(&flexible_before) != OK) {
+		return false;
+	}
+
+	PthreadAttr attr = nullptr;
+	if (PthreadAttrInit(&attr) != OK) {
+		return false;
+	}
+	if (CreateGuestStack(attr) != OK) {
+		PthreadAttrDestroy(&attr);
+		return false;
+	}
+	*first_address = attr->stack_map_addr;
+	*map_size      = attr->stack_map_size;
+	const bool first_owned =
+	    Memory::TestGuestAddressRangeIsOwned(*first_address, static_cast<uint64_t>(*map_size));
+	uint64_t   backing_value = 0;
+	const bool first_private =
+	    !Memory::TryReadBacking(*first_address, &backing_value, sizeof(backing_value));
+	size_t     flexible_during_first = 0;
+	const bool first_capacity_unchanged =
+	    Memory::KernelAvailableFlexibleMemorySize(&flexible_during_first) == OK &&
+	    flexible_during_first == flexible_before;
+	FreeGuestStack(attr);
+
+	if (CreateGuestStack(attr) != OK) {
+		PthreadAttrDestroy(&attr);
+		return false;
+	}
+	*second_address = attr->stack_map_addr;
+	const bool second_owned =
+	    Memory::TestGuestAddressRangeIsOwned(*second_address, static_cast<uint64_t>(*map_size));
+	const bool second_private =
+	    !Memory::TryReadBacking(*second_address, &backing_value, sizeof(backing_value));
+	size_t     flexible_during_second = 0;
+	const bool second_capacity_unchanged =
+	    Memory::KernelAvailableFlexibleMemorySize(&flexible_during_second) == OK &&
+	    flexible_during_second == flexible_before;
+	FreeGuestStack(attr);
+
+	CachedGuestStack cached {};
+	bool             found = false;
+	{
+		Common::LockGuard lock(g_guest_stack_mutex);
+		const auto        entry = std::find_if(
+		    g_guest_stack_cache.begin(), g_guest_stack_cache.end(),
+		    [second_address](const auto& stack) { return stack.address == *second_address; });
+		if (entry != g_guest_stack_cache.end()) {
+			cached = *entry;
+			g_guest_stack_cache.erase(entry);
+			found = true;
+		}
+	}
+
+	const bool unmapped = found && Memory::KernelMunmap(cached.address, cached.map_size) == OK;
+	size_t     flexible_after = 0;
+	const bool final_capacity_unchanged =
+	    Memory::KernelAvailableFlexibleMemorySize(&flexible_after) == OK &&
+	    flexible_after == flexible_before;
+	return PthreadAttrDestroy(&attr) == OK && first_owned && first_private &&
+	       first_capacity_unchanged && second_owned && second_private &&
+	       second_capacity_unchanged && unmapped && final_capacity_unchanged;
+}
+#endif
+
 static KYTY_SYSV_ABI void* RunOnGuestStack(void* arg, pthread_entry_func_t func, void* stack_top) {
 #if defined(__x86_64__) || defined(_M_X64)
 	void*      ret       = nullptr;

src/kernel/pthread.h

@@ -112,11 +112,15 @@ void                  PthreadQueuePendingSignal(Pthread thread, int signum);
 bool                  PthreadHasPendingSignal(Pthread thread, int signum);
 bool                  PthreadTakePendingSignal(Pthread thread, int signum);
 bool PthreadGetGuestStack(Pthread thread, uint64_t* stack_addr, uint64_t* stack_size);
+#if defined(KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS)
+bool TestGuestStackOwnerLifecycle(uint64_t* first_address, uint64_t* second_address,
+                                  uint64_t* map_size);
+#endif
 #if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS
 bool PthreadKillHost(Pthread thread, int host_signal);
 #endif
-int  PthreadGetPriorityForKernel(Pthread thread);
-int  PthreadGetCurrentPriorityForKernel();
+int PthreadGetPriorityForKernel(Pthread thread);
+int PthreadGetCurrentPriorityForKernel();
 
 int KYTY_SYSV_ABI          KernelUsleep(KernelUseconds microseconds);
 unsigned int KYTY_SYSV_ABI KernelSleep(unsigned int seconds);

src/libs/libKernel.cpp

@@ -40,7 +40,10 @@
 #define NOMINMAX
 #endif
 #include <windows.h>
-#elif !defined(__APPLE__)
+#elif defined(__APPLE__)
+#include <csignal>
+#include <sys/ucontext.h>
+#else
 #include <csignal>
 #include <ucontext.h>
 #endif
@@ -837,7 +840,7 @@ static void ApplySignalUcontext(CONTEXT* dst_ctx, const SignalUcontext& src_ctx)
 }
 #endif
 
-#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS && !defined(__APPLE__) && defined(__x86_64__)
+#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS && defined(__x86_64__)
 
 static SignalUcontext CreateSignalUcontextFromHost(const ucontext_t* host_ctx) {
 	SignalUcontext ctx = {};
@@ -845,6 +848,34 @@ static SignalUcontext CreateSignalUcontextFromHost(const ucontext_t* host_ctx) {
 		return ctx;
 	}
 
+#if defined(__APPLE__)
+	const auto& ss = host_ctx->uc_mcontext->__ss;
+
+	ctx.uc_mcontext.mc_rdi    = ss.__rdi;
+	ctx.uc_mcontext.mc_rsi    = ss.__rsi;
+	ctx.uc_mcontext.mc_rdx    = ss.__rdx;
+	ctx.uc_mcontext.mc_rcx    = ss.__rcx;
+	ctx.uc_mcontext.mc_r8     = ss.__r8;
+	ctx.uc_mcontext.mc_r9     = ss.__r9;
+	ctx.uc_mcontext.mc_rax    = ss.__rax;
+	ctx.uc_mcontext.mc_rbx    = ss.__rbx;
+	ctx.uc_mcontext.mc_rbp    = ss.__rbp;
+	ctx.uc_mcontext.mc_r10    = ss.__r10;
+	ctx.uc_mcontext.mc_r11    = ss.__r11;
+	ctx.uc_mcontext.mc_r12    = ss.__r12;
+	ctx.uc_mcontext.mc_r13    = ss.__r13;
+	ctx.uc_mcontext.mc_r14    = ss.__r14;
+	ctx.uc_mcontext.mc_r15    = ss.__r15;
+	ctx.uc_mcontext.mc_rip    = ss.__rip;
+	ctx.uc_mcontext.mc_rsp    = ss.__rsp;
+	ctx.uc_mcontext.mc_rflags = ss.__rflags;
+	ctx.uc_mcontext.mc_cs     = ss.__cs & 0xffffu;
+	ctx.uc_mcontext.mc_gs     = static_cast<uint16_t>(ss.__gs & 0xffffu);
+	ctx.uc_mcontext.mc_fs     = static_cast<uint16_t>(ss.__fs & 0xffffu);
+	ctx.uc_mcontext.mc_len    = sizeof(SignalMcontext);
+
+	return ctx;
+#else
 	const auto* gregs = host_ctx->uc_mcontext.gregs;
 
 	ctx.uc_mcontext.mc_rdi    = static_cast<uint64_t>(gregs[REG_RDI]);
@@ -874,6 +905,7 @@ static SignalUcontext CreateSignalUcontextFromHost(const ucontext_t* host_ctx) {
 	ctx.uc_mcontext.mc_len = sizeof(SignalMcontext);
 
 	return ctx;
+#endif
 }
 
 static void ApplySignalUcontextToHost(ucontext_t* dst_ctx, const SignalUcontext& src_ctx) {
@@ -881,6 +913,29 @@ static void ApplySignalUcontextToHost(ucontext_t* dst_ctx, const SignalUcontext&
 		return;
 	}
 
+#if defined(__APPLE__)
+	auto& ss = dst_ctx->uc_mcontext->__ss;
+
+	ss.__rdi    = src_ctx.uc_mcontext.mc_rdi;
+	ss.__rsi    = src_ctx.uc_mcontext.mc_rsi;
+	ss.__rdx    = src_ctx.uc_mcontext.mc_rdx;
+	ss.__rcx    = src_ctx.uc_mcontext.mc_rcx;
+	ss.__r8     = src_ctx.uc_mcontext.mc_r8;
+	ss.__r9     = src_ctx.uc_mcontext.mc_r9;
+	ss.__rax    = src_ctx.uc_mcontext.mc_rax;
+	ss.__rbx    = src_ctx.uc_mcontext.mc_rbx;
+	ss.__rbp    = src_ctx.uc_mcontext.mc_rbp;
+	ss.__r10    = src_ctx.uc_mcontext.mc_r10;
+	ss.__r11    = src_ctx.uc_mcontext.mc_r11;
+	ss.__r12    = src_ctx.uc_mcontext.mc_r12;
+	ss.__r13    = src_ctx.uc_mcontext.mc_r13;
+	ss.__r14    = src_ctx.uc_mcontext.mc_r14;
+	ss.__r15    = src_ctx.uc_mcontext.mc_r15;
+	ss.__rip    = src_ctx.uc_mcontext.mc_rip;
+	ss.__rsp    = src_ctx.uc_mcontext.mc_rsp;
+	ss.__rflags = src_ctx.uc_mcontext.mc_rflags;
+	// Segment selectors are left untouched; XNU validates them on sigreturn.
+#else
 	auto* gregs = dst_ctx->uc_mcontext.gregs;
 
 	gregs[REG_RDI] = static_cast<greg_t>(src_ctx.uc_mcontext.mc_rdi);
@@ -903,10 +958,17 @@ static void ApplySignalUcontextToHost(ucontext_t* dst_ctx, const SignalUcontext&
 	gregs[REG_EFL] = static_cast<greg_t>(src_ctx.uc_mcontext.mc_rflags);
 
 	// The kernel validates packed segment selectors on sigreturn.
+#endif
 }
 
 static int SignalDispatchHostSignal() {
+#if defined(__APPLE__)
+	// macOS has no realtime signals; SIGUSR1 is otherwise unused on the host side (the
+	// guest's SIGUSR1 is an emulated signal number, not a host registration).
+	static const int host_signal = SIGUSR1;
+#else
 	static const int host_signal = SIGRTMIN + 3;
+#endif
 	return host_signal;
 }
 
@@ -1174,7 +1236,7 @@ static int KYTY_SYSV_ABI KernelRaiseException(Pthread thread, int signum) {
 		}
 		CloseHandle(target_thread);
 		return OK;
-#elif !defined(__APPLE__) && defined(__x86_64__)
+#elif defined(__x86_64__)
 		// Deliver on the target thread.
 		if (thread == PthreadSelfOrNull()) {
 			SignalDispatchScope scope;

src/libs/libNet.cpp

@@ -34,7 +34,7 @@ namespace LibNet {
 
 LIB_VERSION("Net", 1, "Net", 1, 1);
 
-static thread_local int g_net_errno = 0;
+static thread_local int   g_net_errno = 0;
 static constexpr uint32_t g_in6addr_any[4] {};
 
 namespace Net = Network::Net;
@@ -654,6 +654,8 @@ namespace LibHttp2 {
 LIB_VERSION("Http2", 1, "Http2", 1, 1);
 
 constexpr int HTTP2_ERROR_INVALID_ID   = -2122641152; /* 0x817B1100 */
+constexpr int HTTP2_ERROR_BEFORE_SEND  = -2122641307; /* 0x817B1065 */
+constexpr int HTTP2_ERROR_TIMEOUT      = -2122641304; /* 0x817B1068 */
 constexpr int HTTP2_ERROR_NULL_POINTER = -2122640859; /* 0x817B1225 */
 
 struct Http2Options {
@@ -694,14 +696,14 @@ struct Http2Request {
 	std::string                                      url;
 	uint64_t                                         content_length = 0;
 	std::vector<std::pair<std::string, std::string>> headers;
-	bool                                             sent        = false;
-	int                                              status_code = 204;
-	std::string  response_headers                                = "HTTP/2 204 No Content\r\n\r\n";
-	std::string  response_body;
-	size_t       read_offset  = 0;
-	int          async_result = 0;
-	int          async_event  = 0;
-	Http2Options options;
+	int                                              send_result = HTTP2_ERROR_BEFORE_SEND;
+	int                                              status_code = 0;
+	std::string                                      response_headers;
+	std::string                                      response_body;
+	size_t                                           read_offset  = 0;
+	int                                              async_result = HTTP2_ERROR_BEFORE_SEND;
+	int                                              async_event  = 0;
+	Http2Options                                     options;
 };
 
 struct Http2AsyncResult {
@@ -1114,9 +1116,9 @@ static int KYTY_SYSV_ABI Http2SendRequest(int req_id, const void* post_data, siz
 		return HTTP2_ERROR_INVALID_ID;
 	}
 
-	request->second.sent = true;
+	request->second.send_result = HTTP2_ERROR_TIMEOUT;
 
-	return 0;
+	return request->second.send_result;
 }
 
 static int KYTY_SYSV_ABI Http2SendRequestAsync(int req_id, const void* post_data, size_t size,
@@ -1136,8 +1138,8 @@ static int KYTY_SYSV_ABI Http2SendRequestAsync(int req_id, const void* post_data
 		return HTTP2_ERROR_INVALID_ID;
 	}
 
-	request->second.sent         = true;
-	request->second.async_result = 0;
+	request->second.send_result  = HTTP2_ERROR_TIMEOUT;
+	request->second.async_result = request->second.send_result;
 	request->second.async_event  = 0;
 
 	return 0;
@@ -1159,11 +1161,10 @@ static int KYTY_SYSV_ABI Http2WaitAsync(int req_id, Http2AsyncResult* result, ui
 		return HTTP2_ERROR_INVALID_ID;
 	}
 
-	request->second.sent = true;
-	*result              = {};
-	result->event_type   = request->second.async_event;
-	result->req_id       = req_id;
-	result->result       = request->second.async_result;
+	*result            = {};
+	result->event_type = request->second.async_event;
+	result->req_id     = req_id;
+	result->result     = request->second.async_result;
 
 	return 0;
 }
@@ -1178,13 +1179,19 @@ static int KYTY_SYSV_ABI Http2GetStatusCode(int req_id, int* status_code) {
 		return HTTP2_ERROR_NULL_POINTER;
 	}
 
+	*status_code = 0;
+
 	auto request = g_http2_requests.find(req_id);
 	if (request == g_http2_requests.end()) {
 		return HTTP2_ERROR_INVALID_ID;
 	}
 
-	*status_code = request->second.status_code;
+	const int send_result = request->second.send_result;
+	if (send_result != 0) {
+		return send_result;
+	}
 
+	*status_code = request->second.status_code;
 	return 0;
 }
 
@@ -1200,14 +1207,21 @@ static int KYTY_SYSV_ABI Http2GetResponseContentLength(int req_id, int* result,
 		return HTTP2_ERROR_NULL_POINTER;
 	}
 
+	*result         = 0;
+	*content_length = 0;
+
 	auto request = g_http2_requests.find(req_id);
 	if (request == g_http2_requests.end()) {
 		return HTTP2_ERROR_INVALID_ID;
 	}
 
-	*result         = 0; // SCE_HTTP2_CONTENTLEN_EXIST
-	*content_length = request->second.response_body.size();
+	const int send_result = request->second.send_result;
+	if (send_result != 0) {
+		*result = -1;
+		return send_result;
+	}
 
+	*content_length = request->second.response_body.size();
 	return 0;
 }
 
@@ -1223,14 +1237,21 @@ static int KYTY_SYSV_ABI Http2GetAllResponseHeaders(int req_id, char** header,
 		return HTTP2_ERROR_NULL_POINTER;
 	}
 
+	*header      = nullptr;
+	*header_size = 0;
+
 	auto request = g_http2_requests.find(req_id);
 	if (request == g_http2_requests.end()) {
 		return HTTP2_ERROR_INVALID_ID;
 	}
 
+	const int send_result = request->second.send_result;
+	if (send_result != 0) {
+		return send_result;
+	}
+
 	*header      = const_cast<char*>(request->second.response_headers.c_str());
 	*header_size = request->second.response_headers.size();
-
 	return 0;
 }
 
@@ -1250,6 +1271,11 @@ static int KYTY_SYSV_ABI Http2ReadData(int req_id, void* data, size_t size) {
 		return HTTP2_ERROR_INVALID_ID;
 	}
 
+	const int send_result = request->second.send_result;
+	if (send_result != 0) {
+		return send_result;
+	}
+
 	const auto& body = request->second.response_body;
 	const auto  remaining =
 	    request->second.read_offset < body.size() ? body.size() - request->second.read_offset : 0;
@@ -1280,6 +1306,13 @@ static int KYTY_SYSV_ABI Http2ReadDataAsync(int req_id, void* data, size_t size,
 		return HTTP2_ERROR_INVALID_ID;
 	}
 
+	const int send_result = request->second.send_result;
+	if (send_result != 0) {
+		request->second.async_result = send_result;
+		request->second.async_event  = 1;
+		return 0;
+	}
+
 	const auto& body = request->second.response_body;
 	const auto  remaining =
 	    request->second.read_offset < body.size() ? body.size() - request->second.read_offset : 0;
@@ -1398,6 +1431,7 @@ LIB_DEFINE(InitNet_1_NpManager) {
 	LIB_FUNC("O80NrhUOPGY", NpManager::NpCheckPremium);
 	LIB_FUNC("eQH7nWPcAgc", NpManager::NpGetState);
 	LIB_FUNC("e-ZuhGEoeC4", NpManager::NpGetNpReachabilityState);
+	LIB_FUNC("Oad3rvY-NJQ", NpManager::NpHasSignedUp);
 }
 
 } // namespace LibNpManager

src/libs/libTextToSpeech2.cpp

@@ -0,0 +1,31 @@
+#include "common/abi.h"
+#include "libs/errno.h"
+#include "libs/libs.h"
+#include "loader/symbolDatabase.h"
+
+namespace Libs {
+
+LIB_VERSION("TextToSpeech2", 1, "TextToSpeech2", 1, 1);
+
+namespace TextToSpeech2 {
+
+static int KYTY_SYSV_ABI TextToSpeech2GetSpeechStatus() {
+	PRINT_NAME();
+
+	return OK;
+}
+
+static int KYTY_SYSV_ABI TextToSpeech2Cancel() {
+	PRINT_NAME();
+
+	return OK;
+}
+
+} // namespace TextToSpeech2
+
+LIB_DEFINE(InitTextToSpeech2_1) {
+	LIB_FUNC("08JSg9p6bgQ", TextToSpeech2::TextToSpeech2GetSpeechStatus);
+	LIB_FUNC("2jiIxUmcsGo", TextToSpeech2::TextToSpeech2Cancel);
+}
+
+} // namespace Libs

src/libs/libs.cpp

@@ -66,6 +66,7 @@ LIB_DEFINE(InitSaveData_1);
 LIB_DEFINE(InitShare_1);
 LIB_DEFINE(InitSysmodule_1);
 LIB_DEFINE(InitSystemService_1);
+LIB_DEFINE(InitTextToSpeech2_1);
 LIB_DEFINE(InitUserService_1);
 LIB_DEFINE(InitVideoOut_1);
 
@@ -100,6 +101,7 @@ void InitAll(Loader::SymbolDatabase* s) {
 	LIB_LOAD(InitShare_1);
 	LIB_LOAD(InitSysmodule_1);
 	LIB_LOAD(InitSystemService_1);
+	LIB_LOAD(InitTextToSpeech2_1);
 	LIB_LOAD(LibUlt::InitUlt_1);
 	LIB_LOAD(InitUserService_1);
 	LIB_LOAD(VideoDec2::InitVideoDec2_1);

src/libs/network.cpp

@@ -19,7 +19,7 @@
 
 // POSIX uses plain int file descriptors for sockets; provide the Winsock spellings
 // the shared (non-guarded) code paths reference.
-using SOCKET                   = int;
+using SOCKET                           = int;
 static constexpr SOCKET INVALID_SOCKET = -1;
 #endif
 
@@ -820,10 +820,10 @@ struct NetEtherAddr {
 };
 
 #if defined(_WIN32)
-using NativeSocket = SOCKET;
+using NativeSocket                                  = SOCKET;
 static constexpr NativeSocket INVALID_NATIVE_SOCKET = INVALID_SOCKET;
 #else
-using NativeSocket = int;
+using NativeSocket                                  = int;
 static constexpr NativeSocket INVALID_NATIVE_SOCKET = -1;
 #endif
 
@@ -1738,7 +1738,8 @@ int KYTY_SYSV_ABI Accept(int s, void* addr, uint32_t* addrlen) {
 #if defined(_WIN32)
 	sockaddr_storage host_addr {};
 	int              host_addrlen = sizeof(host_addr);
-	NativeSocket accepted = ::accept(socket, reinterpret_cast<sockaddr*>(&host_addr), &host_addrlen);
+	NativeSocket     accepted =
+	    ::accept(socket, reinterpret_cast<sockaddr*>(&host_addr), &host_addrlen);
 	if (accepted == INVALID_NATIVE_SOCKET) {
 		return SetPosixSocketError();
 	}
@@ -3753,8 +3754,6 @@ int KYTY_SYSV_ABI NpGetState(int user_id, uint32_t* state) {
 int KYTY_SYSV_ABI NpGetNpReachabilityState(int user_id, uint32_t* state) {
 	PRINT_NAME();
 
-	constexpr int np_error_invalid_argument = -2141913085; /* 0x80550003 */
-
 	if (state == nullptr) {
 		return np_error_invalid_argument;
 	}
@@ -3767,6 +3766,20 @@ int KYTY_SYSV_ABI NpGetNpReachabilityState(int user_id, uint32_t* state) {
 	return OK;
 }
 
+int KYTY_SYSV_ABI NpHasSignedUp(int user_id, bool* has_signed_up) {
+	PRINT_NAME();
+
+	if (has_signed_up == nullptr) {
+		return np_error_invalid_argument;
+	}
+
+	LOGF("\t user_id = %d\n", user_id);
+
+	*has_signed_up = false;
+
+	return OK;
+}
+
 } // namespace NpManager
 
 } // namespace Libs::Network

src/libs/network.h

@@ -184,6 +184,7 @@ int KYTY_SYSV_ABI  NpCheckPremium(int req_id, const NpCheckPremiumParameter* par
                                   NpCheckPremiumResult* result);
 int KYTY_SYSV_ABI  NpGetState(int user_id, uint32_t* state);
 int KYTY_SYSV_ABI  NpGetNpReachabilityState(int user_id, uint32_t* state);
+int KYTY_SYSV_ABI  NpHasSignedUp(int user_id, bool* has_signed_up);
 
 } // namespace NpManager
 

src/loader/gamePatch.cpp

@@ -2,6 +2,7 @@
 
 #include "common/stringUtils.h"
 #include "common/virtualMemory.h"
+#include "kernel/memory.h"
 #include "loader/elf.h"
 #include "loader/runtimeLinker.h"
 #include "loader/systemContent.h"
@@ -153,18 +154,6 @@ bool ValidateTarget(const Plan& plan, const Program* program, std::string* error
 	return true;
 }
 
-Common::VirtualMemory::Mode ReadableMode(Elf64_Word flags) {
-	const bool executable = (flags & PF_X) != 0;
-	const bool writable   = (flags & PF_W) != 0;
-	if (executable && writable) {
-		return Common::VirtualMemory::Mode::ExecuteReadWrite;
-	}
-	if (executable) {
-		return Common::VirtualMemory::Mode::ExecuteRead;
-	}
-	return writable ? Common::VirtualMemory::Mode::ReadWrite : Common::VirtualMemory::Mode::Read;
-}
-
 bool ResolveWrite(const Program& program, Write* write, std::string* error) {
 	const auto* ehdr = program.elf->GetEhdr();
 	const auto* phdr = program.elf->GetPhdr();
@@ -178,16 +167,9 @@ bool ResolveWrite(const Program& program, Write* write, std::string* error) {
 			continue;
 		}
 
-		const auto                  segment_address = program.base_vaddr + segment.p_vaddr;
-		const bool                  add_read        = (segment.p_flags & PF_R) == 0;
-		Common::VirtualMemory::Mode old_mode {};
-		if (add_read && !Common::VirtualMemory::Protect(segment_address, segment.p_memsz,
-		                                                ReadableMode(segment.p_flags), &old_mode)) {
-			return Fail(error, "could not read a loaded executable segment");
-		}
-
-		const auto* begin = reinterpret_cast<const uint8_t*>(segment_address);
-		const auto* end   = begin + segment.p_filesz;
+		const auto  segment_address = program.base_vaddr + segment.p_vaddr;
+		const auto* begin           = reinterpret_cast<const uint8_t*>(segment_address);
+		const auto* end             = begin + segment.p_filesz;
 		for (auto* current = begin; current < end;) {
 			const auto* found =
 			    std::search(current, end, write->expected.begin(), write->expected.end());
@@ -202,15 +184,9 @@ bool ResolveWrite(const Program& program, Write* write, std::string* error) {
 			match_count++;
 			current = found + 1;
 		}
-
-		if (add_read &&
-		    !Common::VirtualMemory::Protect(segment_address, segment.p_memsz, old_mode)) {
-			return Fail(error, "could not restore executable segment protection");
-		}
 	}
 
-	::printf("Game patch: found %zu entries for '%s'\n", match_count,
-	         write->patch_name.c_str());
+	::printf("Game patch: found %zu entries for '%s'\n", match_count, write->patch_name.c_str());
 	if (match == 0) {
 		return Fail(error, "original bytes not found for '" + write->patch_name + "'");
 	}
@@ -229,16 +205,9 @@ bool PrepareWrites(Plan* plan, const Program& program, std::string* error) {
 
 bool ApplyWrites(Plan* plan, std::string* error) {
 	for (auto& write: plan->writes) {
-		Common::VirtualMemory::Mode old_mode {};
-		const auto                  size = write.replacement.size();
-		if (!Common::VirtualMemory::Protect(
-		        write.address, size, Common::VirtualMemory::Mode::ExecuteReadWrite, &old_mode)) {
-			return Fail(error, "could not make patch memory writable");
-		}
-
+		const auto size = write.replacement.size();
 		std::memcpy(reinterpret_cast<void*>(write.address), write.replacement.data(), size);
-		if (!Common::VirtualMemory::Protect(write.address, size, old_mode) ||
-		    !Common::VirtualMemory::FlushInstructionCache(write.address, size)) {
+		if (!Common::VirtualMemory::FlushInstructionCache(write.address, size)) {
 			return Fail(error, "could not finalize patch");
 		}
 	}

src/loader/runtimeLinker.cpp

@@ -62,14 +62,16 @@ static void FreeTlsBlock(ThreadLocalStorage::Block* block) {
 	if (block->free_func != nullptr) {
 		block->free_func(block->ptr);
 	} else if (block->vm_alloc) {
-		Common::VirtualMemory::Free(reinterpret_cast<uint64_t>(block->ptr));
+		EXIT_IF(!Libs::LibKernel::Memory::FreeGuestMemory(reinterpret_cast<uint64_t>(block->ptr),
+		                                                  block->alloc_size));
 	} else {
 		delete[] block->ptr;
 	}
 
-	block->ptr       = nullptr;
-	block->free_func = nullptr;
-	block->vm_alloc  = false;
+	block->ptr        = nullptr;
+	block->free_func  = nullptr;
+	block->vm_alloc   = false;
+	block->alloc_size = 0;
 }
 
 static uint64_t AlignUp(uint64_t value, uint64_t alignment) {
@@ -131,17 +133,25 @@ static std::vector<StubbedImportRecord> g_stubbed_imports;
 static std::atomic_uint32_t             g_unresolved_stub_call_log_count {0};
 static std::vector<uint64_t>            g_unresolved_stub_thunk_pages;
 static uint64_t                         g_unresolved_stub_thunk_offset = 0;
+static constexpr uint64_t               UNRESOLVED_STUB_PAGE_SIZE      = 4096;
 
 static KYTY_SYSV_ABI uint64_t ResolveImportStubWithId(uint64_t record_id);
 
+static bool PatchGuestMemory64(uint64_t vaddr, uint64_t value) {
+	auto* ptr     = reinterpret_cast<uint64_t*>(vaddr);
+	bool  changed = (*ptr != value);
+	std::memcpy(ptr, &value, sizeof(value));
+	return changed;
+}
+
 static uint64_t AllocateUnresolvedImportThunk(uint64_t record_id) {
-	constexpr uint64_t page_size  = 4096;
 	constexpr uint64_t thunk_size = 162;
 
 	if (g_unresolved_stub_thunk_pages.empty() ||
-	    g_unresolved_stub_thunk_offset + thunk_size > page_size) {
-		auto page = Common::VirtualMemory::Alloc(0, page_size,
-		                                         Common::VirtualMemory::Mode::ExecuteReadWrite);
+	    g_unresolved_stub_thunk_offset + thunk_size > UNRESOLVED_STUB_PAGE_SIZE) {
+		auto page = Libs::LibKernel::Memory::AllocateRuntimeMemory(
+		    0, UNRESOLVED_STUB_PAGE_SIZE, Common::VirtualMemory::Mode::ExecuteReadWrite,
+		    "unresolved_import_thunk");
 		EXIT_NOT_IMPLEMENTED(page == 0);
 		g_unresolved_stub_thunk_pages.push_back(page);
 		g_unresolved_stub_thunk_offset = 0;
@@ -298,7 +308,7 @@ static KYTY_SYSV_ABI uint64_t ResolveImportStubWithId(uint64_t record_id) {
 			     resolved.name.c_str(), resolved.vaddr);
 
 			if (record.patch_vaddr != 0) {
-				*reinterpret_cast<uint64_t*>(record.patch_vaddr) = resolved.vaddr;
+				PatchGuestMemory64(record.patch_vaddr, resolved.vaddr);
 			}
 
 			return resolved.vaddr;
@@ -360,7 +370,7 @@ static KYTY_SYSV_ABI void RunEntry(uint64_t addr, EntryParams* params, atexit_fu
 		register uintptr_t    guest_rbp_reg asm("r15") = guest_rbp;
 #endif
 
-#if defined(__APPLE__) || KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
+#if defined(__APPLE__)
 		asm volatile(
 		    "pushq %%r12\n\t"
 		    "pushq %%r13\n\t"
@@ -374,16 +384,44 @@ static KYTY_SYSV_ABI void RunEntry(uint64_t addr, EntryParams* params, atexit_fu
 		    "popq %%r13\n\t"
 		    "popq %%r12\n\t"
 		    :
-#if defined(__APPLE__)
 		    : [func] "r"(func_reg), "D"(params),
 		      "S"(atexit_func), [guest_rsp] "r"(guest_rsp_reg), [guest_rbp] "r"(guest_rbp_reg)
-#else
-		    : [func] "r"(func), "D"(params),
-		      "S"(atexit_func), [guest_rsp] "r"(guest_rsp), [guest_rbp] "r"(guest_rbp)
-#endif
 		    : "cc", "memory", "rax", "rcx", "rdx", "r8", "r9", "r10", "r11", "xmm0", "xmm1", "xmm2",
 		      "xmm3", "xmm4", "xmm5", "xmm6", "xmm7", "xmm8", "xmm9", "xmm10", "xmm11", "xmm12",
 		      "xmm13", "xmm14", "xmm15");
+#elif KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
+		// Windows stack probes use the TEB stack limits during the guest stack switch.
+		// bounds, which describe the host stack and are invalid while RSP is in guest memory.
+		register entry_func_t func_reg asm("rbx")     = func;
+		register uintptr_t    guest_rsp_reg asm("r8") = guest_rsp;
+		register uintptr_t    guest_rbp_reg asm("r9") = guest_rbp;
+		asm volatile("pushq %%r12\n\t"
+		             "pushq %%r13\n\t"
+		             "pushq %%r14\n\t"
+		             "pushq %%r15\n\t"
+		             "movq %%gs:0x08, %%r14\n\t"
+		             "movq %%gs:0x10, %%r15\n\t"
+		             "xorq %%rcx, %%rcx\n\t"
+		             "movq %%rcx, %%gs:0x08\n\t"
+		             "movq %%rcx, %%gs:0x10\n\t"
+		             "movq %%rsp, %%r12\n\t"
+		             "movq %%rbp, %%r13\n\t"
+		             "movq %[guest_rsp], %%rsp\n\t"
+		             "movq %[guest_rbp], %%rbp\n\t"
+		             "callq *%[func]\n\t"
+		             "movq %%r13, %%rbp\n\t"
+		             "movq %%r12, %%rsp\n\t"
+		             "movq %%r14, %%gs:0x08\n\t"
+		             "movq %%r15, %%gs:0x10\n\t"
+		             "popq %%r15\n\t"
+		             "popq %%r14\n\t"
+		             "popq %%r13\n\t"
+		             "popq %%r12\n\t"
+		             : [guest_rsp] "+r"(guest_rsp_reg), [guest_rbp] "+r"(guest_rbp_reg)
+		             : [func] "r"(func_reg), "D"(params), "S"(atexit_func)
+		             : "cc", "memory", "rax", "rcx", "rdx", "r10", "r11", "xmm0", "xmm1", "xmm2",
+		               "xmm3", "xmm4", "xmm5", "xmm6", "xmm7", "xmm8", "xmm9", "xmm10", "xmm11",
+		               "xmm12", "xmm13", "xmm14", "xmm15");
 #else
 		// Clobbers prevent inputs from being allocated to r12/r13.
 		asm volatile("movq %%rsp, %%r12\n\t"
@@ -449,6 +487,110 @@ static KYTY_SYSV_ABI void RunEntry(uint64_t addr, EntryParams* params, atexit_fu
 #endif
 }
 
+#if defined(KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS)
+struct MainEntryStackTestState {
+	bool      called = false;
+	uintptr_t rsp    = 0;
+#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
+	uintptr_t teb_stack_base  = UINTPTR_MAX;
+	uintptr_t teb_stack_limit = UINTPTR_MAX;
+#endif
+};
+
+static KYTY_SYSV_ABI void TestMainEntryStackCallback(EntryParams* params,
+                                                     atexit_func_t /*atexit_func*/) {
+	auto* state = reinterpret_cast<MainEntryStackTestState*>(const_cast<char*>(params->argv[0]));
+	asm volatile("movq %%rsp, %0" : "=r"(state->rsp) : : "memory");
+#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
+	asm volatile("movq %%gs:0x08, %0\n\t"
+	             "movq %%gs:0x10, %1\n\t"
+	             : "=r"(state->teb_stack_base), "=r"(state->teb_stack_limit)
+	             :
+	             : "memory");
+#endif
+	state->called = true;
+}
+
+bool TestMainEntryUsesGuestStack() {
+	constexpr uint64_t stack_size = 0x10000;
+	const auto         stack_base = Libs::LibKernel::Memory::AllocateRuntimeMemory(
+	    0, stack_size, Common::VirtualMemory::Mode::ReadWrite, "main_entry_stack_test");
+	if (stack_base == 0) {
+		return false;
+	}
+
+	MainEntryStackTestState state {};
+	EntryParams             params {};
+	params.argv[0] = reinterpret_cast<const char*>(&state);
+
+#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
+	uintptr_t original_teb_stack_base  = 0;
+	uintptr_t original_teb_stack_limit = 0;
+	asm volatile("movq %%gs:0x08, %0\n\t"
+	             "movq %%gs:0x10, %1\n\t"
+	             : "=r"(original_teb_stack_base), "=r"(original_teb_stack_limit)
+	             :
+	             : "memory");
+#endif
+
+	RunEntry(reinterpret_cast<uint64_t>(TestMainEntryStackCallback), &params, nullptr,
+	         reinterpret_cast<void*>(stack_base + stack_size));
+
+#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
+	uintptr_t restored_teb_stack_base  = 0;
+	uintptr_t restored_teb_stack_limit = 0;
+	asm volatile("movq %%gs:0x08, %0\n\t"
+	             "movq %%gs:0x10, %1\n\t"
+	             : "=r"(restored_teb_stack_base), "=r"(restored_teb_stack_limit)
+	             :
+	             : "memory");
+	const bool teb_ok = state.teb_stack_base == 0 && state.teb_stack_limit == 0 &&
+	                    restored_teb_stack_base == original_teb_stack_base &&
+	                    restored_teb_stack_limit == original_teb_stack_limit;
+#else
+	constexpr bool teb_ok = true;
+#endif
+
+	const bool rsp_ok = state.rsp >= stack_base && state.rsp < stack_base + stack_size;
+	const bool freed  = Libs::LibKernel::Memory::FreeGuestMemory(stack_base, stack_size);
+	return state.called && rsp_ok && teb_ok && freed;
+}
+
+bool TestModuleRelocationUsesWritableHostMapping() {
+	constexpr uint64_t page_size = 0x4000;
+	constexpr uint64_t value     = 0x4b59545950415443;
+	const auto         base      = Libs::LibKernel::Memory::AllocateProgramMemory(
+	    0, page_size, Common::VirtualMemory::Mode::ReadWrite, "host_only_patch_test");
+	if (base == 0) {
+		return false;
+	}
+	Libs::LibKernel::Memory::SetProgramMemoryProtection(base, page_size,
+	                                                    Common::VirtualMemory::Mode::Read);
+
+	Libs::LibKernel::Memory::VirtualQueryInfo before {};
+	Libs::LibKernel::Memory::VirtualQueryInfo after {};
+	const bool                                before_ok =
+	    Libs::LibKernel::Memory::KernelVirtualQuery(reinterpret_cast<const void*>(base), 0, &before,
+	                                                sizeof(before)) == 0;
+	const bool changed  = PatchGuestMemory64(base, value);
+	const bool after_ok = Libs::LibKernel::Memory::KernelVirtualQuery(
+	                          reinterpret_cast<const void*>(base), 0, &after, sizeof(after)) == 0;
+	const bool value_ok = *reinterpret_cast<const uint64_t*>(base) == value;
+#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
+	MEMORY_BASIC_INFORMATION mbi {};
+	const bool               host_mode_ok =
+	    VirtualQuery(reinterpret_cast<const void*>(base), &mbi, sizeof(mbi)) != 0 &&
+	    mbi.Protect == PAGE_READWRITE;
+#else
+	constexpr bool host_mode_ok = true;
+#endif
+	const bool freed = Libs::LibKernel::Memory::FreeGuestMemory(base, page_size);
+
+	return before_ok && after_ok && changed && value_ok && host_mode_ok && freed &&
+	       before.protection == after.protection;
+}
+#endif
+
 static uint64_t GetAlignedSize(const Elf64_Phdr* p) {
 	return (p->p_align != 0 ? (p->p_memsz + (p->p_align - 1)) & ~(p->p_align - 1) : p->p_memsz);
 }
@@ -1038,7 +1180,7 @@ static void RelocateRecord(uint32_t index, Elf64_Rela* r, Program* program, bool
 	// KYTY_PROFILER_BLOCK("patch");
 
 	if (ri.resolved) {
-		patched = Common::VirtualMemory::PatchReplace(ri.vaddr, ri.value);
+		patched = PatchGuestMemory64(ri.vaddr, ri.value);
 	} else {
 		uint64_t value = 0;
 		bool     weak  = (ri.bind == BindType::Weak || !program->fail_if_global_not_resolved);
@@ -1056,7 +1198,7 @@ static void RelocateRecord(uint32_t index, Elf64_Rela* r, Program* program, bool
 		}
 
 		if (value != 0) {
-			patched = Common::VirtualMemory::PatchReplace(ri.vaddr, value);
+			patched = PatchGuestMemory64(ri.vaddr, value);
 		} else {
 			auto dbg_str = fmt::format("[{:016x}] <- {:016x}, {}, {}, {}, {}", ri.vaddr, ri.value,
 			                           ri.name.c_str(), Common::EnumName(ri.type).c_str(),
@@ -1079,7 +1221,7 @@ static void RelocateRecord(uint32_t index, Elf64_Rela* r, Program* program, bool
 			}
 
 			if (value != 0) {
-				patched = Common::VirtualMemory::PatchReplace(ri.vaddr, value);
+				patched = PatchGuestMemory64(ri.vaddr, value);
 			}
 		}
 	}
@@ -1459,6 +1601,7 @@ void RuntimeLinker::Execute(const std::filesystem::path& game_patch) {
 
 	PreloadAdjacentPrograms();
 	RelocateAll();
+
 	if (!game_patch.empty()) {
 		GamePatch::Apply(game_patch, m_programs.empty() ? nullptr : m_programs.front());
 	}
@@ -1489,6 +1632,21 @@ void RuntimeLinker::Clear() {
 		DeleteProgram(p);
 	}
 	m_programs.clear();
+	for (const auto page: g_unresolved_stub_thunk_pages) {
+		EXIT_IF(!Libs::LibKernel::Memory::FreeGuestMemory(page, UNRESOLVED_STUB_PAGE_SIZE));
+	}
+	g_unresolved_stub_thunk_pages.clear();
+	g_unresolved_stub_thunk_offset = 0;
+	g_stubbed_imports.clear();
+	g_unresolved_stub_call_log_count.store(0);
+	if (g_invalid_memory != 0) {
+		EXIT_IF(!Libs::LibKernel::Memory::FreeGuestMemory(g_invalid_memory, 4096));
+		g_invalid_memory = 0;
+	}
+	g_tls_main_program        = nullptr;
+	g_tls_cached_main_program = nullptr;
+	g_tls_cached_main_tcb     = nullptr;
+	g_desired_base_addr       = SYSTEM_RESERVED + CODE_BASE_OFFSET;
 	m_symbols.reset();
 	m_relocated = false;
 }
@@ -1926,10 +2084,11 @@ uint8_t* RuntimeLinker::TlsGetAddr(Program* program) {
 		const auto tcb_offset =
 		    program->tls.tcb_offset != 0 ? program->tls.tcb_offset : program->tls.image_size;
 		const auto alloc_size = AlignUp(tcb_offset, TCB_ALIGN) + TCB_SIZE;
-		tls.ptr               = reinterpret_cast<uint8_t*>(
-		    Common::VirtualMemory::Alloc(0, alloc_size, Common::VirtualMemory::Mode::ReadWrite));
-		tls.free_func = nullptr;
-		tls.vm_alloc  = true;
+		tls.ptr        = reinterpret_cast<uint8_t*>(Libs::LibKernel::Memory::AllocateRuntimeMemory(
+		    0, alloc_size, Common::VirtualMemory::Mode::ReadWrite, "thread_local_storage"));
+		tls.free_func  = nullptr;
+		tls.vm_alloc   = true;
+		tls.alloc_size = alloc_size;
 
 		EXIT_IF(tls.ptr == nullptr);
 
@@ -2006,8 +2165,9 @@ void RuntimeLinker::LoadProgramToMemory(Program* program) {
 	EXIT_IF(tls_handler_size > UINT64_MAX - program->base_size_aligned);
 	program->mapped_size = program->base_size_aligned + tls_handler_size;
 
-	program->base_vaddr = Common::VirtualMemory::Alloc(
-	    g_desired_base_addr, program->mapped_size, Common::VirtualMemory::Mode::ExecuteReadWrite);
+	program->base_vaddr = Libs::LibKernel::Memory::AllocateProgramMemory(
+	    g_desired_base_addr, program->mapped_size, Common::VirtualMemory::Mode::ExecuteReadWrite,
+	    Common::PathToString(program->file_name.filename()).c_str());
 
 	if (!is_shared) {
 		program->tls.handler_vaddr = program->base_vaddr + program->base_size_aligned;
@@ -2017,10 +2177,6 @@ void RuntimeLinker::LoadProgramToMemory(Program* program) {
 
 	EXIT_IF(program->base_vaddr == 0);
 	EXIT_IF(program->base_size_aligned < program->base_size);
-	Libs::LibKernel::Memory::RegisterProgramMemory(
-	    program->base_vaddr, program->mapped_size, Common::VirtualMemory::Mode::ExecuteReadWrite,
-	    Common::PathToString(program->file_name.filename()).c_str());
-
 	LOGF("base_vaddr             = 0x%016" PRIx64 "\n"
 	     "base_size              = 0x%016" PRIx64 "\n"
 	     "base_size_aligned      = 0x%016" PRIx64 "\n"
@@ -2060,11 +2216,8 @@ void RuntimeLinker::LoadProgramToMemory(Program* program) {
 			}
 
 			if (!skip_protect) {
-				if (!Common::VirtualMemory::Protect(segment_addr, segment_memory_size, mode)) {
-					EXIT("failed to protect ELF segment %u\n", static_cast<unsigned>(i));
-				}
-				Libs::LibKernel::Memory::UpdateProgramMemoryProtection(segment_addr,
-				                                                       segment_memory_size, mode);
+				Libs::LibKernel::Memory::SetProgramMemoryProtection(segment_addr,
+				                                                    segment_memory_size, mode);
 
 				if (Common::VirtualMemory::IsExecute(mode)) {
 					Common::VirtualMemory::FlushInstructionCache(segment_addr, segment_memory_size);
@@ -2105,15 +2258,29 @@ void RuntimeLinker::LoadProgramToMemory(Program* program) {
 
 void RuntimeLinker::DeleteProgram(Program* p) {
 	auto program = std::unique_ptr<Program>(p);
+	if (g_tls_main_program == program.get()) {
+		g_tls_main_program = nullptr;
+	}
+	if (g_tls_cached_main_program == program.get()) {
+		g_tls_cached_main_program = nullptr;
+		g_tls_cached_main_tcb     = nullptr;
+	}
+	for (auto& record: g_stubbed_imports) {
+		if (record.patch_vaddr >= program->base_vaddr &&
+		    record.patch_vaddr < program->base_vaddr + program->mapped_size) {
+			record.patch_vaddr = 0;
+		}
+	}
 
 	if (program->base_vaddr != 0 || program->mapped_size != 0) {
 		EXIT_IF(program->base_vaddr == 0 || program->mapped_size == 0);
-		Libs::LibKernel::Memory::UnregisterProgramMemory(program->base_vaddr, program->mapped_size);
-		EXIT_IF(!Common::VirtualMemory::Free(program->base_vaddr));
+		EXIT_IF(
+		    !Libs::LibKernel::Memory::FreeGuestMemory(program->base_vaddr, program->mapped_size));
 	}
 
 	if (program->custom_call_plt_vaddr != 0 || program->custom_call_plt_num != 0) {
-		Common::VirtualMemory::Free(program->custom_call_plt_vaddr);
+		const auto size = Jit::CallPlt::GetSize(program->custom_call_plt_num);
+		EXIT_IF(!Libs::LibKernel::Memory::FreeGuestMemory(program->custom_call_plt_vaddr, size));
 	}
 }
 
@@ -2237,13 +2404,13 @@ static void InstallRelocateHandler(Program* program) {
 	void**   pltgot       = reinterpret_cast<void**>(pltgot_vaddr);
 
 	Common::VirtualMemory::Mode old_mode {};
-	Common::VirtualMemory::Protect(pltgot_vaddr, pltgot_size, Common::VirtualMemory::Mode::Write,
-	                               &old_mode);
+	EXIT_IF(!Libs::LibKernel::Memory::ProtectGuestMemory(
+	    pltgot_vaddr, pltgot_size, Common::VirtualMemory::Mode::Write, &old_mode));
 
 	pltgot[1] = program;
 	pltgot[2] = reinterpret_cast<void*>(RelocateHandler);
 
-	Common::VirtualMemory::Protect(pltgot_vaddr, pltgot_size, old_mode);
+	EXIT_IF(!Libs::LibKernel::Memory::ProtectGuestMemory(pltgot_vaddr, pltgot_size, old_mode));
 
 	if (Common::VirtualMemory::IsExecute(old_mode)) {
 		Common::VirtualMemory::FlushInstructionCache(pltgot_vaddr, pltgot_size);
@@ -2253,15 +2420,15 @@ static void InstallRelocateHandler(Program* program) {
 	if (program->custom_call_plt_vaddr == 0) {
 		program->custom_call_plt_num =
 		    program->dynamic_info->jmprela_table_size / sizeof(Elf64_Rela);
-		auto size = Jit::CallPlt::GetSize(program->custom_call_plt_num);
-		program->custom_call_plt_vaddr =
-		    Common::VirtualMemory::Alloc(SYSTEM_RESERVED, size, Common::VirtualMemory::Mode::Write);
+		auto size                      = Jit::CallPlt::GetSize(program->custom_call_plt_num);
+		program->custom_call_plt_vaddr = Libs::LibKernel::Memory::AllocateRuntimeMemory(
+		    SYSTEM_RESERVED, size, Common::VirtualMemory::Mode::Write, "custom_call_plt");
 		EXIT_NOT_IMPLEMENTED(program->custom_call_plt_vaddr == 0);
 		auto* code = new (reinterpret_cast<void*>(program->custom_call_plt_vaddr))
 		    Jit::CallPlt(program->custom_call_plt_num);
 		code->SetPltGot(pltgot_vaddr);
-		Common::VirtualMemory::Protect(program->custom_call_plt_vaddr, size,
-		                               Common::VirtualMemory::Mode::Execute);
+		EXIT_IF(!Libs::LibKernel::Memory::ProtectGuestMemory(program->custom_call_plt_vaddr, size,
+		                                                     Common::VirtualMemory::Mode::Execute));
 		Common::VirtualMemory::FlushInstructionCache(program->custom_call_plt_vaddr, size);
 	}
 }
@@ -2272,8 +2439,8 @@ void RuntimeLinker::Relocate(Program* program) {
 	EXIT_IF(program == nullptr);
 
 	if (g_invalid_memory == 0) {
-		g_invalid_memory = Common::VirtualMemory::Alloc(INVALID_MEMORY, 4096,
-		                                                Common::VirtualMemory::Mode::NoAccess);
+		g_invalid_memory = Libs::LibKernel::Memory::AllocateRuntimeMemory(
+		    INVALID_MEMORY, 4096, Common::VirtualMemory::Mode::NoAccess, "invalid_memory", true);
 		EXIT_NOT_IMPLEMENTED(g_invalid_memory == 0);
 	}
 
@@ -2447,12 +2614,9 @@ void RuntimeLinker::SetupTlsHandler(Program* program) {
 		stub->SetOutputReg(reg);
 	}
 
-	if (!Common::VirtualMemory::Protect(program->tls.handler_vaddr, Jit::SafeCall::GetSize(),
-	                                    Common::VirtualMemory::Mode::Execute)) {
-		EXIT("failed to protect program TLS handler\n");
-	}
-	Libs::LibKernel::Memory::UpdateProgramMemoryProtection(
-	    program->tls.handler_vaddr, Jit::SafeCall::GetSize(), Common::VirtualMemory::Mode::Execute);
+	EXIT_IF(!Libs::LibKernel::Memory::ProtectGuestMemory(program->tls.handler_vaddr,
+	                                                     Jit::SafeCall::GetSize(),
+	                                                     Common::VirtualMemory::Mode::Execute));
 	Common::VirtualMemory::FlushInstructionCache(program->tls.handler_vaddr,
 	                                             Jit::SafeCall::GetSize());
 }

src/loader/runtimeLinker.h

@@ -48,9 +48,10 @@ struct LibraryId {
 
 struct ThreadLocalStorage {
 	struct Block {
-		uint8_t*                     ptr       = nullptr;
-		application_heap_free_func_t free_func = nullptr;
-		bool                         vm_alloc  = false;
+		uint8_t*                     ptr        = nullptr;
+		application_heap_free_func_t free_func  = nullptr;
+		bool                         vm_alloc   = false;
+		uint64_t                     alloc_size = 0;
 	};
 
 	~ThreadLocalStorage();
@@ -204,6 +205,11 @@ private:
 	application_heap_posix_memalign_func_t m_application_heap_posix_memalign = nullptr;
 };
 
+#if defined(KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS)
+bool TestMainEntryUsesGuestStack();
+bool TestModuleRelocationUsesWritableHostMapping();
+#endif
+
 } // namespace Loader
 
 #endif /* EMULATOR_INCLUDE_EMULATOR_LOADER_RUNTIMELINKER_H_ */

src/loader/x64InstructionEmulator.cpp

@@ -2,6 +2,8 @@
 
 #include "common/common.h"
 
+#include <cstring>
+
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
 #include <windows.h> // IWYU pragma: keep
 #elif !defined(__APPLE__)
@@ -56,6 +58,362 @@ static uint64_t InsertBitField(uint64_t dst, uint64_t src, uint32_t length, uint
 	return (dst & ~shifted) | src_shifted;
 }
 
+struct XmmWords {
+	uint32_t w[4];
+};
+
+static uint32_t Rol32(uint32_t value, unsigned int shift) {
+	shift &= 31u;
+	return (value << shift) | (value >> (32u - shift));
+}
+
+static uint32_t Rotr32(uint32_t value, unsigned int shift) {
+	shift &= 31u;
+	return (value >> shift) | (value << (32u - shift));
+}
+
+static void Sha1Msg1(XmmWords& dest, const XmmWords& src2) {
+	const uint32_t w0 = dest.w[3];
+	const uint32_t w1 = dest.w[2];
+	const uint32_t w2 = dest.w[1];
+	const uint32_t w3 = dest.w[0];
+	const uint32_t w4 = src2.w[3];
+	const uint32_t w5 = src2.w[2];
+	dest.w[3] = w2 ^ w0;
+	dest.w[2] = w3 ^ w1;
+	dest.w[1] = w4 ^ w2;
+	dest.w[0] = w5 ^ w3;
+}
+
+static void Sha1Msg2(XmmWords& dest, const XmmWords& src2) {
+	const uint32_t w13 = src2.w[2];
+	const uint32_t w14 = src2.w[1];
+	const uint32_t w15 = src2.w[0];
+	const uint32_t w16 = Rol32(dest.w[3] ^ w13, 1u);
+	const uint32_t w17 = Rol32(dest.w[2] ^ w14, 1u);
+	const uint32_t w18 = Rol32(dest.w[1] ^ w15, 1u);
+	const uint32_t w19 = Rol32(dest.w[0] ^ w16, 1u);
+	dest.w[3] = w16;
+	dest.w[2] = w17;
+	dest.w[1] = w18;
+	dest.w[0] = w19;
+}
+
+static void Sha1Nexte(XmmWords& dest, const XmmWords& src2) {
+	const uint32_t tmp  = Rol32(dest.w[3], 30u);
+	dest.w[3]           = src2.w[3] + tmp;
+	dest.w[2]           = src2.w[2];
+	dest.w[1]           = src2.w[1];
+	dest.w[0]           = src2.w[0];
+}
+
+static uint32_t Sha1RoundFunc(uint8_t group, uint32_t b, uint32_t c, uint32_t d) {
+	switch (group & 3u) {
+		case 0: return (b & c) ^ ((~b) & d);
+		case 1: return b ^ c ^ d;
+		case 2: return (b & c) ^ (b & d) ^ (c & d);
+		default: return b ^ c ^ d;
+	}
+}
+
+static uint32_t Sha1RoundConstant(uint8_t group) {
+	switch (group & 3u) {
+		case 0: return 0x5a827999u;
+		case 1: return 0x6ed9eba1u;
+		case 2: return 0x8f1bbcdcu;
+		default: return 0xca62c1d6u;
+	}
+}
+
+static void Sha1Rnds4(XmmWords& dest, const XmmWords& src2, uint8_t imm8) {
+	const uint8_t  group = imm8 & 3u;
+	const uint32_t k     = Sha1RoundConstant(group);
+	const uint32_t w[4]  = {src2.w[3], src2.w[2], src2.w[1], src2.w[0]};
+
+	uint32_t a = dest.w[3];
+	uint32_t b = dest.w[2];
+	uint32_t c = dest.w[1];
+	uint32_t d = dest.w[0];
+	uint32_t e = 0;
+
+	for (unsigned int round = 0; round < 4u; round++) {
+		uint32_t term = Sha1RoundFunc(group, b, c, d) + Rol32(a, 5u) + w[round] + k;
+		if (round > 0u) {
+			term += e;
+		}
+		const uint32_t a1 = term;
+		e                 = d;
+		d                 = c;
+		c                 = Rol32(b, 30u);
+		b                 = a;
+		a                 = a1;
+	}
+
+	dest.w[3] = a;
+	dest.w[2] = b;
+	dest.w[1] = c;
+	dest.w[0] = d;
+}
+
+static uint32_t Sha256Sigma0(uint32_t x) {
+	return Rotr32(x, 7u) ^ Rotr32(x, 18u) ^ (x >> 3u);
+}
+
+static uint32_t Sha256Sigma1(uint32_t x) {
+	return Rotr32(x, 17u) ^ Rotr32(x, 19u) ^ (x >> 10u);
+}
+
+static uint32_t Sha256Sum0(uint32_t x) {
+	return Rotr32(x, 2u) ^ Rotr32(x, 13u) ^ Rotr32(x, 22u);
+}
+
+static uint32_t Sha256Sum1(uint32_t x) {
+	return Rotr32(x, 6u) ^ Rotr32(x, 11u) ^ Rotr32(x, 25u);
+}
+
+static uint32_t Sha256Ch(uint32_t e, uint32_t f, uint32_t g) {
+	return (e & f) ^ ((~e) & g);
+}
+
+static uint32_t Sha256Maj(uint32_t a, uint32_t b, uint32_t c) {
+	return (a & b) ^ (a & c) ^ (b & c);
+}
+
+static void Sha256Msg1(XmmWords& dest, const XmmWords& src2) {
+	const uint32_t w4 = src2.w[0];
+	const uint32_t w3 = dest.w[3];
+	const uint32_t w2 = dest.w[2];
+	const uint32_t w1 = dest.w[1];
+	const uint32_t w0 = dest.w[0];
+	dest.w[3]           = w3 + Sha256Sigma0(w4);
+	dest.w[2]           = w2 + Sha256Sigma0(w3);
+	dest.w[1]           = w1 + Sha256Sigma0(w2);
+	dest.w[0]           = w0 + Sha256Sigma0(w1);
+}
+
+static void Sha256Msg2(XmmWords& dest, const XmmWords& src2) {
+	const uint32_t w14 = src2.w[2];
+	const uint32_t w15 = src2.w[3];
+	const uint32_t w16 = dest.w[0] + Sha256Sigma1(w14);
+	const uint32_t w17 = dest.w[1] + Sha256Sigma1(w15);
+	const uint32_t w18 = dest.w[2] + Sha256Sigma1(w16);
+	const uint32_t w19 = dest.w[3] + Sha256Sigma1(w17);
+	dest.w[3]          = w19;
+	dest.w[2]          = w18;
+	dest.w[1]          = w17;
+	dest.w[0]          = w16;
+}
+
+static void Sha256Rnds2(XmmWords& dest, const XmmWords& src2, const XmmWords& xmm0) {
+	uint32_t a = src2.w[3];
+	uint32_t b = src2.w[2];
+	uint32_t c = dest.w[3];
+	uint32_t d = dest.w[2];
+	uint32_t e = src2.w[1];
+	uint32_t f = src2.w[0];
+	uint32_t g = dest.w[1];
+	uint32_t h = dest.w[0];
+
+	for (unsigned int round = 0; round < 2u; round++) {
+		const uint32_t wk = xmm0.w[round];
+		const uint32_t t1 = Sha256Ch(e, f, g) + Sha256Sum1(e) + wk + h;
+		const uint32_t t2 = Sha256Maj(a, b, c) + Sha256Sum0(a);
+		const uint32_t a1 = t1 + t2;
+		const uint32_t e1 = t1 + d;
+		const uint32_t b1 = a;
+		const uint32_t c1 = b;
+		const uint32_t d1 = c;
+		const uint32_t f1 = e;
+		const uint32_t g1 = f;
+		const uint32_t h1 = g;
+		a                 = a1;
+		b                 = b1;
+		c                 = c1;
+		d                 = d1;
+		e                 = e1;
+		f                 = f1;
+		g                 = g1;
+		h                 = h1;
+	}
+
+	dest.w[3] = a;
+	dest.w[2] = b;
+	dest.w[1] = e;
+	dest.w[0] = f;
+}
+
+struct ShaNiInsn {
+	uint8_t escape;
+	uint8_t opcode;
+	uint8_t imm8;
+	uint8_t rex;
+	size_t  modrm_offset;
+	size_t  length;
+};
+
+static bool DecodeShaNiInsn(const uint8_t* rip, ShaNiInsn& insn) {
+	size_t  offset = 0;
+	uint8_t rex    = 0;
+	if ((rip[0] & 0xf0u) == 0x40u) {
+		rex    = rip[0];
+		offset = 1;
+	}
+
+	if (rip[offset] != 0x0f) {
+		return false;
+	}
+
+	if (rip[offset + 1] == 0x38) {
+		const uint8_t op = rip[offset + 2];
+		if (op != 0xc8 && op != 0xc9 && op != 0xca && op != 0xcb && op != 0xcc && op != 0xcd) {
+			return false;
+		}
+		insn.escape       = 0x38;
+		insn.opcode       = op;
+		insn.imm8         = 0;
+		insn.rex          = rex;
+		insn.modrm_offset = offset + 3;
+	} else if (rip[offset + 1] == 0x3a && rip[offset + 2] == 0xcc) {
+		insn.escape       = 0x3a;
+		insn.opcode       = 0xcc;
+		insn.rex          = rex;
+		insn.modrm_offset = offset + 3;
+	} else {
+		return false;
+	}
+
+	const uint8_t modrm = rip[insn.modrm_offset];
+	const uint8_t mod   = modrm >> 6u;
+	const uint8_t rm    = modrm & 0x07u;
+	size_t        end   = insn.modrm_offset + 1;
+
+	if (mod != 3u) {
+		uint8_t sib_base = 0xffu;
+		if (rm == 4u) {
+			sib_base = rip[end] & 0x07u;
+			end++;
+		}
+
+		if (mod == 0u && (rm == 5u || (rm == 4u && sib_base == 5u))) {
+			end += 4;
+		} else if (mod == 1u) {
+			end++;
+		} else if (mod == 2u) {
+			end += 4;
+		}
+	}
+
+	if (insn.escape == 0x3a) {
+		insn.imm8 = rip[end];
+		end++;
+	}
+
+	insn.length = end;
+	return true;
+}
+
+static bool ShaNiModrmIsRegister(uint8_t modrm) { return (modrm & 0xc0u) == 0xc0u; }
+
+static uint8_t ShaNiRegIndex(uint8_t modrm, uint8_t rex, bool reg_field) {
+	if (reg_field) {
+		return ((modrm >> 3u) & 0x07u) | ((rex & 0x04u) << 1u);
+	}
+	return (modrm & 0x07u) | ((rex & 0x01u) << 3u);
+}
+
+static bool ResolveShaNiMemoryAddress(const uint8_t* rip, const ShaNiInsn& insn,
+                                      const uint64_t (&gpr)[16], const void*& address) {
+	const uint8_t modrm = rip[insn.modrm_offset];
+	const uint8_t mod   = modrm >> 6u;
+	const uint8_t rm    = modrm & 0x07u;
+	if (mod == 3u) {
+		return false;
+	}
+
+	size_t   offset = insn.modrm_offset + 1;
+	uint64_t result = 0;
+
+	if (rm == 4u) {
+		const uint8_t sib        = rip[offset++];
+		const uint8_t scale      = sib >> 6u;
+		const uint8_t index_low  = (sib >> 3u) & 0x07u;
+		const uint8_t base_low   = sib & 0x07u;
+		const bool    has_index  = index_low != 4u || (insn.rex & 0x02u) != 0;
+		const bool    has_base   = mod != 0u || base_low != 5u;
+
+		if (has_base) {
+			const uint8_t base = base_low | ((insn.rex & 0x01u) << 3u);
+			result += gpr[base];
+		}
+		if (has_index) {
+			const uint8_t index = index_low | ((insn.rex & 0x02u) << 2u);
+			result += gpr[index] << scale;
+		}
+
+		if (!has_base) {
+			int32_t displacement = 0;
+			std::memcpy(&displacement, rip + offset, sizeof(displacement));
+			result += static_cast<uint64_t>(static_cast<int64_t>(displacement));
+			offset += sizeof(displacement);
+		}
+	} else if (mod == 0u && rm == 5u) {
+		int32_t displacement = 0;
+		std::memcpy(&displacement, rip + offset, sizeof(displacement));
+		result = reinterpret_cast<uint64_t>(rip + insn.length) +
+		         static_cast<uint64_t>(static_cast<int64_t>(displacement));
+		offset += sizeof(displacement);
+	} else {
+		const uint8_t base = rm | ((insn.rex & 0x01u) << 3u);
+		result             = gpr[base];
+	}
+
+	if (mod == 1u) {
+		const auto displacement = static_cast<int8_t>(rip[offset]);
+		result += static_cast<uint64_t>(static_cast<int64_t>(displacement));
+	} else if (mod == 2u) {
+		int32_t displacement = 0;
+		std::memcpy(&displacement, rip + offset, sizeof(displacement));
+		result += static_cast<uint64_t>(static_cast<int64_t>(displacement));
+	}
+
+	address = reinterpret_cast<const void*>(result);
+	return true;
+}
+
+static bool ExecuteShaNiInsn(const ShaNiInsn& insn, const XmmWords& src2, const XmmWords& xmm0,
+                             XmmWords& dest) {
+	if (insn.escape == 0x3a && insn.opcode == 0xcc) {
+		Sha1Rnds4(dest, src2, insn.imm8);
+		return true;
+	}
+
+	switch (insn.opcode) {
+		case 0xc8: Sha1Nexte(dest, src2); return true;
+		case 0xc9: Sha1Msg1(dest, src2); return true;
+		case 0xca: Sha1Msg2(dest, src2); return true;
+		case 0xcb: Sha256Rnds2(dest, src2, xmm0); return true;
+		case 0xcc: Sha256Msg1(dest, src2); return true;
+		case 0xcd: Sha256Msg2(dest, src2); return true;
+		default: return false;
+	}
+}
+
+#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
+
+static void LoadXmmWordsWin(const M128A* xmm, XmmWords& out) {
+	out.w[0] = static_cast<uint32_t>(xmm->Low);
+	out.w[1] = static_cast<uint32_t>(xmm->Low >> 32u);
+	out.w[2] = static_cast<uint32_t>(xmm->High);
+	out.w[3] = static_cast<uint32_t>(xmm->High >> 32u);
+}
+
+static void StoreXmmWordsWin(M128A* xmm, const XmmWords& in) {
+	xmm->Low  = static_cast<uint64_t>(in.w[0]) | (static_cast<uint64_t>(in.w[1]) << 32u);
+	xmm->High = static_cast<uint64_t>(in.w[2]) | (static_cast<uint64_t>(in.w[3]) << 32u);
+}
+
+#endif
+
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
 
 static M128A* GetContextXmm(PCONTEXT context, uint8_t index) {
@@ -66,6 +424,76 @@ static M128A* GetContextXmm(PCONTEXT context, uint8_t index) {
 	return &context->Xmm0 + index;
 }
 
+static void LoadContextGprsWin(PCONTEXT context, uint64_t (&gpr)[16]) {
+	gpr[0]  = context->Rax;
+	gpr[1]  = context->Rcx;
+	gpr[2]  = context->Rdx;
+	gpr[3]  = context->Rbx;
+	gpr[4]  = context->Rsp;
+	gpr[5]  = context->Rbp;
+	gpr[6]  = context->Rsi;
+	gpr[7]  = context->Rdi;
+	gpr[8]  = context->R8;
+	gpr[9]  = context->R9;
+	gpr[10] = context->R10;
+	gpr[11] = context->R11;
+	gpr[12] = context->R12;
+	gpr[13] = context->R13;
+	gpr[14] = context->R14;
+	gpr[15] = context->R15;
+}
+
+static bool TryEmulateShaNi(PCONTEXT context) {
+	if (context == nullptr) {
+		return false;
+	}
+
+	const auto* rip = reinterpret_cast<const uint8_t*>(context->Rip);
+	ShaNiInsn   insn {};
+	if (!DecodeShaNiInsn(rip, insn)) {
+		return false;
+	}
+
+	const uint8_t modrm_byte = rip[insn.modrm_offset];
+	const uint8_t dest_index = ShaNiRegIndex(modrm_byte, insn.rex, true);
+	auto*         dest_xmm   = GetContextXmm(context, dest_index);
+	auto*         xmm0       = GetContextXmm(context, 0);
+	if (dest_xmm == nullptr || xmm0 == nullptr) {
+		return false;
+	}
+
+	XmmWords dest {};
+	XmmWords src2 {};
+	XmmWords xmm0_words {};
+	LoadXmmWordsWin(dest_xmm, dest);
+	LoadXmmWordsWin(xmm0, xmm0_words);
+
+	if (ShaNiModrmIsRegister(modrm_byte)) {
+		const uint8_t src_index = ShaNiRegIndex(modrm_byte, insn.rex, false);
+		auto*         src_xmm   = GetContextXmm(context, src_index);
+		if (src_xmm == nullptr) {
+			return false;
+		}
+		LoadXmmWordsWin(src_xmm, src2);
+	} else {
+		uint64_t    gpr[16] {};
+		const void* source = nullptr;
+		LoadContextGprsWin(context, gpr);
+		if (!ResolveShaNiMemoryAddress(rip, insn, gpr, source)) {
+			return false;
+		}
+		std::memcpy(&src2, source, sizeof(src2));
+	}
+
+	if (!ExecuteShaNiInsn(insn, src2, xmm0_words, dest)) {
+		return false;
+	}
+
+	StoreXmmWordsWin(dest_xmm, dest);
+	context->Rip += insn.length;
+	return true;
+}
+
 static bool TryEmulateSse4a(PCONTEXT context) {
 	if (context == nullptr) {
 		return false;
@@ -160,6 +588,91 @@ static uint32_t* GetContextXmm(ucontext_t* context, uint8_t index) {
 	return static_cast<uint32_t*>(fpregs->_xmm[index].element);
 }
 
+static void LoadContextGprsLin(ucontext_t* context, uint64_t (&gpr)[16]) {
+	gpr[0]  = static_cast<uint64_t>(context->uc_mcontext.gregs[REG_RAX]);
+	gpr[1]  = static_cast<uint64_t>(context->uc_mcontext.gregs[REG_RCX]);
+	gpr[2]  = static_cast<uint64_t>(context->uc_mcontext.gregs[REG_RDX]);
+	gpr[3]  = static_cast<uint64_t>(context->uc_mcontext.gregs[REG_RBX]);
+	gpr[4]  = static_cast<uint64_t>(context->uc_mcontext.gregs[REG_RSP]);
+	gpr[5]  = static_cast<uint64_t>(context->uc_mcontext.gregs[REG_RBP]);
+	gpr[6]  = static_cast<uint64_t>(context->uc_mcontext.gregs[REG_RSI]);
+	gpr[7]  = static_cast<uint64_t>(context->uc_mcontext.gregs[REG_RDI]);
+	gpr[8]  = static_cast<uint64_t>(context->uc_mcontext.gregs[REG_R8]);
+	gpr[9]  = static_cast<uint64_t>(context->uc_mcontext.gregs[REG_R9]);
+	gpr[10] = static_cast<uint64_t>(context->uc_mcontext.gregs[REG_R10]);
+	gpr[11] = static_cast<uint64_t>(context->uc_mcontext.gregs[REG_R11]);
+	gpr[12] = static_cast<uint64_t>(context->uc_mcontext.gregs[REG_R12]);
+	gpr[13] = static_cast<uint64_t>(context->uc_mcontext.gregs[REG_R13]);
+	gpr[14] = static_cast<uint64_t>(context->uc_mcontext.gregs[REG_R14]);
+	gpr[15] = static_cast<uint64_t>(context->uc_mcontext.gregs[REG_R15]);
+}
+
+static void LoadXmmWordsLin(const uint32_t* xmm, XmmWords& out) {
+	out.w[0] = xmm[0];
+	out.w[1] = xmm[1];
+	out.w[2] = xmm[2];
+	out.w[3] = xmm[3];
+}
+
+static void StoreXmmWordsLin(uint32_t* xmm, const XmmWords& in) {
+	xmm[0] = in.w[0];
+	xmm[1] = in.w[1];
+	xmm[2] = in.w[2];
+	xmm[3] = in.w[3];
+}
+
+static bool TryEmulateShaNi(ucontext_t* context) {
+	if (context == nullptr) {
+		return false;
+	}
+
+	auto&       rip_reg = context->uc_mcontext.gregs[REG_RIP];
+	const auto* rip     = reinterpret_cast<const uint8_t*>(rip_reg);
+	ShaNiInsn   insn {};
+	if (!DecodeShaNiInsn(rip, insn)) {
+		return false;
+	}
+
+	const uint8_t modrm_byte = rip[insn.modrm_offset];
+	const uint8_t dest_index = ShaNiRegIndex(modrm_byte, insn.rex, true);
+	auto*         dest_xmm   = GetContextXmm(context, dest_index);
+	auto*         xmm0       = GetContextXmm(context, 0);
+	if (dest_xmm == nullptr || xmm0 == nullptr) {
+		return false;
+	}
+
+	XmmWords dest {};
+	XmmWords src2 {};
+	XmmWords xmm0_words {};
+	LoadXmmWordsLin(dest_xmm, dest);
+	LoadXmmWordsLin(xmm0, xmm0_words);
+
+	if (ShaNiModrmIsRegister(modrm_byte)) {
+		const uint8_t src_index = ShaNiRegIndex(modrm_byte, insn.rex, false);
+		auto*         src_xmm   = GetContextXmm(context, src_index);
+		if (src_xmm == nullptr) {
+			return false;
+		}
+		LoadXmmWordsLin(src_xmm, src2);
+	} else {
+		uint64_t    gpr[16] {};
+		const void* source = nullptr;
+		LoadContextGprsLin(context, gpr);
+		if (!ResolveShaNiMemoryAddress(rip, insn, gpr, source)) {
+			return false;
+		}
+		std::memcpy(&src2, source, sizeof(src2));
+	}
+
+	if (!ExecuteShaNiInsn(insn, src2, xmm0_words, dest)) {
+		return false;
+	}
+
+	StoreXmmWordsLin(dest_xmm, dest);
+	rip_reg += static_cast<greg_t>(insn.length);
+	return true;
+}
+
 static uint64_t GetXmmLow(const uint32_t* xmm) {
 	return static_cast<uint64_t>(xmm[0]) | (static_cast<uint64_t>(xmm[1]) << 32u);
 }
@@ -258,10 +771,12 @@ static bool TryEmulateMonitorxMwaitx(ucontext_t* context) {
 bool TryEmulate(void* native_context) {
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
 	auto* context = static_cast<PCONTEXT>(native_context);
-	return TryEmulateMonitorxMwaitx(context) || TryEmulateSse4a(context);
+	return TryEmulateMonitorxMwaitx(context) || TryEmulateSse4a(context) ||
+	       TryEmulateShaNi(context);
 #elif !defined(__APPLE__)
 	auto* context = static_cast<ucontext_t*>(native_context);
-	return TryEmulateMonitorxMwaitx(context) || TryEmulateSse4a(context);
+	return TryEmulateMonitorxMwaitx(context) || TryEmulateSse4a(context) ||
+	       TryEmulateShaNi(context);
 #else
 	(void)native_context;
 	return false;

tests/MemoryTrackerTests.cpp

@@ -1,6 +1,7 @@
 #include "graphics/host_gpu/memoryTracker.h"
 #include "graphics/host_gpu/rangeSet.h"
 #include "common/assert.h"
+#include "common/virtualMemory.h"
 
 #include <atomic>
 #include <cstdint>
@@ -209,6 +210,20 @@ class SharedPage final {
   HANDLE mapping_ = nullptr;
 };
 #endif
+
+bool ProtectAddressSpace(uint64_t vaddr, uint64_t size,
+                         Common::VirtualMemory::Mode mode) {
+  uint32_t protection = PAGE_NOACCESS;
+  if (mode == Common::VirtualMemory::Mode::Read) {
+    protection = PAGE_READONLY;
+  } else if (mode == Common::VirtualMemory::Mode::ReadWrite) {
+    protection = PAGE_READWRITE;
+  }
+  DWORD old_protection = 0;
+  return VirtualProtect(reinterpret_cast<void *>(vaddr), size, protection,
+                        &old_protection) != 0;
+}
+
 #if 1
 
 bool DummyFault(void *, PageFaultAccess, uint64_t, uint64_t, PageFaultPhase) noexcept {
@@ -353,7 +368,8 @@ struct DownloadTrackerHarness {
     return completed;
   }
 
-  DownloadTrackerHarness() : page_manager(Fault, this), tracker(page_manager) {}
+  DownloadTrackerHarness()
+      : page_manager(Fault, this), tracker(page_manager) {}
 
   PageFaultAccess pending_access = PageFaultAccess::Unknown;
   uint64_t download_address = 0;
@@ -366,11 +382,6 @@ struct DownloadTrackerHarness {
 
 std::atomic<PageManager *> g_native_page_manager{nullptr};
 std::atomic_bool g_native_fault_entered{false};
-std::atomic_bool g_unmap_contended{false};
-
-void UnmapContended() noexcept {
-  g_unmap_contended.store(true, std::memory_order_release);
-}
 
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
 LONG CALLBACK NativeTrackerFaultHandler(EXCEPTION_POINTERS *exception) {
@@ -552,7 +563,7 @@ void TestGpuDownloadFaultOwnership() {
             !harness.tracker.IsRegionGpuModified(address, page_size) &&
             harness.tracker.IsRegionCpuModified(address, page_size) && IsWritable(memory),
         "GPU write fault did not download before granting CPU ownership");
-  harness.tracker.UnmapMemory(address, page_size);
+  harness.tracker.UntrackMemory(address, page_size);
 }
 
 void TestVirtualGpuWriteDiscard() {
@@ -578,7 +589,7 @@ void TestVirtualGpuWriteDiscard() {
   Check(!harness.tracker.IsRegionGpuModified(address, page_size) &&
             harness.tracker.IsRegionCpuModified(address, page_size) && IsWritable(memory),
         "virtual GPU discard did not transfer the page to CPU ownership");
-  harness.tracker.UnmapMemory(address, page_size);
+  harness.tracker.UntrackMemory(address, page_size);
   Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
 }
 
@@ -684,6 +695,15 @@ void TestRangeSet() {
         "range set subtraction did not preserve both exact tails");
 }
 
+void TestQueriesDoNotRequireMappedOwnership() {
+  constexpr uint64_t address = 0x0000000203000000ull;
+  TrackerHarness harness;
+  const auto page_size = harness.page_manager.GetPageSize();
+  Check(harness.tracker.IsRegionCpuModified(address, page_size) &&
+            !harness.tracker.IsRegionGpuModified(address, page_size),
+        "unowned tracker range did not expose its initial CPU-dirty state");
+}
+
 void TestRangeInvalidation() {
   constexpr uintptr_t base = 0x0000000201000000ull;
   TrackerHarness harness;
@@ -767,7 +787,7 @@ void TestCpuDirtyUploadAndFault() {
   Check(IsWritable(memory),
         "explicit CPU dirty transition did not release the rearmed watch");
 
-  tracker.UnmapMemory(address, page_size * 2);
+  tracker.UntrackMemory(address, page_size * 2);
   Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
 }
 
@@ -795,7 +815,7 @@ void TestFaultDuringUploadRemainsDirty() {
       });
   Check(tracker.IsRegionCpuModified(address, page_size) && IsWritable(memory),
         "upload completion erased a racing CPU dirty transition");
-  tracker.UnmapMemory(address, page_size);
+  tracker.UntrackMemory(address, page_size);
   Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
 }
 
@@ -840,7 +860,7 @@ void TestNativeStoreDuringRangeEnumeration() {
             IsWritable(memory),
         "native store during range enumeration was lost");
 
-  tracker.UnmapMemory(address, page_size);
+  tracker.UntrackMemory(address, page_size);
   Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
 }
 
@@ -924,7 +944,7 @@ void TestFaultDuringDownloadSynchronization() {
             page_manager.IsTracked(address + page_size * 2),
         "uncontended dirty page did not retain its clean write watch");
 
-  tracker.UnmapMemory(address, page_size * 3);
+  tracker.UntrackMemory(address, page_size * 3);
 }
 
 void TestFaultAndExplicitDirtyRace() {
@@ -964,7 +984,7 @@ void TestFaultAndExplicitDirtyRace() {
               IsWritable(memory),
           "fault/explicit-dirty race lost dirty state or write access");
   }
-  tracker.UnmapMemory(address, page_size);
+  tracker.UntrackMemory(address, page_size);
   Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
 }
 
@@ -1016,7 +1036,7 @@ void TestSharedTrackersAndConcurrentPageFaults() {
   }
 
   harness.first.UntrackMemory(address, page_size * 2);
-  harness.second.UnmapMemory(address, page_size * 2);
+  harness.second.UntrackMemory(address, page_size * 2);
   Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
 }
 
@@ -1049,7 +1069,7 @@ void TestGpuDirtyBits() {
         "explicit GPU dirty transition did not trap CPU access");
   tracker.UnmarkRegionAsGpuModified(address, page_size);
   tracker.MarkRegionAsCpuModified(address, page_size);
-  tracker.UnmapMemory(address, page_size * 2);
+  tracker.UntrackMemory(address, page_size * 2);
   Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
 }
 
@@ -1084,7 +1104,7 @@ void TestCrossRegionUpload() {
         "cross-region written upload did not mark GPU dirty state");
   tracker.UnmarkRegionAsGpuModified(boundary - page_size, page_size * 2);
   tracker.MarkRegionAsCpuModified(boundary - page_size, page_size * 2);
-  tracker.UnmapMemory(address, region_size * 2);
+  tracker.UntrackMemory(address, region_size * 2);
   Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
 }
 
@@ -1094,9 +1114,6 @@ void TestCrossRegionUpload() {
   auto &tracker = harness.tracker;
   auto &page_manager = harness.page_manager;
   const auto page_size = page_manager.GetPageSize();
-  if (std::strcmp(name, "unmapped") == 0) {
-    (void)tracker.IsRegionCpuModified(base, page_size);
-  }
   const auto allocation_size =
       std::strcmp(name, "missing-download-bytes") == 0 ? page_size * 2
                                                        : page_size;
@@ -1149,20 +1166,6 @@ void TestCrossRegionUpload() {
           }
     });
     fault.join();
-  } else if (std::strcmp(name, "gpu-dirty-unmap-race") == 0) {
-    g_unmap_contended.store(false, std::memory_order_release);
-    MemoryTracker::SetUnmapContentionHook(UnmapContended);
-    std::thread unmap;
-    tracker.ForEachUploadRange(
-        address, page_size, true, [](uint64_t, uint64_t) noexcept {},
-        [&]() noexcept {
-          unmap = std::thread(
-              [&] { tracker.UnmapMemory(address, page_size); });
-          while (!g_unmap_contended.load(std::memory_order_acquire)) {
-            std::this_thread::yield();
-          }
-        });
-    unmap.join();
   } else if (std::strcmp(name, "missing-download-bytes") == 0) {
     tracker.ForEachUploadRange(
         address, allocation_size, true, [](uint64_t, uint64_t) noexcept {},
@@ -1193,8 +1196,7 @@ void TestFatalPaths() {
 #endif
   for (const char *name : {"gpu-dirty-fault", "gpu-dirty-read", "virtual-gpu-read",
                            "gpu-dirty-explicit-cpu",
-                           "unmapped", "reentrant-upload",
-                           "writable-upload-race", "gpu-dirty-unmap-race",
+                           "reentrant-upload", "writable-upload-race",
                            "missing-download-bytes"}) {
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
     std::string command = std::string("\"") + path + "\" --death " + name;
@@ -1236,6 +1238,14 @@ void TestFatalPaths() {
 
 } // namespace
 
+namespace Libs::LibKernel::Memory {
+
+bool ProtectGuestHostMemory(uint64_t vaddr, uint64_t size, Common::VirtualMemory::Mode mode) {
+  return ProtectAddressSpace(vaddr, size, mode);
+}
+
+} // namespace Libs::LibKernel::Memory
+
 int main(int argc, char **argv) {
 #if 1
   if (argc == 3 && std::strcmp(argv[1], "--death") == 0) {
@@ -1249,6 +1259,7 @@ int main(int argc, char **argv) {
   TestSameSlabTrackerArbitration();
   TestSharedMetadataAndImagePageFault();
   TestRangeSet();
+  TestQueriesDoNotRequireMappedOwnership();
   TestRangeInvalidation();
   TestGpuDirtyBits();
   TestCrossRegionUpload();

tests/PageManagerTests.cpp

@@ -1,4 +1,5 @@
 #include "graphics/host_gpu/pageManager.h"
+#include "common/virtualMemory.h"
 
 #include <atomic>
 #include <cstdint>
@@ -28,7 +29,6 @@
 
 namespace {
 
-using Libs::Graphics::GpuAccess;
 using Libs::Graphics::PageFaultAccess;
 using Libs::Graphics::PageManager;
 
@@ -122,6 +122,23 @@ uint32_t Protection(const void *address) {
   return info.Protect;
 }
 #endif
+
+std::atomic_uint64_t g_protection_calls{0};
+
+bool ProtectAddressSpace(uint64_t vaddr, uint64_t size,
+                         Common::VirtualMemory::Mode mode) {
+  uint32_t protection = PAGE_NOACCESS;
+  if (mode == Common::VirtualMemory::Mode::Read) {
+    protection = PAGE_READONLY;
+  } else if (mode == Common::VirtualMemory::Mode::ReadWrite) {
+    protection = PAGE_READWRITE;
+  }
+  DWORD old_protection = 0;
+  g_protection_calls.fetch_add(1, std::memory_order_relaxed);
+  return VirtualProtect(reinterpret_cast<void *>(vaddr), size, protection,
+                        &old_protection) != 0;
+}
+
 #if 1
 
 struct FaultContext {
@@ -254,6 +271,7 @@ uint8_t *Allocate(uint64_t size, uint32_t protection = PAGE_READWRITE) {
 }
 
 void TestWatchFaultAndUnwatch() {
+  g_protection_calls.store(0, std::memory_order_relaxed);
   FaultContext context;
   PageManager manager(InvalidateFault, &context);
   context.manager = &manager;
@@ -266,6 +284,8 @@ void TestWatchFaultAndUnwatch() {
   Check(manager.IsTracked(reinterpret_cast<uint64_t>(memory)) &&
             !IsWritable(memory),
         "watch did not protect the page");
+  Check(g_protection_calls.load(std::memory_order_relaxed) != 0,
+        "watch protection bypassed the address-space owner callback");
   Check(manager.HandleFault(PageFaultAccess::Write,
                             reinterpret_cast<uint64_t>(memory + 32)),
         "tracked write fault was not handled");
@@ -343,14 +363,6 @@ void TestPermittedMappedLateFaultsResume() {
         "second delayed mapped write was not accepted");
   Check(manager.HandleFault(PageFaultAccess::Read, address),
         "delayed mapped read was not accepted on readable backing");
-  DWORD old_protection = 0;
-  Check(VirtualProtect(memory, page_size, PAGE_READONLY, &old_protection) != 0 &&
-            old_protection == PAGE_READWRITE,
-        "failed to prepare intentional read-only protection");
-  Check(!manager.HandleFault(PageFaultAccess::Write, address),
-        "intentional read-only mapping accepted a write fault");
-  Check(VirtualProtect(memory, page_size, PAGE_READWRITE, &old_protection) != 0,
-        "failed to restore writable protection");
   manager.OnGpuUnmap(address, page_size);
   Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
 }
@@ -496,31 +508,6 @@ void TestNativeAccessViolation() {
   Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
 }
 
-void TestInvalidLateWriteTokenIsConsumed() {
-  FaultContext context;
-  PageManager manager(InvalidateFault, &context);
-  context.manager = &manager;
-  const auto page_size = manager.GetPageSize();
-  auto *memory = Allocate(page_size);
-  const auto address = reinterpret_cast<uint64_t>(memory);
-  manager.OnGpuMap(address, page_size);
-  manager.UpdatePageWatchers(true, address, page_size);
-  Check(manager.HandleFault(PageFaultAccess::Write, address),
-        "initial write fault was not handled");
-  DWORD old_protection = 0;
-  Check(VirtualProtect(memory, page_size, PAGE_READONLY, &old_protection) !=
-                0 &&
-            old_protection == PAGE_READWRITE,
-        "failed to create invalid late-write protection state");
-  Check(!manager.HandleFault(PageFaultAccess::Write, address) &&
-            !manager.HandleFault(PageFaultAccess::Write, address),
-        "invalid late-write token was accepted or retained");
-  Check(VirtualProtect(memory, page_size, PAGE_READWRITE, &old_protection) != 0,
-        "failed to restore test protection");
-  manager.OnGpuUnmap(address, page_size);
-  Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
-}
-
 void TestCrossRegionRange() {
   FaultContext context;
   PageManager manager(InvalidateFault, &context);
@@ -627,9 +614,7 @@ void TestBatchedWatcherRanges() {
     manager->UpdatePageWatchers(false, 0x1000, page_size);
   } else {
     const bool two_pages = std::strcmp(name, "cross-reentrant") == 0;
-    auto *memory = Allocate(
-        two_pages ? page_size * 2 : page_size,
-        std::strcmp(name, "protection") == 0 ? PAGE_READONLY : PAGE_READWRITE);
+    auto *memory = Allocate(two_pages ? page_size * 2 : page_size);
     const auto address = reinterpret_cast<uint64_t>(memory);
     manager->OnGpuMap(address, two_pages ? page_size * 2 : page_size);
     manager->UpdatePageWatchers(true, address, page_size);
@@ -659,9 +644,7 @@ void TestBatchedWatcherRanges() {
       }
       (void)manager->HandleFault(PageFaultAccess::Read, address);
       first.join();
-    } else if (std::strcmp(name, "watched-unmap") == 0) {
-      manager->OnGpuUnmap(address, page_size);
-    } else if (std::strcmp(name, "protection") != 0) {
+    } else {
       std::_Exit(0x7f);
     }
   }
@@ -712,7 +695,7 @@ void TestFatalPaths() {
   for (const char *name :
        {"invalid-range", "unknown-untrack", "destructor-watch", "non-write",
         "callback-false", "reentrant", "cross-reentrant",
-        "concurrent-non-write", "watched-unmap", "protection"}) {
+        "concurrent-non-write"}) {
     CheckDeathCase(name);
   }
 }
@@ -773,51 +756,18 @@ void TestExternalDirtyTransferDuringResolution() {
   Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
 }
 
-void TestMappingDoesNotRequireCpuWriteAccess() {
-  FaultContext context;
-  PageManager manager(InvalidateFault, &context);
-  context.manager = &manager;
-  const auto page_size = manager.GetPageSize();
-  auto *memory = Allocate(page_size);
-  DWORD old_protection = 0;
-  Check(VirtualProtect(memory, page_size, PAGE_NOACCESS, &old_protection) != 0 &&
-            old_protection == PAGE_READWRITE,
-        "failed to prepare CPU-inaccessible mapping");
-  const auto address = reinterpret_cast<uint64_t>(memory);
-  manager.OnGpuMap(address, page_size);
-  Check(manager.IsMapped(address, page_size),
-        "CPU-inaccessible committed range was not GPU mapped");
-  manager.OnGpuUnmap(address, page_size);
-  Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
-}
-
-void TestGpuAccessPermissions() {
-  FaultContext context;
-  PageManager manager(InvalidateFault, &context);
-  context.manager = &manager;
-  const auto page_size = manager.GetPageSize();
-  auto *memory = Allocate(page_size);
-  const auto address = reinterpret_cast<uint64_t>(memory);
-  manager.OnGpuMap(address, page_size, GpuAccess::Read);
-  Check(manager.HasGpuAccess(address, page_size, GpuAccess::Read) &&
-            !manager.HasGpuAccess(address, page_size, GpuAccess::Write),
-        "read-only GPU mapping granted write access");
-  manager.OnGpuMap(address, page_size, GpuAccess::Write);
-  Check(manager.HasGpuAccess(address, page_size, GpuAccess::ReadWrite),
-        "overlapping GPU mappings did not combine permissions");
-  manager.OnGpuUnmap(address, page_size, GpuAccess::Read);
-  Check(!manager.HasGpuAccess(address, page_size, GpuAccess::Read) &&
-            manager.HasGpuAccess(address, page_size, GpuAccess::Write),
-        "GPU read unmap removed the wrong permission");
-  manager.OnGpuUnmap(address, page_size, GpuAccess::Write);
-  Check(!manager.IsMapped(address, page_size),
-        "GPU permission mappings were not fully balanced");
-  Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
-}
 #endif
 
 } // namespace
 
+namespace Libs::LibKernel::Memory {
+
+bool ProtectGuestHostMemory(uint64_t vaddr, uint64_t size, Common::VirtualMemory::Mode mode) {
+  return ProtectAddressSpace(vaddr, size, mode);
+}
+
+} // namespace Libs::LibKernel::Memory
+
 int main(int argc, char **argv) {
 #if 1
   if (argc == 3 && std::strcmp(argv[1], "--death") == 0) {
@@ -831,13 +781,10 @@ int main(int argc, char **argv) {
   TestNativeDelayedReadAfterModeDowngrade();
   TestDelayedFaultAfterExplicitUnwatch();
   TestNativeAccessViolation();
-  TestInvalidLateWriteTokenIsConsumed();
   TestCrossRegionRange();
   TestBatchedWatcherRanges();
   TestConcurrentFault();
   TestExternalDirtyTransferDuringResolution();
-  TestMappingDoesNotRequireCpuWriteAccess();
-  TestGpuAccessPermissions();
   TestFatalPaths();
   std::puts("PageManagerTests: all cases passed");
   return 0;

tests/ResourceTrackingTests.cpp

@@ -1217,6 +1217,34 @@ void TestResourceSpecializationIsTypedAndTransactional() {
                 Decoder::ImageDimension::Dim2DArray,
         "array MIMG intent did not produce a 2D-array view");
 
+  Program cube_view;
+  cube_view.stage = ShaderType::Compute;
+  cube_view.blocks.resize(1);
+  cube_view.blocks[0].instructions = {
+      ImageUse(0x24, Opcode::ImageLoad, ResourceKind::Image,
+               Decoder::ImageDimension::Dim2DArray)};
+  Prepare(cube_view);
+  auto cube_snapshot = array_2d_snapshot;
+  cube_snapshot.images[0].dwords[3] =
+      Prospero::GpuEnumValue(Prospero::ImageType::kCube) << 28u;
+  Check(SpecializeResources(cube_view, cube_snapshot, &error) &&
+            ValidateResourceSpecialization(cube_view, cube_snapshot, &error) &&
+            cube_view.info.images[0].cube &&
+            cube_view.blocks[0].instructions[0].memory.image_cube,
+        "cube descriptor identity did not reach the specialized image and IR");
+  auto array_after_cube = cube_snapshot;
+  array_after_cube.images[0].dwords[3] =
+      Prospero::GpuEnumValue(Prospero::ImageType::kColor2DArray) << 28u;
+  Check(!ValidateResourceSpecialization(cube_view, array_after_cube, &error),
+        "2D-array descriptor reused a cube-coordinate specialization");
+  auto null_after_cube = cube_snapshot;
+  null_after_cube.images[0].dwords.fill(0);
+  Check(SpecializeResources(cube_view, null_after_cube, &error) &&
+            ValidateResourceSpecialization(cube_view, null_after_cube, &error) &&
+            !cube_view.info.images[0].cube &&
+            !cube_view.blocks[0].instructions[0].memory.image_cube,
+        "canonical null respecialization retained stale cube-coordinate state");
+
   Program program;
   program.stage = ShaderType::Compute;
   program.blocks.resize(1);

tests/ScalarProvenanceTests.cpp

@@ -186,6 +186,39 @@ void TestCfgPhi() {
 	      "acyclic control-flow descriptor phi was not classified dynamic");
 }
 
+void TestNestedLoopPhiConvergence() {
+	Program program;
+	program.blocks.resize(4);
+	program.blocks[0].predecessors = {1};
+	program.blocks[0].successors   = {1};
+	program.blocks[1].predecessors = {0, 3};
+	program.blocks[1].successors   = {0, 2};
+	program.blocks[2].predecessors = {1};
+	program.blocks[2].successors   = {3};
+	program.blocks[3].predecessors = {2};
+	program.blocks[3].successors   = {1};
+	Instruction increment;
+	increment.op                     = Opcode::IAddU32;
+	increment.dst                    = Sgpr(0);
+	increment.src[0]                 = Sgpr(0);
+	increment.src[1]                 = Imm(1);
+	increment.src_count              = 2;
+	program.blocks[0].instructions   = {increment};
+	program.blocks[2].instructions   = {BufferUse(4, 0)};
+
+	std::string error;
+	Check(BuildScalarProvenance(program, &error), error.c_str());
+	const auto* source =
+	    GetDescriptorSource(program, program.blocks[2].instructions[0].memory.resource_source);
+	Check(source != nullptr, "nested-loop descriptor source was not attached");
+	const auto  value_id = source->dwords[0];
+	const auto& phi      = Value(program, value_id);
+	Check(phi.op == ScalarValueOp::Phi && phi.phi_args.size() == 2 &&
+	          ((phi.phi_args[0] == value_id && phi.phi_args[1] != value_id) ||
+	           (phi.phi_args[1] == value_id && phi.phi_args[0] != value_id)),
+	      "nested loop did not retain its recursive scalar provenance phi");
+}
+
 void TestDiamondReadPathsAreDynamic() {
 	std::array<uint32_t, 1> left  = {0x11111111u};
 	std::array<uint32_t, 1> right = {0x22222222u};
@@ -1045,6 +1078,7 @@ int main() {
 	try {
 		TestPerUseDescriptorDefinitions();
 		TestCfgPhi();
+		TestNestedLoopPhiConvergence();
 		TestDiamondReadPathsAreDynamic();
 		TestEquivalentConstantPhiIsStatic();
 		TestWideMoveInvalidatesAndCopiesBothDwords();

tests/ShaderRecompilerComputeTests.cpp

@@ -1274,6 +1274,59 @@ public:
     std::printf("[host]    %-32s ok\n", "SchedulerTimeline");
   }
 
+  void CheckGpuMappedRangeLifecycle() {
+    EnsureRuntimeContext();
+    CommandScheduler scheduler(Renderer(), m_runtime_context);
+    HW::Context registers{};
+    HW::UserConfig user_config{};
+    HW::Shader shaders{};
+    scheduler.Begin(registers, user_config, shaders);
+    Gpu gpu(Renderer());
+    GpuResourceManager resources(m_runtime_context, scheduler);
+    resources.SetGpu(&gpu);
+
+    constexpr uint64_t base = 0x0000000200000000ull;
+    constexpr uint64_t page = 0x4000;
+    resources.MapMemory(base, page * 4);
+    resources.MapMemory(base + page * 2, page * 4);
+    Require("GpuMappedRangeLifecycle", "union",
+            resources.IsMapped(base, page * 6) &&
+                !resources.IsMapped(base, page * 7),
+            "overlapping maps did not form one interval union");
+
+    resources.UnmapMemory(base + page * 2, page * 2);
+    Require("GpuMappedRangeLifecycle", "subtract",
+            resources.IsMapped(base, page * 2) &&
+                resources.IsMapped(base + page * 4, page * 2) &&
+                !resources.IsMapped(base, page * 6),
+            "partial unmap did not punch the expected interval hole");
+
+    resources.UnmapMemory(base + page * 2, page * 2);
+    Require("GpuMappedRangeLifecycle", "idempotent unmap",
+            resources.IsMapped(base, page * 2) &&
+                resources.IsMapped(base + page * 4, page * 2),
+            "unmapping an absent interval changed neighboring mappings");
+
+    resources.UnmapMemory(base, page * 6);
+    Require("GpuMappedRangeLifecycle", "clear",
+            !resources.IsMapped(base, page * 6),
+            "full unmap did not clear the interval union");
+
+    constexpr uint64_t old_prt = base + page * 8;
+    constexpr uint64_t new_prt = base + page * 16;
+    resources.MapMemory(old_prt, page * 4);
+    resources.UnmapMemory(old_prt, page * 4);
+    resources.MapMemory(new_prt, page * 6);
+    Require("GpuMappedRangeLifecycle", "PRT replacement",
+            !resources.IsMapped(old_prt, page * 4) &&
+                resources.IsMapped(new_prt, page * 6),
+            "old-unmap/new-map did not replace full PRT coverage");
+
+    resources.SetGpu(nullptr);
+    scheduler.Finish();
+    std::printf("[host]    %-32s ok\n", "GpuMappedRangeLifecycle");
+  }
+
   void CheckStreamBufferRing() {
     EnsureRuntimeContext();
     CommandScheduler scheduler(Renderer(), m_runtime_context);
@@ -1526,7 +1579,7 @@ public:
             fault_memory == reinterpret_cast<void *>(fault_base),
             "fixed processor-fault allocation failed");
     auto &resources = context.GetGpuResources();
-    resources.MapMemory(fault_base, fault_size, GpuAccess::ReadWrite);
+    resources.MapMemory(fault_base, fault_size);
 
     constexpr uint64_t immediate_dst = fault_base + 0x1000;
     constexpr uint64_t immediate_memory_dst = fault_base + 0x2000;
@@ -1683,7 +1736,7 @@ public:
               resources.InvalidateMemory(fault_base, sizeof(uint32_t)),
               "processor memory invalidation did not find its mapped range");
     });
-    resources.UnmapMemory(fault_base, fault_size, GpuAccess::ReadWrite);
+    resources.UnmapMemory(fault_base, fault_size);
     Require("GpuCommandLane", "processor fault unmap",
             Libs::LibKernel::Memory::KernelMunmap(fault_base, fault_size) == 0,
             "processor-fault direct-memory mapping release failed");
@@ -1953,7 +2006,7 @@ public:
       GpuResourceManager resources(m_runtime_context, scheduler);
       resources.SetGpu(&gpu);
       auto &cache = resources.GetBufferCache();
-      resources.MapMemory(base, allocation_size, GpuAccess::ReadWrite);
+      resources.MapMemory(base, allocation_size);
 
       const auto MarkGpuWrite = [&](uint64_t address, uint64_t size) {
         auto allocation =
@@ -2376,7 +2429,7 @@ public:
                                  sizeof(reacquire_value));
 
       resources.SetGpu(nullptr);
-      resources.UnmapMemory(base, allocation_size, GpuAccess::ReadWrite);
+      resources.UnmapMemory(base, allocation_size);
       scheduler.Finish();
     }
     gpu.Shutdown();
@@ -2431,7 +2484,7 @@ public:
               narrow_download != nullptr && narrow_download_offset % 4 == 0 &&
                   wide_download != nullptr && wide_download_offset % 16 == 0,
               "wide/block image readback was not aligned to its texel block");
-      resources.MapMemory(base, allocation_size, GpuAccess::ReadWrite);
+      resources.MapMemory(base, allocation_size);
 
       ImageDesc sampled{};
       sampled.type = BindingType::Texture;
@@ -4772,6 +4825,52 @@ public:
               "successive near-capacity image transfers replaced the shared "
               "download buffer");
 
+      constexpr uint64_t tile_alias_offset = 0x2000000;
+      constexpr uint64_t tile_alias_size = 0x400000;
+      constexpr uint32_t tile_alias_extent = 1024;
+      std::memset(memory + tile_alias_offset, 0,
+                  static_cast<size_t>(tile_alias_size));
+      auto render_target_alias = MakeLinearDesc(
+          base + tile_alias_offset, tile_alias_size,
+          vk::Format::eR8G8B8A8Unorm,
+          Prospero::GpuEnumValue(Prospero::BufferFormat::k8_8_8_8UNorm),
+          Prospero::ImageType::kColor2D,
+          {tile_alias_extent, tile_alias_extent, 1}, 1, 4, 1);
+      render_target_alias.type = BindingType::Storage;
+      render_target_alias.info.tile_mode =
+          Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget);
+      render_target_alias.view_info.usage =
+          vk::ImageUsageFlagBits::eStorage;
+      const auto render_target_alias_image =
+          texture_cache.FindImage(render_target_alias);
+
+      auto standard_4kb_alias = render_target_alias;
+      standard_4kb_alias.type = BindingType::Texture;
+      standard_4kb_alias.info.tile_mode =
+          Prospero::GpuEnumValue(Prospero::TileMode::kStandard4KB);
+      standard_4kb_alias.view_info.usage =
+          vk::ImageUsageFlagBits::eSampled;
+      const auto standard_4kb_alias_image =
+          texture_cache.FindImage(standard_4kb_alias);
+      auto repeated_standard_4kb_alias = standard_4kb_alias;
+      const auto repeated_standard_4kb_alias_image =
+          texture_cache.FindImage(repeated_standard_4kb_alias);
+      Require(
+          name, "equal-size tile-mode alias",
+          render_target_alias_image && standard_4kb_alias_image &&
+              standard_4kb_alias_image != render_target_alias_image &&
+              repeated_standard_4kb_alias_image ==
+                  standard_4kb_alias_image &&
+              texture_cache.GetImage(render_target_alias_image)
+                      .info.tile_mode ==
+                  Prospero::GpuEnumValue(
+                      Prospero::TileMode::kRenderTarget) &&
+              texture_cache.GetImage(standard_4kb_alias_image)
+                      .info.tile_mode ==
+                  Prospero::GpuEnumValue(
+                      Prospero::TileMode::kStandard4KB),
+          "equal address/size lookup reused an incompatible tiled backing");
+
       for (auto &output : ms_observer_outputs) {
         DestroyBuffer(&output);
       }
@@ -4788,7 +4887,7 @@ public:
       m_device.destroyShaderModule(ms_depth_module, nullptr);
 
       resources.SetGpu(nullptr);
-      resources.UnmapMemory(base, allocation_size, GpuAccess::ReadWrite);
+      resources.UnmapMemory(base, allocation_size);
       scheduler.Finish();
     }
     gpu.Shutdown();
@@ -4833,7 +4932,7 @@ public:
     scheduler.Begin(registers, user_config, shaders);
     {
       GpuResourceManager resources(m_runtime_context, scheduler);
-      resources.MapMemory(base, allocation_size, GpuAccess::ReadWrite);
+      resources.MapMemory(base, allocation_size);
       const uint32_t pitch = TileGetTexturePitch(format, 1, 1, tile);
       TileSizeAlign total{};
       TileSizeOffset mip{};
@@ -4932,7 +5031,7 @@ public:
                   std::vector<u32>{0x40004200u, 0x44003c00u},
               "tiled BGRA16 Buffer mirror changed guest component order");
       DestroyBuffer(&mirror_readback);
-      resources.UnmapMemory(base, allocation_size, GpuAccess::ReadWrite);
+      resources.UnmapMemory(base, allocation_size);
       scheduler.Finish();
     }
     Require(name, "unmap",
@@ -4979,7 +5078,7 @@ public:
       auto &resources = context.GetGpuResources();
       auto &texture_cache = resources.GetTextureCache();
       auto &executor = context.GetRenderExecutor();
-      resources.MapMemory(base, allocation_size, GpuAccess::ReadWrite);
+      resources.MapMemory(base, allocation_size);
 
       constexpr auto stencil_format =
           Prospero::GpuEnumValue(Prospero::BufferFormat::k8UInt);
@@ -5965,10 +6064,8 @@ public:
       const auto stale_ordered_color =
           texture_cache.FindImage(ordered_color_desc);
       RenderExecutorTestAccess::BindRenderTarget(executor, stale_ordered_color);
-      resources.UnmapMemory(ordered_color_address, target_mip_size,
-                            GpuAccess::ReadWrite);
-      resources.MapMemory(ordered_color_address, target_mip_size,
-                          GpuAccess::ReadWrite);
+      resources.UnmapMemory(ordered_color_address, target_mip_size);
+      resources.MapMemory(ordered_color_address, target_mip_size);
 
       auto ordered_depth_desc = depth;
       ordered_depth_desc.info.stencil = {ordered_color_address,
@@ -6121,7 +6218,7 @@ public:
               texture_cache.GetImage(depth_id).usage.storage,
           "storage stencil binding did not acquire the associated depth owner");
       RenderExecutorTestAccess::ResetBindings(executor);
-      resources.UnmapMemory(base, allocation_size, GpuAccess::ReadWrite);
+      resources.UnmapMemory(base, allocation_size);
       scheduler.Finish();
     }
 
@@ -12083,6 +12180,84 @@ TestCase BufferLoadVariants() {
            O::BufferLoadDwordx4, O::VMovB32, O::BufferStoreDword, O::SEndpgm}};
 }
 
+TestCase BufferLoadDwordx4SnapshotsOverlappingAddress() {
+  using O = ShaderOpcode;
+
+  std::vector<u32> code;
+  AppendVMovU32(&code, 21, 0);
+  AppendVMovU32(&code, 22, 0);
+  code.push_back(EncodeMubuf0(0x0eu, 0, true, true));
+  code.push_back(EncodeMubuf1(21, 0, 21));
+  for (u32 i = 0; i < 4; i++) {
+    AppendStoreVgpr(&code, 21 + i, 4 + i);
+  }
+  AppendEnd(&code);
+
+  TestCase test;
+  test.name = "BufferLoadDwordx4SnapshotsOverlappingAddress";
+  test.code = std::move(code);
+  test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u,
+                  0,           0,           0,           0};
+  test.expected = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u,
+                   0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u};
+  test.opcodes = {O::VMovB32, O::BufferLoadDwordx4, O::BufferStoreDword,
+                  O::SEndpgm};
+  test.user_data = MakeStructuredStorageBufferData(16, 2);
+  test.has_user_data = true;
+  return test;
+}
+
+TestCase BufferLoadDwordx2SnapshotsOverlappingAddress() {
+  using O = ShaderOpcode;
+
+  std::vector<u32> code;
+  AppendVMovU32(&code, 21, 0);
+  AppendVMovU32(&code, 22, 0);
+  code.push_back(EncodeMubuf0(0x0du, 0, true, true));
+  code.push_back(EncodeMubuf1(21, 0, 21));
+  for (u32 i = 0; i < 2; i++) {
+    AppendStoreVgpr(&code, 21 + i, 2 + i);
+  }
+  AppendEnd(&code);
+
+  TestCase test;
+  test.name = "BufferLoadDwordx2SnapshotsOverlappingAddress";
+  test.code = std::move(code);
+  test.initial = {0x11111111u, 0x22222222u, 0, 0};
+  test.expected = {0x11111111u, 0x22222222u, 0x11111111u, 0x22222222u};
+  test.opcodes = {O::VMovB32, O::BufferLoadDwordx2, O::BufferStoreDword,
+                  O::SEndpgm};
+  test.user_data = MakeStructuredStorageBufferData(8, 2);
+  test.has_user_data = true;
+  return test;
+}
+
+TestCase BufferLoadDwordx3SnapshotsOverlappingAddress() {
+  using O = ShaderOpcode;
+
+  std::vector<u32> code;
+  AppendVMovU32(&code, 21, 0);
+  AppendVMovU32(&code, 22, 0);
+  code.push_back(EncodeMubuf0(0x0fu, 0, true, true));
+  code.push_back(EncodeMubuf1(21, 0, 21));
+  for (u32 i = 0; i < 3; i++) {
+    AppendStoreVgpr(&code, 21 + i, 3 + i);
+  }
+  AppendEnd(&code);
+
+  TestCase test;
+  test.name = "BufferLoadDwordx3SnapshotsOverlappingAddress";
+  test.code = std::move(code);
+  test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0, 0, 0};
+  test.expected = {0x11111111u, 0x22222222u, 0x33333333u,
+                   0x11111111u, 0x22222222u, 0x33333333u};
+  test.opcodes = {O::VMovB32, O::BufferLoadDwordx3, O::BufferStoreDword,
+                  O::SEndpgm};
+  test.user_data = MakeStructuredStorageBufferData(12, 2);
+  test.has_user_data = true;
+  return test;
+}
+
 TestCase BufferStoreVariants() {
   using O = ShaderOpcode;
 
@@ -12151,6 +12326,69 @@ TestCase BufferFormatVariants() {
   return load;
 }
 
+TestCase BufferLoadFormatXyzwSnapshotsOverlappingAddress() {
+  using O = ShaderOpcode;
+
+  std::vector<u32> code;
+  AppendVMovU32(&code, 21, 0);
+  AppendVMovU32(&code, 22, 0);
+  code.push_back(EncodeMubuf0(0x03u, 0, true, true));
+  code.push_back(EncodeMubuf1(21, 0, 21));
+  for (u32 i = 0; i < 4; i++) {
+    AppendStoreVgpr(&code, 21 + i, 4 + i);
+  }
+  AppendEnd(&code);
+
+  TestCase test;
+  test.name = "BufferLoadFormatXyzwSnapshotsOverlappingAddress";
+  test.code = std::move(code);
+  test.initial = {0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u,
+                  0,           0,           0,           0};
+  test.expected = {0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u,
+                   0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u};
+  test.opcodes = {O::VMovB32, O::BufferLoadFormatXyzw, O::BufferStoreDword,
+                  O::SEndpgm};
+  test.user_data = MakeStructuredStorageBufferData(
+      16, 2, false,
+      BufferFormat(Prospero::BufferFormat::k32_32_32_32Float));
+  test.has_user_data = true;
+  return test;
+}
+
+TestCase BufferLoadFormatXyzwInactiveExecPreservesOverlappingAddress() {
+  using O = ShaderOpcode;
+
+  std::vector<u32> code;
+  AppendVMovLiteral(&code, 21, 0x11111111u);
+  AppendVMovLiteral(&code, 22, 0x22222222u);
+  AppendVMovLiteral(&code, 23, 0x33333333u);
+  AppendVMovLiteral(&code, 24, 0x44444444u);
+  code.push_back(EncodeSop1(0x04, 126, InlineU32(0)));
+  code.push_back(EncodeMubuf0(0x03u, 0, true, true));
+  code.push_back(EncodeMubuf1(21, 0, 21));
+  code.push_back(EncodeSMovB32(126, InlineU32(1)));
+  code.push_back(EncodeSMovB32(127, InlineU32(0)));
+  for (u32 i = 0; i < 4; i++) {
+    AppendStoreVgpr(&code, 21 + i, 4 + i);
+  }
+  AppendEnd(&code);
+
+  TestCase test;
+  test.name = "BufferLoadFormatXyzwInactiveExecPreservesOverlappingAddress";
+  test.code = std::move(code);
+  test.initial = {0xaaaaaaaa, 0xbbbbbbbb, 0xcccccccc, 0xdddddddd,
+                  0,          0,          0,          0};
+  test.expected = {0xaaaaaaaau, 0xbbbbbbbbu, 0xccccccccu, 0xddddddddu,
+                   0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u};
+  test.opcodes = {O::VMovB32, O::SMovB64, O::BufferLoadFormatXyzw,
+                  O::SMovB32, O::BufferStoreDword, O::SEndpgm};
+  test.user_data = MakeStructuredStorageBufferData(
+      16, 2, false,
+      BufferFormat(Prospero::BufferFormat::k32_32_32_32Float));
+  test.has_user_data = true;
+  return test;
+}
+
 TestCase BufferFormatStoreVariants() {
   using O = ShaderOpcode;
 
@@ -12459,6 +12697,63 @@ TestCase TBufferLoadVariants() {
            O::BufferStoreDword, O::SEndpgm}};
 }
 
+TestCase TBufferLoadFormatXyzwSnapshotsOverlappingAddress() {
+  using O = ShaderOpcode;
+
+  std::vector<u32> code;
+  AppendVMovU32(&code, 21, 0);
+  AppendVMovU32(&code, 22, 0);
+  constexpr auto format =
+      BufferFormat(Prospero::BufferFormat::k32_32_32_32Float);
+  code.push_back(
+      EncodeMtbuf0(0x03u, format & 0xfu, (format >> 4u) & 0x7u, 0, true, true));
+  code.push_back(EncodeMtbuf1(0x03u, 21, 0, 21));
+  for (u32 i = 0; i < 4; i++) {
+    AppendStoreVgpr(&code, 21 + i, 4 + i);
+  }
+  AppendEnd(&code);
+
+  TestCase test;
+  test.name = "TBufferLoadFormatXyzwSnapshotsOverlappingAddress";
+  test.code = std::move(code);
+  test.initial = {0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u,
+                  0,           0,           0,           0};
+  test.expected = {0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u,
+                   0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u};
+  test.opcodes = {O::VMovB32, O::TBufferLoadFormatXyzw,
+                  O::BufferStoreDword, O::SEndpgm};
+  test.user_data = MakeStructuredStorageBufferData(16, 2);
+  test.has_user_data = true;
+  return test;
+}
+
+TestCase TBufferLoadFormatXyzwPackedSnapshotsOverlappingAddress() {
+  using O = ShaderOpcode;
+
+  std::vector<u32> code;
+  AppendVMovU32(&code, 21, 0);
+  AppendVMovU32(&code, 22, 0);
+  constexpr auto format = BufferFormat(Prospero::BufferFormat::k8_8_8_8UInt);
+  code.push_back(
+      EncodeMtbuf0(0x03u, format & 0xfu, (format >> 4u) & 0x7u, 0, true, true));
+  code.push_back(EncodeMtbuf1(0x03u, 21, 0, 21));
+  for (u32 i = 0; i < 4; i++) {
+    AppendStoreVgpr(&code, 21 + i, 4 + i);
+  }
+  AppendEnd(&code);
+
+  TestCase test;
+  test.name = "TBufferLoadFormatXyzwPackedSnapshotsOverlappingAddress";
+  test.code = std::move(code);
+  test.initial = {0x44332211u, 0, 0, 0, 0, 0, 0, 0};
+  test.expected = {0x44332211u, 0, 0, 0, 0x11u, 0x22u, 0x33u, 0x44u};
+  test.opcodes = {O::VMovB32, O::TBufferLoadFormatXyzw,
+                  O::BufferStoreDword, O::SEndpgm};
+  test.user_data = MakeStructuredStorageBufferData(4, 8);
+  test.has_user_data = true;
+  return test;
+}
+
 TestCase TBufferStoreFormatX8UintWritesOneByte() {
   using O = ShaderOpcode;
 
@@ -13593,23 +13888,36 @@ TestCase DsAppendUsesEncodedGdsSelector() {
   using O = ShaderOpcode;
 
   std::vector<u32> code;
-  AppendSMovLiteral(&code, 124, 0x00000001u);
+  AppendSMovLiteral(&code, 124, 0x00000008u);
   code.push_back(EncodeDs0(0x3e, 0, true));
   code.push_back(EncodeDs1(0, 0, 0));
   code.push_back(EncodeDs0(0x3d, 0, true));
   code.push_back(EncodeDs1(1, 0, 0));
+  code.push_back(EncodeDs0(0x3e, 4, true));
+  code.push_back(EncodeDs1(2, 0, 0));
+  code.push_back(EncodeDs0(0x3d, 4, true));
+  code.push_back(EncodeDs1(3, 0, 0));
+  AppendSMovLiteral(&code, 124, 0x00080008u);
+  code.push_back(EncodeDs0(0x3e, 4, true));
+  code.push_back(EncodeDs1(4, 0, 0));
+  code.push_back(EncodeDs0(0x3d, 4, true));
+  code.push_back(EncodeDs1(5, 0, 0));
   AppendStoreVgpr(&code, 0, 0);
   AppendStoreVgpr(&code, 1, 1);
+  AppendStoreVgpr(&code, 2, 2);
+  AppendStoreVgpr(&code, 3, 3);
+  AppendStoreVgpr(&code, 4, 4);
+  AppendStoreVgpr(&code, 5, 5);
   AppendEnd(&code);
 
   TestCase test{
       "DsAppendGdsSelector",
       code,
       {},
-      {10, 74},
+      {10, 74, 20, 84, 40, 104},
       {O::SMovB32, O::DsAppend, O::DsConsume, O::BufferStoreDword, O::SEndpgm}};
-  test.gds_initial = {10};
-  test.expected_gds = {10};
+  test.gds_initial = {10, 20, 30, 40};
+  test.expected_gds = {10, 20, 30, 40};
   return test;
 }
 
@@ -14780,8 +15088,13 @@ std::vector<TestCase> MakeCases() {
   AddCase(BufferStoreDwordAppliesHostOffset);
   AddCase(BufferOffsetsUsePackedLaneAndStorageFallback);
   AddCase(BufferLoadVariants);
+  AddCase(BufferLoadDwordx2SnapshotsOverlappingAddress);
+  AddCase(BufferLoadDwordx3SnapshotsOverlappingAddress);
+  AddCase(BufferLoadDwordx4SnapshotsOverlappingAddress);
   AddCase(BufferStoreVariants);
   AddCase(BufferFormatVariants);
+  AddCase(BufferLoadFormatXyzwSnapshotsOverlappingAddress);
+  AddCase(BufferLoadFormatXyzwInactiveExecPreservesOverlappingAddress);
   AddCase(BufferFormatStoreVariants);
   AddCase(BufferStoreFormatXResource16UintWritesHalfword);
   AddCase(BufferLoadFormatXResource8UintZeroExtendsByte);
@@ -14795,6 +15108,8 @@ std::vector<TestCase> MakeCases() {
   AddCase(BufferStoreFormatXAddTidUsesLaneIndex);
   AddCase(BufferStoreFormatXDropsOutOfRangeRecord);
   AddCase(TBufferLoadVariants);
+  AddCase(TBufferLoadFormatXyzwSnapshotsOverlappingAddress);
+  AddCase(TBufferLoadFormatXyzwPackedSnapshotsOverlappingAddress);
   AddCase(TBufferLoadFormatX8UintZeroExtendsByte);
   AddCase(TBufferLoadFormatX8888UintExtractsFirstByte);
   AddCase(TBufferLoadFormatXIdxenUsesDescriptorStride);
@@ -15220,7 +15535,7 @@ void CheckRenderTargetFormatContract() {
       resource.kind = ShaderRecompiler::IR::ResourceKind::Image;
     } else if (std::strcmp(kind, "storage-no-write") == 0) {
       resource.written = false;
-    } else if (std::strcmp(kind, "storage-atomic") == 0) {
+    } else if (std::strcmp(kind, "storage-nonuint-atomic") == 0) {
       resource.atomic = true;
     } else if (std::strcmp(kind, "storage-compare") == 0) {
       resource.depth_compare = true;
@@ -15437,6 +15752,12 @@ void CheckSampledColorViews() {
   Require("SampledColorViews", "write-only uint 2D-array storage resource",
           IsSupportedStorageImageResource(storage_resource),
           "basic write-only uint 2D-array storage resource was rejected");
+  storage_resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D;
+  storage_resource.read = true;
+  storage_resource.atomic = true;
+  Require("SampledColorViews", "atomic uint 2D storage resource",
+          IsSupportedStorageImageResource(storage_resource),
+          "atomic uint storage resource was rejected");
 
   char path[MAX_PATH]{};
   Require("SampledColorViews", "host",
@@ -15446,7 +15767,7 @@ void CheckSampledColorViews() {
        {"sampled-invalid-selector", "sampled-incompatible-format",
         "sampled-invalid-high", "sampled-depth-format", "sampled-depth-swizzle",
         "storage-incompatible-format", "storage-kind", "storage-no-write",
-        "storage-atomic", "storage-compare", "storage-mip", "storage-dimension",
+        "storage-nonuint-atomic", "storage-compare", "storage-mip", "storage-dimension",
         "volume-mip-count", "volume-slice-range"}) {
     std::string command =
         std::string("\"") + path + "\" --image-view-death " + kind;
@@ -16155,6 +16476,19 @@ ShaderTextureResource BasicUintVolumeStorageTextureDescriptor() {
            0x00700000u, 0x00000000u, 0x00000000u}};
 }
 
+ShaderRecompiler::IR::ImageResource AtomicStorageTextureResource() {
+  auto resource = BasicLinearStorageTextureResource();
+  resource.kind = ShaderRecompiler::IR::ResourceKind::StorageImageUint;
+  resource.read = true;
+  resource.atomic = true;
+  return resource;
+}
+
+ShaderTextureResource AtomicStorageTextureDescriptor() {
+  return {{0x304bb700u, 0xc1400000u, 0x0000001fu, 0x91b00204u, 0x00000000u,
+           0x00700000u, 0x00000000u, 0x00000000u}};
+}
+
 [[noreturn]] void RunStorageTextureDescriptorDeathCase(const char *kind) {
   auto resource = BasicStorageTextureResource();
   auto descriptor = BasicStorageTextureDescriptor();
@@ -16216,6 +16550,12 @@ ShaderTextureResource BasicUintVolumeStorageTextureDescriptor() {
   } else if (std::strcmp(kind, "uint-resource-float-format") == 0) {
     resource = BasicUintArrayStorageTextureResource();
     descriptor = BasicArrayStorageTextureDescriptor();
+  } else if (std::strcmp(kind, "atomic-format") == 0) {
+    resource = AtomicStorageTextureResource();
+    descriptor = AtomicStorageTextureDescriptor();
+    descriptor.fields[1] =
+        (descriptor.fields[1] & ~0x1ff00000u) |
+        (Prospero::GpuEnumValue(Prospero::BufferFormat::k8UInt) << 20u);
   } else if (std::strcmp(kind, "depth-tile-read") == 0) {
     resource = Ppsa14053DepthTileStorageTextureResource();
     descriptor = Ppsa14053DepthTileStorageTextureDescriptor();
@@ -16299,7 +16639,7 @@ void CheckBasicStorageTextureDescriptor() {
       "PPSA06228 R11G11B10 storage descriptor fixture is malformed");
   ValidateStorageTexture(BasicBgraStorageTextureResource(), r11g11b10,
                          0x870000);
-  ValidateStorageColorView(vk::Format::eB10G11R11UfloatPack32,
+  ValidateStorageColorView(vk::Format::eB8G8R8A8Unorm,
                            vk::Format::eB10G11R11UfloatPack32,
                            r11g11b10.DstSelXYZW());
 
@@ -16590,6 +16930,18 @@ void CheckBasicStorageTextureDescriptor() {
           IsValidImageSwizzle(DstSel(4, 4, 4, 4)),
           "single-channel replicated destination selection was rejected");
 
+  const auto atomic = AtomicStorageTextureDescriptor();
+  Require("BasicStorageTexture", "atomic R32_UINT descriptor",
+          atomic.Width5() + 1u == 128 && atomic.Height5() + 1u == 1 &&
+              atomic.Depth() + 1u == 1 &&
+              atomic.Type() ==
+                  Prospero::GpuEnumValue(Prospero::ImageType::kColor2D) &&
+              atomic.Format() ==
+                  Prospero::GpuEnumValue(Prospero::BufferFormat::k32UInt) &&
+              atomic.DstSelXYZW() == DstSel(4, 0, 0, 1),
+          "PPSA22102 image-atomic descriptor fixture is malformed");
+  ValidateStorageTexture(AtomicStorageTextureResource(), atomic, 0x10000);
+
   char path[MAX_PATH]{};
   Require("BasicStorageTexture", "host",
           GetModuleFileNameA(nullptr, path, MAX_PATH) != 0,
@@ -16598,7 +16950,7 @@ void CheckBasicStorageTextureDescriptor() {
        {"resource", "type", "tile", "mip", "swizzle", "linear-rgb1-read",
         "bgra-read", "r16-float-read", "r8-unorm-read", "yzwx-read",
         "reserved-swizzle", "array-base-out-of-range", "array-mip-view",
-        "reserved", "uint-format", "uint-resource-float-format",
+        "reserved", "uint-format", "uint-resource-float-format", "atomic-format",
         "depth-tile-read", "depth-tile-extent", "depth-tile-fmask"}) {
     std::string command = std::string("\"") + path +
                           "\" --storage-texture-descriptor-death " + kind;
@@ -16897,12 +17249,12 @@ void CheckStandard64RenderTargetTileRoundTrip() {
 void CheckStorageTextureGpuOwnedRebindState() {
   constexpr uintptr_t base = 0x0000000200200000ull;
   constexpr uint64_t size = 0x10000;
-  auto *memory = static_cast<uint8_t *>(
-      VirtualAlloc(reinterpret_cast<void *>(base), size,
-                   MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE));
+  const auto guest_memory = Libs::LibKernel::Memory::AllocateRuntimeMemory(
+      base, size, Common::VirtualMemory::Mode::ReadWrite,
+      "storage_texture_gpu_owned_rebind", true);
+  auto *memory = reinterpret_cast<uint8_t *>(guest_memory);
   Require("StorageTextureGpuOwnedRebind", "allocation",
-          memory == reinterpret_cast<void *>(base),
-          "fixed VirtualAlloc failed");
+          guest_memory == base, "fixed guest-owner allocation failed");
   PageManager page_manager(CacheFault, nullptr);
   MemoryTracker tracker(page_manager);
   page_manager.OnGpuMap(base, size);
@@ -16914,7 +17266,6 @@ void CheckStorageTextureGpuOwnedRebindState() {
   Require(
       "StorageTextureGpuOwnedRebind", "owned",
       tracker.IsRegionGpuModified(base, size) &&
-          page_manager.IsMapped(base, size) &&
           (!HostMemoryQueryReadable(base, size, readable) || readable < size) &&
           HostMemoryQueryRange(base, size, HostMemoryAccess::Mapped, mapped) &&
           mapped == size &&
@@ -16960,7 +17311,8 @@ void CheckStorageTextureGpuOwnedRebindState() {
   tracker.UntrackMemory(base, size);
   page_manager.OnGpuUnmap(base, size);
   Require("StorageTextureGpuOwnedRebind", "free",
-          VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
+          Libs::LibKernel::Memory::FreeGuestMemory(base, size),
+          "guest-owner free failed");
   std::printf("[host]    %-32s ok\n", "StorageTextureGpuOwnedRebind");
 }
 #endif
@@ -17728,6 +18080,11 @@ int main(int argc, char **argv) {
     vulkan.CheckSchedulerTimeline();
     return 0;
   }
+  if (argc == 2 && std::strcmp(argv[1], "--mapped-range-only") == 0) {
+    VulkanHarness vulkan;
+    vulkan.CheckGpuMappedRangeLifecycle();
+    return 0;
+  }
   if (argc == 2 && std::strcmp(argv[1], "--stream-buffer-only") == 0) {
     VulkanHarness vulkan;
     vulkan.CheckStreamBufferRing();
@@ -17883,6 +18240,7 @@ int main(int argc, char **argv) {
   CheckEmbeddedFetchLaneSpill();
   CheckPs5GameExampleImageClearRuntimeShape();
   vulkan.CheckSchedulerTimeline();
+  vulkan.CheckGpuMappedRangeLifecycle();
   vulkan.CheckStreamBufferRing();
   vulkan.CheckCommandPoolGrowth();
   vulkan.CheckGpuTilerCpuParity();

tests/VirtualMemoryAllocationTests.cpp

@@ -1,15 +1,21 @@
 #include "common/commonSubsystem.h"
 #include "common/emulatorConfig.h"
+#include "common/file.h"
 #include "common/logging/log.h"
 #include "common/subsystems.h"
 #include "common/threads.h"
+#include "common/virtualMemory.h"
 #include "kernel/memory.h"
+#include "kernel/pthread.h"
 #include "libs/errno.h"
+#include "loader/runtimeLinker.h"
+#include "loader/systemContent.h"
 
 #include <cinttypes>
 #include <cstdint>
 #include <cstdio>
 #include <cstring>
+#include <filesystem>
 #include <string>
 
 namespace {
@@ -18,11 +24,16 @@ using Libs::LibKernel::Memory::VirtualQueryInfo;
 
 // Prospero ABI?
 constexpr uint64_t SceKernelPageSize             = 0x4000;
+constexpr uint64_t SceKernelTotalPhysicalSize    = 13824ull * 1024ull * 1024ull;
+constexpr uint64_t TestFlexibleMemorySize        = 3072ull * 1024ull * 1024ull;
 constexpr int      SceKernelProtCpuRead          = 0x01;
 constexpr int      SceKernelProtCpuRw            = 0x02;
+constexpr int      SceKernelProtCpuExec          = 0x04;
 constexpr int      SceKernelMapFixed             = 0x10;
 constexpr int      SceKernelMapNoOverwrite       = 0x80;
+constexpr int      SceKernelMapDmemCompat        = 0x400;
 constexpr int      SceKernelMapNoCoalesce        = 0x400000;
+constexpr int      SceKernelMapAligned64Kb       = 16 << 24;
 constexpr int      SceKernelVqFindNext           = 1;
 constexpr int      SceKernelMtypeC               = 11;
 constexpr uint64_t SceKernelDirectMemoryStart    = 0;
@@ -94,6 +105,29 @@ void InitSubsystems() {
 	Config::Load(options);
 
 	slist->Add(log, {core, config});
+	Check("InitSubsystems", slist->InitAll(false), "failed to initialize logging subsystem");
+
+	const auto param_json =
+	    std::filesystem::temp_directory_path() /
+	    ("kyty_virtual_memory_" +
+	     std::to_string(reinterpret_cast<uintptr_t>(&initialized)) + ".json");
+	constexpr char json[] = R"({"kernel":{"flexibleMemorySize":3221225472}})";
+	Common::File  param_file;
+	Check("InitSubsystems", param_file.Create(param_json), "failed to create temporary param.json");
+	uint32_t bytes_written = 0;
+	param_file.Write(json, sizeof(json) - 1, &bytes_written);
+	param_file.Close();
+	Check("InitSubsystems", bytes_written == sizeof(json) - 1,
+	      "failed to write temporary param.json");
+
+	Loader::SystemContentLoadParamSfo(param_json);
+	const auto flexible_memory_size = Loader::SystemContentGetFlexibleMemorySize();
+	Check("InitSubsystems", Common::File::DeleteFile(param_json),
+	      "failed to remove temporary param.json");
+	Check("InitSubsystems", flexible_memory_size == TestFlexibleMemorySize,
+	      "failed to read flexible memory size from param.json");
+	Libs::LibKernel::Memory::SetFlexibleMemorySize(flexible_memory_size);
+
 	slist->Add(memory, {core, log, thread});
 	Check("InitSubsystems", slist->InitAll(false), "failed to initialize memory subsystem");
 
@@ -131,6 +165,13 @@ size_t AvailableFlexibleMemory(const char* test) {
 	return size;
 }
 
+size_t ConfiguredFlexibleMemory(const char* test) {
+	size_t size = 0;
+	CheckOk(test, Libs::LibKernel::Memory::KernelConfiguredFlexibleMemorySize(&size),
+	        "KernelConfiguredFlexibleMemorySize");
+	return size;
+}
+
 uint64_t MapNamedFlexible(const char* test, uint64_t size, int prot, const char* name) {
 	void*     addr = nullptr;
 	const int ret =
@@ -190,6 +231,387 @@ void TestProsperoArgumentAndInfoSizeContracts() {
 	std::printf("[host]    %-48s ok\n", test);
 }
 
+void TestGuestAddressSpaceOwnsReservationsBeforeBacking() {
+	const char* test = "GuestAddressSpaceOwnsReservationsBeforeBacking";
+	void*       addr = nullptr;
+
+	Check(test, Libs::LibKernel::Memory::TestGuestBackingOutsideAddressSpace(),
+	      "boot-time shared backing alias overlaps an owned guest interval");
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelReserveVirtualRange(&addr, SceKernelPageSize, 0,
+	                                                           SceKernelPageSize),
+	        "KernelReserveVirtualRange");
+	const auto base = reinterpret_cast<uint64_t>(addr);
+	Check(test, Libs::LibKernel::Memory::TestGuestAddressRangeIsOwned(base, SceKernelPageSize),
+	      "guest reservation was allocated outside the early owner");
+	Check(test, Libs::LibKernel::Memory::TestPlaceholderRangeIsFree(base, SceKernelPageSize),
+	      "semantic reservation replaced the owner's placeholder");
+	Check(test,
+	      Libs::LibKernel::Memory::ProtectGuestHostMemory(
+	          base, SceKernelPageSize, Common::VirtualMemory::Mode::NoAccess),
+	      "owner rejected a sparse placeholder protection no-op");
+	CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(base, SceKernelPageSize), "KernelMunmap");
+	Check(test, Libs::LibKernel::Memory::TestPlaceholderRangeIsFree(base, SceKernelPageSize),
+	      "released semantic reservation escaped owner control");
+
+	std::printf("[host]    %-48s ok\n", test);
+}
+
+void TestGuestAddressSpaceHasNoFixedFallback() {
+	const char* test            = "GuestAddressSpaceHasNoFixedFallback";
+	const auto  unowned_address = reinterpret_cast<void*>(0x10000);
+	void*       addr            = unowned_address;
+
+	CheckFailed(test,
+	            Libs::LibKernel::Memory::KernelReserveVirtualRange(
+	                &addr, SceKernelPageSize, SceKernelMapFixed | SceKernelMapNoOverwrite,
+	                SceKernelPageSize),
+	            "KernelReserveVirtualRange(unowned fixed address)");
+	Check(test, reinterpret_cast<uint64_t>(addr) == 0x10000,
+	      "failed fixed reservation unexpectedly moved");
+
+	addr = unowned_address;
+	CheckFailed(test,
+	            Libs::LibKernel::Memory::KernelMapNamedFlexibleMemory(
+	                &addr, SceKernelPageSize, SceKernelProtCpuRw,
+	                SceKernelMapFixed | SceKernelMapNoOverwrite, "unowned_flexible"),
+	            "KernelMapNamedFlexibleMemory(unowned fixed address)");
+
+	int64_t phys_addr = -1;
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelAllocateDirectMemory(
+	            0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(), SceKernelPageSize,
+	            SceKernelPageSize, SceKernelMtypeC, &phys_addr),
+	        "KernelAllocateDirectMemory");
+	addr = unowned_address;
+	CheckFailed(test,
+	            Libs::LibKernel::Memory::KernelMapNamedDirectMemory(
+	                &addr, SceKernelPageSize, SceKernelProtCpuRw,
+	                SceKernelMapFixed | SceKernelMapNoOverwrite, phys_addr, SceKernelPageSize,
+	                "unowned_direct"),
+	            "KernelMapNamedDirectMemory(unowned fixed address)");
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelCheckedReleaseDirectMemory(phys_addr, SceKernelPageSize),
+	        "KernelCheckedReleaseDirectMemory");
+
+	std::printf("[host]    %-48s ok\n", test);
+}
+
+void TestGuestFreeRangeSearchDoesNotUnderflow() {
+	const char* test = "GuestFreeRangeSearchDoesNotUnderflow";
+
+	Check(test, Libs::LibKernel::Memory::TestGuestFreeRangeBounds(),
+	      "free-range containment accepted a candidate beyond the range end");
+
+	std::printf("[host]    %-48s ok\n", test);
+}
+
+void TestFlexibleMemoryCapacityIsBootFixed() {
+	const char* test       = "FlexibleMemoryCapacityIsBootFixed";
+	const auto  configured = ConfiguredFlexibleMemory(test);
+	const auto  baseline   = AvailableFlexibleMemory(test);
+	const auto  backing    = Libs::LibKernel::Memory::TestGuestBackingSize();
+
+	Check(test, configured == TestFlexibleMemorySize,
+	      "boot flexible pool did not use the param.json value");
+	Check(test, configured == baseline, "boot flexible pool did not start at configured capacity");
+	Check(test, backing == SceKernelTotalPhysicalSize,
+	      "boot backing is not the single 13.5 GiB physical file");
+	Check(test, backing == Libs::LibKernel::Memory::KernelGetDirectMemorySize() + configured,
+	      "direct and flexible regions do not partition the boot backing");
+
+	const auto address =
+	    MapNamedFlexible(test, SceKernelPageSize, SceKernelProtCpuRw, "boot_fixed_flexible");
+	Check(test, ConfiguredFlexibleMemory(test) == configured,
+	      "configured flexible capacity changed after allocation");
+	Check(test, Libs::LibKernel::Memory::TestGuestBackingSize() == backing,
+	      "shared backing size changed after allocation");
+	Check(test, AvailableFlexibleMemory(test) == baseline - SceKernelPageSize,
+	      "flexible allocation did not consume the boot-time pool");
+
+	CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(address, SceKernelPageSize),
+	        "KernelMunmap");
+	Check(test, ConfiguredFlexibleMemory(test) == configured,
+	      "configured flexible capacity changed after release");
+	Check(test, AvailableFlexibleMemory(test) == baseline,
+	      "flexible release did not restore the boot-time pool");
+
+	std::printf("[host]    %-48s ok\n", test);
+}
+
+void TestFlexibleMemoryUsesSharedBacking() {
+	const char* test     = "FlexibleMemoryUsesSharedBacking";
+	const auto  baseline = AvailableFlexibleMemory(test);
+	void*       address  = nullptr;
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelMapNamedFlexibleMemory(
+	            &address, SceKernelPageSize * 2, SceKernelProtCpuRw, 0, "shared_flexible"),
+	        "KernelMapNamedFlexibleMemory");
+	const auto base = reinterpret_cast<uint64_t>(address);
+	Check(test, Libs::LibKernel::Memory::TestGuestAddressRangeIsOwned(base, SceKernelPageSize * 2),
+	      "flexible mapping escaped the guest owner");
+
+	constexpr uint64_t first_value     = 0x464c45584241434bull; // "FLEXBACK"
+	constexpr uint64_t second_value    = 0x534841524544464cull; // "SHAREDFL"
+	*reinterpret_cast<uint64_t*>(base) = first_value;
+	uint64_t value                     = 0;
+	Check(test, Libs::LibKernel::Memory::TryReadBacking(base, &value, sizeof(value)),
+	      "TryReadBacking did not resolve flexible memory");
+	Check(test, value == first_value, "backing did not observe a flexible-memory CPU write");
+	Check(test,
+	      Libs::LibKernel::Memory::TryWriteBacking(base + SceKernelPageSize, &second_value,
+	                                               sizeof(second_value)),
+	      "TryWriteBacking did not resolve flexible memory");
+	Check(test, *reinterpret_cast<uint64_t*>(base + SceKernelPageSize) == second_value,
+	      "flexible-memory view did not observe a backing write");
+
+	CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(base, SceKernelPageSize * 2),
+	        "KernelMunmap");
+	Check(test, AvailableFlexibleMemory(test) == baseline,
+	      "flexible backing offsets were not returned to the boot-time pool");
+	Check(test, !Libs::LibKernel::Memory::TryReadBacking(base, &value, sizeof(value)),
+	      "unmapped flexible memory remained registered in the backing owner");
+
+	std::printf("[host]    %-48s ok\n", test);
+}
+
+void TestFlexibleDmemCompatAndAlignmentFlags() {
+	const char* test     = "FlexibleDmemCompatAndAlignmentFlags";
+	const auto  baseline = AvailableFlexibleMemory(test);
+	void*       address  = nullptr;
+
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelMapNamedFlexibleMemory(
+	            &address, SceKernelPageSize, SceKernelProtCpuRw,
+	            SceKernelMapDmemCompat | SceKernelMapAligned64Kb, "dmem_compat"),
+	        "KernelMapNamedFlexibleMemory(DMEM_COMPAT|ALIGNED_64KB)");
+	const auto base = reinterpret_cast<uint64_t>(address);
+	Check(test, (base & (0x10000 - 1u)) == 0, "SDK alignment flag was not honored");
+	const auto info = Query(test, base);
+	Check(test, info.is_flexible == 1 && info.is_stack == 0,
+	      "SCE_KERNEL_MAP_DMEM_COMPAT was misclassified as MAP_STACK");
+	Check(test, AvailableFlexibleMemory(test) + SceKernelPageSize == baseline,
+	      "DMEM_COMPAT mapping did not consume boot-time flexible backing");
+
+	void* stack_start = reinterpret_cast<void*>(UINT64_MAX);
+	void* stack_end   = reinterpret_cast<void*>(UINT64_MAX);
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelIsStack(reinterpret_cast<void*>(base), &stack_start,
+	                                               &stack_end),
+	        "KernelIsStack");
+	Check(test, stack_start == nullptr && stack_end == nullptr,
+	      "DMEM_COMPAT flexible mapping was reported as a stack");
+
+	CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(base, SceKernelPageSize),
+	        "KernelMunmap");
+	Check(test, AvailableFlexibleMemory(test) == baseline,
+	      "DMEM_COMPAT cleanup did not restore flexible capacity");
+
+	void* opaque = nullptr;
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelMapNamedFlexibleMemory(
+	            &opaque, SceKernelPageSize, SceKernelProtCpuRw, 0x8000, "opaque_runtime_flag"),
+	        "KernelMapNamedFlexibleMemory(opaque runtime flag)");
+	Check(test, AvailableFlexibleMemory(test) + SceKernelPageSize == baseline,
+	      "opaque runtime flag mapping did not consume boot-time flexible backing");
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelMunmap(reinterpret_cast<uint64_t>(opaque),
+	                                              SceKernelPageSize),
+	        "KernelMunmap(opaque runtime flag)");
+	Check(test, AvailableFlexibleMemory(test) == baseline,
+	      "opaque runtime flag cleanup did not restore flexible capacity");
+
+	void* invalid_flag = nullptr;
+	CheckFailed(
+	    test,
+	    Libs::LibKernel::Memory::KernelMapNamedFlexibleMemory(
+	        &invalid_flag, SceKernelPageSize, SceKernelProtCpuRw, 0x10000, "unsupported_flag"),
+	    "KernelMapNamedFlexibleMemory(unsupported flag)");
+
+	void* invalid_alignment = nullptr;
+	CheckFailed(test,
+	            Libs::LibKernel::Memory::KernelMapNamedFlexibleMemory(
+	                &invalid_alignment, SceKernelPageSize, SceKernelProtCpuRw, 13 << 24,
+	                "invalid_alignment"),
+	            "KernelMapNamedFlexibleMemory(invalid alignment)");
+
+	std::printf("[host]    %-48s ok\n", test);
+}
+
+void TestFlexibleNoCoalescePreservesBoundaries() {
+	const char* test     = "FlexibleNoCoalescePreservesBoundaries";
+	const auto  baseline = AvailableFlexibleMemory(test);
+	void*       reserve  = nullptr;
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelReserveVirtualRange(
+	            &reserve, SceKernelPageSize * 2, 0, SceKernelPageSize),
+	        "KernelReserveVirtualRange");
+	const auto base = reinterpret_cast<uint64_t>(reserve);
+
+	void* left = reinterpret_cast<void*>(base);
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelMapNamedFlexibleMemory(
+	            &left, SceKernelPageSize, SceKernelProtCpuRw,
+	            SceKernelMapFixed | SceKernelMapNoCoalesce, "no_coalesce"),
+	        "KernelMapNamedFlexibleMemory(left)");
+	void* right = reinterpret_cast<void*>(base + SceKernelPageSize);
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelMapNamedFlexibleMemory(
+	            &right, SceKernelPageSize, SceKernelProtCpuRw,
+	            SceKernelMapFixed | SceKernelMapNoCoalesce, "no_coalesce"),
+	        "KernelMapNamedFlexibleMemory(right)");
+
+	ExpectRange(test, Query(test, base), base, base + SceKernelPageSize, SceKernelProtCpuRw, 1, 0,
+	            0, 1, "no_coalesce");
+	ExpectRange(test, Query(test, base + SceKernelPageSize), base + SceKernelPageSize,
+	            base + SceKernelPageSize * 2, SceKernelProtCpuRw, 1, 0, 0, 1, "no_coalesce");
+
+	CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(base, SceKernelPageSize * 2),
+	        "KernelMunmap");
+	Check(test, AvailableFlexibleMemory(test) == baseline,
+	      "NO_COALESCE cleanup did not restore flexible capacity");
+
+	std::printf("[host]    %-48s ok\n", test);
+}
+
+void TestFlexibleMemoryReuseIsZeroFilled() {
+	const char* test     = "FlexibleMemoryReuseIsZeroFilled";
+	const auto  baseline = AvailableFlexibleMemory(test);
+	const auto  first =
+	    MapNamedFlexible(test, SceKernelPageSize, SceKernelProtCpuRw, "flexible_zero_source");
+	std::memset(reinterpret_cast<void*>(first), 0xa5, SceKernelPageSize);
+	CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(first, SceKernelPageSize),
+	        "KernelMunmap(source)");
+
+	const auto reused =
+	    MapNamedFlexible(test, SceKernelPageSize, SceKernelProtCpuRw, "flexible_zero_reuse");
+	const auto* bytes = reinterpret_cast<const uint8_t*>(reused);
+	Check(test,
+	      std::all_of(bytes, bytes + SceKernelPageSize, [](uint8_t value) { return value == 0; }),
+	      "reused flexible backing exposed stale bytes");
+	CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(reused, SceKernelPageSize),
+	        "KernelMunmap(reuse)");
+	Check(test, AvailableFlexibleMemory(test) == baseline,
+	      "zero-fill test leaked flexible backing capacity");
+
+	std::printf("[host]    %-48s ok\n", test);
+}
+
+void TestGuestStackUsesPrivateOwnerMemoryAndCache() {
+	const char* test     = "GuestStackUsesPrivateOwnerMemoryAndCache";
+	const auto  baseline = AvailableFlexibleMemory(test);
+	uint64_t    first    = 0;
+	uint64_t    second   = 0;
+	uint64_t    map_size = 0;
+
+	Check(test, Libs::LibKernel::TestGuestStackOwnerLifecycle(&first, &second, &map_size),
+	      "guest stack owner lifecycle failed");
+	Check(test, first != 0 && first == second, "guest stack cache did not reuse its owner mapping");
+	Check(test, map_size != 0 && (map_size & (SceKernelPageSize - 1u)) == 0,
+	      "guest stack mapping is not 16 KiB aligned");
+	Check(test, AvailableFlexibleMemory(test) == baseline,
+	      "private guest stack changed flexible backing capacity");
+
+	std::printf("[host]    %-48s ok\n", test);
+}
+
+void TestMainEntryUsesGuestStackAndDisablesHostChecks() {
+	const char* test = "MainEntryUsesGuestStackAndDisablesHostChecks";
+
+	Check(test, Loader::TestMainEntryUsesGuestStack(),
+	      "main-entry stack switch did not preserve the guest/host stack invariants");
+
+	std::printf("[host]    %-48s ok\n", test);
+}
+
+void TestFragmentedBackingUnmapRollback() {
+	const char* test     = "FragmentedBackingUnmapRollback";
+	const auto  baseline = AvailableFlexibleMemory(test);
+	const auto  left =
+	    MapNamedFlexible(test, SceKernelPageSize, SceKernelProtCpuRw, "backing_hole_left");
+	const auto blocker =
+	    MapNamedFlexible(test, SceKernelPageSize, SceKernelProtCpuRw, "backing_blocker");
+	const auto right =
+	    MapNamedFlexible(test, SceKernelPageSize, SceKernelProtCpuRw, "backing_hole_right");
+	CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(left, SceKernelPageSize),
+	        "KernelMunmap(left hole)");
+	CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(right, SceKernelPageSize),
+	        "KernelMunmap(right hole)");
+
+	const auto fragmented =
+	    MapNamedFlexible(test, SceKernelPageSize * 2, SceKernelProtCpuRw, "fragmented_backing");
+	auto* first_word = reinterpret_cast<uint64_t*>(fragmented);
+	auto* last_word =
+	    reinterpret_cast<uint64_t*>(fragmented + SceKernelPageSize * 2 - sizeof(uint64_t));
+	*first_word = 0x465241474c454654ull; // "FRAGLEFT"
+	*last_word  = 0x4652414752474854ull; // "FRAGRGHT"
+
+	Libs::LibKernel::Memory::TestFailGuestBackingStoreUnmapAfter(1);
+	CheckFailed(test, Libs::LibKernel::Memory::KernelMunmap(fragmented, SceKernelPageSize * 2),
+	            "KernelMunmap(injected second-view failure)");
+	ExpectRange(test, Query(test, fragmented), fragmented, fragmented + SceKernelPageSize * 2,
+	            SceKernelProtCpuRw, 1, 0, 0, 1, "fragmented_backing");
+	Check(test, *first_word == 0x465241474c454654ull && *last_word == 0x4652414752474854ull,
+	      "transactional backing-unmap rollback lost mapped contents");
+
+	CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(fragmented, SceKernelPageSize * 2),
+	        "KernelMunmap(retry)");
+	CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(blocker, SceKernelPageSize),
+	        "KernelMunmap(blocker)");
+	Check(test, AvailableFlexibleMemory(test) == baseline,
+	      "fragmented backing rollback test leaked flexible capacity");
+
+	std::printf("[host]    %-48s ok\n", test);
+}
+
+void TestRuntimeMemoryOwnerLifecycle() {
+	const char* test = "RuntimeMemoryOwnerLifecycle";
+	Check(test,
+	      Libs::LibKernel::Memory::AllocateRuntimeMemory(0x10000, SceKernelPageSize,
+	                                                     Common::VirtualMemory::Mode::ReadWrite,
+	                                                     "runtime_outside_owner", true) == 0,
+	      "fixed runtime allocation escaped the guest owner");
+
+	const auto base = Libs::LibKernel::Memory::AllocateRuntimeMemory(
+	    0, SceKernelPageSize * 2, Common::VirtualMemory::Mode::ReadWrite, "runtime_lifecycle");
+	Check(test, base != 0, "runtime allocation failed");
+	Check(test, Libs::LibKernel::Memory::TestGuestAddressRangeIsOwned(base, SceKernelPageSize * 2),
+	      "runtime allocation is outside the owner");
+	*reinterpret_cast<uint64_t*>(base) = 0x52554e54494d454full; // "RUNTIMEO"
+	Check(test,
+	      Libs::LibKernel::Memory::ProtectGuestMemory(base, SceKernelPageSize,
+	                                                  Common::VirtualMemory::Mode::Read),
+	      "runtime protection failed");
+	Check(test, Libs::LibKernel::Memory::FreeGuestMemory(base, SceKernelPageSize * 2),
+	      "runtime free failed");
+	Check(test, Libs::LibKernel::Memory::TestPlaceholderRangeIsFree(base, SceKernelPageSize * 2),
+	      "runtime free did not restore the owner placeholder");
+
+	const auto reused = Libs::LibKernel::Memory::AllocateRuntimeMemory(
+	    base, SceKernelPageSize * 2, Common::VirtualMemory::Mode::ReadWrite, "runtime_reuse", true);
+	Check(test, reused == base, "fixed runtime allocation did not reuse the owner placeholder");
+	Check(test, Libs::LibKernel::Memory::FreeGuestMemory(reused, SceKernelPageSize * 2),
+	      "reused runtime free failed");
+
+	const auto adjacent_first = Libs::LibKernel::Memory::AllocateRuntimeMemory(
+	    0, SceKernelPageSize, Common::VirtualMemory::Mode::ReadWrite, "runtime_adjacent_first");
+	Check(test, adjacent_first != 0, "first adjacent runtime allocation failed");
+	const auto adjacent_second = Libs::LibKernel::Memory::AllocateRuntimeMemory(
+	    adjacent_first + SceKernelPageSize, SceKernelPageSize,
+	    Common::VirtualMemory::Mode::ReadWrite, "runtime_adjacent_second", true);
+	Check(test, adjacent_second == adjacent_first + SceKernelPageSize,
+	      "second adjacent runtime allocation failed");
+	Check(test,
+	      Libs::LibKernel::Memory::FreeGuestMemory(adjacent_first, SceKernelPageSize * 2),
+	      "combined adjacent runtime free failed");
+	Check(test,
+	      Libs::LibKernel::Memory::TestPlaceholderRangeIsFree(adjacent_first,
+	                                                          SceKernelPageSize * 2),
+	      "combined adjacent runtime free did not restore one owner placeholder");
+
+	std::printf("[host]    %-48s ok\n", test);
+}
+
 void TestFlexibleMapQueryAndWholeMunmap() {
 	const char* test     = "FlexibleMapQueryAndWholeMunmap";
 	const auto  baseline = AvailableFlexibleMemory(test);
@@ -341,9 +763,11 @@ void TestDirectMapQueryOffsetAndPartialMunmap() {
 	            &addr, SceKernelPageSize * 4, SceKernelProtCpuRw, 0, phys_addr, SceKernelPageSize,
 	            "prospero_direct"),
 	        "KernelMapNamedDirectMemory");
-	const auto base  = reinterpret_cast<uint64_t>(addr);
-	const auto phys  = static_cast<uint64_t>(phys_addr);
-	void*      alias = nullptr;
+	const auto base = reinterpret_cast<uint64_t>(addr);
+	const auto phys = static_cast<uint64_t>(phys_addr);
+	Check(test, Libs::LibKernel::Memory::TestGuestAddressRangeIsOwned(base, SceKernelPageSize * 4),
+	      "direct mapping escaped the guest owner");
+	void* alias = nullptr;
 	CheckOk(test,
 	        Libs::LibKernel::Memory::KernelMapNamedDirectMemory(
 	            &alias, SceKernelPageSize * 4, SceKernelProtCpuRw, 0, phys_addr, SceKernelPageSize,
@@ -405,6 +829,10 @@ void TestDirectMapQueryOffsetAndPartialMunmap() {
 	      "TryReadBacking should reject a range crossing an unmapped span");
 	Check(test, rejected_read == transaction_sentinel,
 	      "failed backing reads must not modify a destination prefix");
+	Check(test,
+	      Libs::LibKernel::Memory::ClampRangeSize(base + SceKernelPageSize - 0xf30, 0x1560) ==
+	          0xf30,
+	      "ClampRangeSize did not stop at an unmapped span");
 
 	info = Query(test, base + SceKernelPageSize, SceKernelVqFindNext);
 	ExpectRange(test, info, base + SceKernelPageSize * 2, base + SceKernelPageSize * 4,
@@ -426,6 +854,192 @@ void TestDirectMapQueryOffsetAndPartialMunmap() {
 	std::printf("[host]    %-48s ok\n", test);
 }
 
+void TestDirectPartialProtectUnmapPreservesNeighbors() {
+	const char* test      = "DirectPartialProtectUnmapPreservesNeighbors";
+	const auto  size      = SceKernelPageSize * 3;
+	int64_t     phys_addr = 0;
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelAllocateDirectMemory(
+	            0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(), size, SceKernelPageSize,
+	            SceKernelMtypeC, &phys_addr),
+	        "KernelAllocateDirectMemory");
+
+	void* address = nullptr;
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelMapNamedDirectMemory(&address, size, SceKernelProtCpuRw,
+	                                                            0, phys_addr, SceKernelPageSize,
+	                                                            "partial_protect_direct"),
+	        "KernelMapNamedDirectMemory");
+	const auto base = reinterpret_cast<uint64_t>(address);
+	CheckOk(
+	    test,
+	    Libs::LibKernel::Memory::KernelMprotect(reinterpret_cast<void*>(base + SceKernelPageSize),
+	                                            SceKernelPageSize, SceKernelProtCpuRead),
+	    "KernelMprotect(middle)");
+	Check(test,
+	      Libs::LibKernel::Memory::ProtectGuestHostMemory(
+	          base, size, Common::VirtualMemory::Mode::Read),
+	      "owner could not protect fragmented backing views");
+	Check(test,
+	      Libs::LibKernel::Memory::ProtectGuestHostMemory(
+	          base, size, Common::VirtualMemory::Mode::ReadWrite),
+	      "owner could not restore fragmented backing views");
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelMunmap(base + SceKernelPageSize, SceKernelPageSize),
+	        "KernelMunmap(middle)");
+
+	Common::VirtualMemory::Mode old_left {};
+	Common::VirtualMemory::Mode old_right {};
+	Check(test,
+	      Common::VirtualMemory::Protect(base, SceKernelPageSize,
+	                                     Common::VirtualMemory::Mode::ReadWrite, &old_left),
+	      "could not inspect left-page protection");
+	Check(test,
+	      Common::VirtualMemory::Protect(base + SceKernelPageSize * 2, SceKernelPageSize,
+	                                     Common::VirtualMemory::Mode::ReadWrite, &old_right),
+	      "could not inspect right-page protection");
+	Check(test, old_left == Common::VirtualMemory::Mode::ReadWrite,
+	      "partial unmap changed the left neighbor protection");
+	Check(test, old_right == Common::VirtualMemory::Mode::ReadWrite,
+	      "partial unmap changed the right neighbor protection");
+	*reinterpret_cast<uint64_t*>(base) = 0x4c45465450524f54ull; // "LEFTPROT"
+	*reinterpret_cast<uint64_t*>(base + SceKernelPageSize * 2) =
+	    0x5247485450524f54ull; // "RGHTPROT"
+
+	CheckOk(test, Libs::LibKernel::Memory::KernelReleaseDirectMemory(phys_addr, size),
+	        "KernelReleaseDirectMemory");
+	ExpectUnmapped(test, base);
+	ExpectUnmapped(test, base + SceKernelPageSize * 2);
+
+	std::printf("[host]    %-48s ok\n", test);
+}
+
+void TestDirectMapValidationBeforeOwnerMutation() {
+	const char* test    = "DirectMapValidationBeforeOwnerMutation";
+	int64_t     invalid = -1;
+	CheckFailed(test,
+	            Libs::LibKernel::Memory::KernelAllocateDirectMemory(
+	                0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(), SceKernelPageSize + 1,
+	                SceKernelPageSize, SceKernelMtypeC, &invalid),
+	            "KernelAllocateDirectMemory(unaligned size)");
+	Check(test, invalid == -1, "invalid direct allocation changed the output address");
+	CheckFailed(test,
+	            Libs::LibKernel::Memory::KernelAllocateDirectMemory(
+	                0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(), SceKernelPageSize,
+	                0x1000, SceKernelMtypeC, &invalid),
+	            "KernelAllocateDirectMemory(sub-page alignment)");
+	Check(test, invalid == -1, "invalid alignment changed the output address");
+
+	int64_t phys_addr = 0;
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelAllocateDirectMemory(
+	            0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(), SceKernelPageSize * 2,
+	            SceKernelPageSize, SceKernelMtypeC, &phys_addr),
+	        "KernelAllocateDirectMemory");
+
+	auto expect_invalid = [&](size_t len, int prot, int flags, int64_t phys, size_t alignment,
+	                          const char* action) {
+		void* address = nullptr;
+		CheckFailed(test,
+		            Libs::LibKernel::Memory::KernelMapDirectMemory(&address, len, prot, flags, phys,
+		                                                           alignment),
+		            action);
+		Check(test, address == nullptr, "invalid direct map changed the output address");
+	};
+	expect_invalid(SceKernelPageSize + 1, SceKernelProtCpuRw, 0, phys_addr, SceKernelPageSize,
+	               "KernelMapDirectMemory(unaligned size)");
+	expect_invalid(SceKernelPageSize, SceKernelProtCpuRw, 0, phys_addr + 1, SceKernelPageSize,
+	               "KernelMapDirectMemory(unaligned physical address)");
+	expect_invalid(SceKernelPageSize, SceKernelProtCpuExec, 0, phys_addr, SceKernelPageSize,
+	               "KernelMapDirectMemory(executable)");
+
+	void* aligned = nullptr;
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelMapDirectMemory(
+	            &aligned, SceKernelPageSize, SceKernelProtCpuRw, 0, phys_addr, 0xc000),
+	        "KernelMapDirectMemory(16K-multiple alignment)");
+	Check(test, reinterpret_cast<uint64_t>(aligned) % 0xc000 == 0,
+	      "non-power-of-two 16K alignment was not honored");
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelMunmap(reinterpret_cast<uint64_t>(aligned),
+	                                              SceKernelPageSize),
+	        "KernelMunmap(16K-multiple alignment)");
+
+	void* ignored_flag = nullptr;
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelMapDirectMemory(&ignored_flag, SceKernelPageSize,
+	                                                       SceKernelProtCpuRw, 0x08, phys_addr,
+	                                                       SceKernelPageSize),
+	        "KernelMapDirectMemory(ignored flag)");
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelMunmap(reinterpret_cast<uint64_t>(ignored_flag),
+	                                              SceKernelPageSize),
+	        "KernelMunmap(ignored flag)");
+
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelReleaseDirectMemory(phys_addr, SceKernelPageSize * 2),
+	        "KernelReleaseDirectMemory");
+	std::printf("[host]    %-48s ok\n", test);
+}
+
+void TestDirectReleaseRollbackRestoresOwnerMapping() {
+	const char* test      = "DirectReleaseRollbackRestoresOwnerMapping";
+	int64_t     phys_addr = 0;
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelAllocateDirectMemory(
+	            0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(), SceKernelPageSize,
+	            SceKernelPageSize, SceKernelMtypeC, &phys_addr),
+	        "KernelAllocateDirectMemory");
+	void* address = nullptr;
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelMapNamedDirectMemory(
+	            &address, SceKernelPageSize, SceKernelProtCpuRw, 0, phys_addr, SceKernelPageSize,
+	            "release_rollback"),
+	        "KernelMapNamedDirectMemory");
+	const auto base                    = reinterpret_cast<uint64_t>(address);
+	*reinterpret_cast<uint64_t*>(base) = 0x52454c524f4c4c42ull; // "RELROLLB"
+
+	Libs::LibKernel::Memory::TestFailNextPhysicalMemoryUnmap();
+	CheckFailed(
+	    test,
+	    Libs::LibKernel::Memory::KernelCheckedReleaseDirectMemory(phys_addr, SceKernelPageSize),
+	    "KernelCheckedReleaseDirectMemory(injected failure)");
+	ExpectRange(test, Query(test, base), base, base + SceKernelPageSize, SceKernelProtCpuRw, 0, 1,
+	            0, 1, "release_rollback", static_cast<uint64_t>(phys_addr));
+	Check(test, *reinterpret_cast<uint64_t*>(base) == 0x52454c524f4c4c42ull,
+	      "release rollback lost the shared-backing contents");
+
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelCheckedReleaseDirectMemory(phys_addr, SceKernelPageSize),
+	        "KernelCheckedReleaseDirectMemory(retry)");
+	ExpectUnmapped(test, base);
+	std::printf("[host]    %-48s ok\n", test);
+}
+
+void TestDirectReleaseContracts() {
+	const char* test = "DirectReleaseContracts";
+	CheckOk(test, Libs::LibKernel::Memory::KernelReleaseDirectMemory(0, 0),
+	        "KernelReleaseDirectMemory(zero length)");
+	CheckOk(test, Libs::LibKernel::Memory::KernelCheckedReleaseDirectMemory(0, 0),
+	        "KernelCheckedReleaseDirectMemory(zero length)");
+	CheckFailed(test, Libs::LibKernel::Memory::KernelReleaseDirectMemory(1, SceKernelPageSize),
+	            "KernelReleaseDirectMemory(unaligned start)");
+	CheckFailed(test, Libs::LibKernel::Memory::KernelReleaseDirectMemory(0, SceKernelPageSize + 1),
+	            "KernelReleaseDirectMemory(unaligned size)");
+
+	const auto free_offset = static_cast<int64_t>(
+	    Libs::LibKernel::Memory::KernelGetDirectMemorySize() - SceKernelPageSize);
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelReleaseDirectMemory(free_offset, SceKernelPageSize),
+	        "KernelReleaseDirectMemory(unallocated range)");
+	Check(test,
+	      Libs::LibKernel::Memory::KernelCheckedReleaseDirectMemory(
+	          free_offset, SceKernelPageSize) == Libs::LibKernel::KERNEL_ERROR_ENOENT,
+	      "checked release did not report an unallocated range");
+
+	std::printf("[host]    %-48s ok\n", test);
+}
+
 void TestReleasedReserveCanBeReused() {
 	const char* test = "ReleasedReserveCanBeReused";
 	void*       addr = nullptr;
@@ -473,17 +1087,24 @@ void TestMunmapAcrossAdjacentFlexibleMappings() {
 	            &right, SceKernelPageSize, SceKernelProtCpuRw, SceKernelMapFixed, "adjacent_right"),
 	        "KernelMapNamedFlexibleMemory(right)");
 
+	Check(test,
+	      Libs::LibKernel::Memory::ClampRangeSize(base + SceKernelPageSize - 0x100, 0x200) == 0x200,
+	      "ClampRangeSize did not cross adjacent committed mappings");
+	Check(test,
+	      Libs::LibKernel::Memory::ProtectGuestHostMemory(
+	          base, SceKernelPageSize * 2, Common::VirtualMemory::Mode::Read),
+	      "owner could not protect adjacent backing mappings");
+	Check(test,
+	      Libs::LibKernel::Memory::ProtectGuestHostMemory(
+	          base, SceKernelPageSize * 2, Common::VirtualMemory::Mode::ReadWrite),
+	      "owner could not restore adjacent backing mappings");
+
 	CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(base, SceKernelPageSize * 2),
 	        "KernelMunmap(adjacent mappings)");
 	Check(test, AvailableFlexibleMemory(test) == baseline,
 	      "multi-range unmap leaked flexible-memory budget");
-	ExpectRange(test, Query(test, base), base, base + SceKernelPageSize, 0, 0, 0, 0, 0,
-	            "adjacent_left");
-	ExpectRange(test, Query(test, base + SceKernelPageSize), base + SceKernelPageSize,
-	            base + SceKernelPageSize * 2, 0, 0, 0, 0, 0, "adjacent_right");
-
-	CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(base, SceKernelPageSize * 2),
-	        "KernelMunmap(restored reserve)");
+	ExpectUnmapped(test, base);
+	ExpectUnmapped(test, base + SceKernelPageSize);
 
 	std::printf("[host]    %-48s ok\n", test);
 }
@@ -539,6 +1160,52 @@ void TestNonzeroDirectOffsetAliasesSharedBacking() {
 	std::printf("[host]    %-48s ok\n", test);
 }
 
+void TestDirectMapAcrossContiguousAllocations() {
+	const char* test   = "DirectMapAcrossContiguousAllocations";
+	const auto  end    = Libs::LibKernel::Memory::KernelGetDirectMemorySize();
+	int64_t     first  = 0;
+	int64_t     second = 0;
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelAllocateDirectMemory(
+	            0, end, SceKernelPageSize, SceKernelPageSize, SceKernelMtypeC, &first),
+	        "KernelAllocateDirectMemory(first)");
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelAllocateDirectMemory(
+	            0, end, SceKernelPageSize, SceKernelPageSize, SceKernelMtypeC, &second),
+	        "KernelAllocateDirectMemory(second)");
+	Check(test, second == first + static_cast<int64_t>(SceKernelPageSize),
+	      "test allocations are not physically contiguous");
+
+	void* mapping = nullptr;
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelMapNamedDirectMemory(
+	            &mapping, SceKernelPageSize * 2, SceKernelProtCpuRw, 0, first, SceKernelPageSize,
+	            "contiguous_allocations"),
+	        "KernelMapNamedDirectMemory");
+	auto* words = reinterpret_cast<uint64_t*>(mapping);
+	words[0]    = 0x434f4e5449474c46ull; // "CONTIGLF"
+	*reinterpret_cast<uint64_t*>(reinterpret_cast<uint64_t>(mapping) + SceKernelPageSize) =
+	    0x434f4e5449475254ull; // "CONTIGRT"
+
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelCheckedReleaseDirectMemory(first, SceKernelPageSize * 2),
+	        "KernelCheckedReleaseDirectMemory(contiguous span)");
+	ExpectUnmapped(test, reinterpret_cast<uint64_t>(mapping));
+
+	int64_t reclaimed = -1;
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelAllocateDirectMemory(
+	            0, end, SceKernelPageSize * 2, SceKernelPageSize, SceKernelMtypeC, &reclaimed),
+	        "KernelAllocateDirectMemory(reclaimed)");
+	Check(test, reclaimed == first, "released contiguous span was not coalesced");
+	CheckOk(
+	    test,
+	    Libs::LibKernel::Memory::KernelCheckedReleaseDirectMemory(reclaimed, SceKernelPageSize * 2),
+	    "KernelCheckedReleaseDirectMemory(reclaimed)");
+
+	std::printf("[host]    %-48s ok\n", test);
+}
+
 void TestDirectPhysicalFreeRangeReuseAndCoalescing() {
 	const char* test = "DirectPhysicalFreeRangeReuseAndCoalescing";
 	const auto  end  = Libs::LibKernel::Memory::KernelGetDirectMemorySize();
@@ -924,38 +1591,6 @@ void TestFixedReserveRollbackConsumesRestoredPlaceholder() {
 	std::printf("[host]    %-48s ok\n", test);
 }
 
-void TestFixedReserveRollbackRestoresDecommittedHostPages() {
-	const char*        test   = "FixedReserveRollbackRestoresDecommittedHostPages";
-	constexpr uint64_t size   = SceKernelPageSize * 3;
-	void*              mapped = nullptr;
-
-	CheckOk(test,
-	        Libs::LibKernel::Memory::KernelMapNamedFlexibleMemory(&mapped, size, SceKernelProtCpuRw,
-	                                                              0, "host_reserve_rollback"),
-	        "KernelMapNamedFlexibleMemory");
-	const auto base                    = reinterpret_cast<uint64_t>(mapped);
-	*reinterpret_cast<uint64_t*>(base) = 0x4b595459484f5354ull; // "KYTYHOST"
-	*reinterpret_cast<uint64_t*>(base + SceKernelPageSize * 2) =
-	    0x4b5954595441494cull; // "KYTYTAIL"
-
-	Libs::LibKernel::Memory::TestFailHostReservationAfter(1);
-	void* replacement = mapped;
-	CheckFailed(
-	    test,
-	    Libs::LibKernel::Memory::KernelReserveVirtualRange(
-	        &replacement, size, SceKernelMapFixed | SceKernelMapNoCoalesce, SceKernelPageSize),
-	    "KernelReserveVirtualRange(partial host reservation)");
-	Check(test, *reinterpret_cast<uint64_t*>(base) == 0x4b595459484f5354ull,
-	      "rollback did not restore the first flexible page");
-	Check(test, *reinterpret_cast<uint64_t*>(base + SceKernelPageSize * 2) == 0x4b5954595441494cull,
-	      "rollback damaged the flexible tail page");
-	ExpectRange(test, Query(test, base), base, base + size, SceKernelProtCpuRw, 1, 0, 0, 1,
-	            "host_reserve_rollback");
-
-	CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(base, size), "KernelMunmap");
-	std::printf("[host]    %-48s ok\n", test);
-}
-
 void TestFixedReserveRangeAddRollbackKeepsPlaceholder() {
 	const char*        test      = "FixedReserveRangeAddRollbackKeepsPlaceholder";
 	constexpr uint64_t size      = SceKernelPageSize * 4;
@@ -1046,8 +1681,10 @@ void TestLargeHintedReserveHostsSmallDirectMap() {
 	CheckOk(test, Libs::LibKernel::Memory::KernelReleaseDirectMemory(phys, SceKernelPageSize * 2),
 	        "KernelReleaseDirectMemory");
 	CheckOk(test,
-	        Libs::LibKernel::Memory::KernelMunmap(reinterpret_cast<uint64_t>(window), window_size),
-	        "KernelMunmap(window reserve)");
+	        Libs::LibKernel::Memory::KernelMunmap(reinterpret_cast<uint64_t>(window) +
+	                                                  SceKernelPageSize * 2,
+	                                              window_size - SceKernelPageSize * 2),
+	        "KernelMunmap(window reserve remainder)");
 	CheckOk(test,
 	        Libs::LibKernel::Memory::KernelMunmap(reinterpret_cast<uint64_t>(arena), arena_size),
 	        "KernelMunmap(arena reserve)");
@@ -1144,18 +1781,25 @@ void TestProsperoSampleMemoryPoolExpandCommit() {
 	                                                        SceKernelProtCpuRw, 0),
 	        "KernelMemoryPoolCommit");
 	ExpectRange(test, Query(test, base), base, base + commit_len, SceKernelProtCpuRw, 0, 0, 1, 1);
+	Check(test, Libs::LibKernel::Memory::TestGuestAddressRangeIsOwned(base, commit_len),
+	      "pooled commit escaped the guest owner");
 	Check(test, AvailableFlexibleMemory(test) == flexible_baseline,
 	      "pooled commit consumed flexible memory instead of expanded direct "
 	      "backing");
 	CheckFailed(test,
-	            Libs::LibKernel::Memory::KernelReleaseDirectMemory(pool_offset,
-	                                                               SceKernelMemoryPoolExpandLen),
-	            "KernelReleaseDirectMemory(committed pool expansion)");
+	            Libs::LibKernel::Memory::KernelCheckedReleaseDirectMemory(
+	                pool_offset, SceKernelMemoryPoolExpandLen),
+	            "KernelCheckedReleaseDirectMemory(committed pool expansion)");
 
 	constexpr uint64_t first_value     = 0x504f4f4c4241434bull; // "POOLBACK"
 	constexpr uint64_t second_value    = 0x5348415245444d45ull; // "SHAREDME"
 	*reinterpret_cast<uint64_t*>(base) = first_value;
 	*reinterpret_cast<uint64_t*>(base + SceKernelMemoryPoolCommitLen) = second_value;
+	uint64_t backing_read                                             = 0;
+	Check(test, Libs::LibKernel::Memory::TryReadBacking(base, &backing_read, sizeof(backing_read)),
+	      "TryReadBacking did not resolve pooled memory");
+	Check(test, backing_read == first_value,
+	      "shared backing did not observe a pooled-memory CPU write");
 
 	CheckOk(
 	    test,
@@ -1267,8 +1911,10 @@ void TestFragmentedMemoryPoolBacking() {
 	        "KernelMemoryPoolCommit(fragmented recommit)");
 	CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(base, commit_len),
 	        "KernelMunmap(fragmented commit)");
-	CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(base, SceKernelMemoryPoolReserveLen),
-	        "KernelMunmap(fragmented reserve cleanup)");
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelMunmap(base + commit_len,
+	                                              SceKernelMemoryPoolReserveLen - commit_len),
+	        "KernelMunmap(fragmented reserve remainder)");
 
 	CheckOk(test,
 	        Libs::LibKernel::Memory::KernelReleaseDirectMemory(first_pool,
@@ -1418,22 +2064,32 @@ void TestMemoryPoolCommitDecommitQueryFlags() {
 	std::printf("[host]    %-48s ok\n", test);
 }
 
-void TestProgramMemoryRegistrationAndProtection() {
-	const char* test = "ProgramMemoryRegistrationAndProtection";
+void TestProgramMemoryAllocationAndProtection() {
+	const char* test = "ProgramMemoryAllocationAndProtection";
 	const auto  size = SceKernelPageSize * 3;
-	const auto base = Common::VirtualMemory::Alloc(0, size, Common::VirtualMemory::Mode::ReadWrite);
-	Check(test, base != 0, "program host allocation failed");
-
-	Libs::LibKernel::Memory::RegisterProgramMemory(
-	    base, size, Common::VirtualMemory::Mode::ReadWrite, "program_test");
+	const auto  base = Libs::LibKernel::Memory::AllocateProgramMemory(
+	    0x900000000, size, Common::VirtualMemory::Mode::ReadWrite, "program_test");
+	Check(test, base != 0, "program guest allocation failed");
+	Check(test, Libs::LibKernel::Memory::TestGuestAddressRangeIsOwned(base, size),
+	      "program allocation escaped the guest owner");
 	ExpectRange(test, Query(test, base), base, base + size,
 	            SceKernelProtCpuRead | SceKernelProtCpuRw, 0, 0, 0, 1, "program_test");
 
-	Libs::LibKernel::Memory::UpdateProgramMemoryProtection(base, SceKernelPageSize,
-	                                                       Common::VirtualMemory::Mode::Read);
+	Check(test,
+	      Libs::LibKernel::Memory::ProtectGuestMemory(base, SceKernelPageSize,
+	                                                  Common::VirtualMemory::Mode::Read),
+	      "ProtectGuestMemory(first page) failed");
 	ExpectRange(test, Query(test, base), base, base + SceKernelPageSize, SceKernelProtCpuRead, 0, 0,
 	            0, 1, "program_test");
 
+	Common::VirtualMemory::Mode previous_mode = Common::VirtualMemory::Mode::NoAccess;
+	Check(test,
+	      Libs::LibKernel::Memory::ProtectGuestMemory(
+	          base, SceKernelPageSize, Common::VirtualMemory::Mode::ReadWrite, &previous_mode),
+	      "ProtectGuestMemory(tracked restore) failed");
+	Check(test, previous_mode == Common::VirtualMemory::Mode::Read,
+	      "semantic guest protection did not preserve its tracked old mode");
+
 	CheckOk(test,
 	        Libs::LibKernel::Memory::KernelMprotect(
 	            reinterpret_cast<void*>(base + SceKernelPageSize - 0x10), 0x20,
@@ -1442,24 +2098,44 @@ void TestProgramMemoryRegistrationAndProtection() {
 	ExpectRange(test, Query(test, base), base, base + size,
 	            SceKernelProtCpuRead | SceKernelProtCpuRw, 0, 0, 0, 1, "program_test");
 
-	Libs::LibKernel::Memory::UpdateProgramMemoryProtection(
-	    base + SceKernelPageSize * 2, SceKernelPageSize, Common::VirtualMemory::Mode::Read);
+	Check(test,
+	      Libs::LibKernel::Memory::ProtectGuestMemory(
+	          base + SceKernelPageSize * 2, SceKernelPageSize, Common::VirtualMemory::Mode::Read),
+	      "ProtectGuestMemory(last page) failed");
 	ExpectRange(test, Query(test, base + SceKernelPageSize * 2), base + SceKernelPageSize * 2,
 	            base + size, SceKernelProtCpuRead, 0, 0, 0, 1, "program_test");
 
-	Libs::LibKernel::Memory::UnregisterProgramMemory(base, size);
+	Check(test, Libs::LibKernel::Memory::FreeGuestMemory(base, size), "program guest free failed");
 	ExpectUnmapped(test, base);
-	Check(test, Common::VirtualMemory::Free(base), "program host free failed");
 
 	std::printf("[host]    %-48s ok\n", test);
 }
 
+void TestModuleRelocationUsesWritableHostMapping() {
+	const char* test = "ModuleRelocationUsesWritableHostMapping";
+	Check(test, Loader::TestModuleRelocationUsesWritableHostMapping(),
+	      "module relocation did not retain writable host memory and semantic guest protection");
+	std::printf("[host]    %-48s ok\n", test);
+}
+
 } // namespace
 
 int main() {
 	InitSubsystems();
 
 	RunTest(TestProsperoArgumentAndInfoSizeContracts);
+	RunTest(TestGuestAddressSpaceOwnsReservationsBeforeBacking);
+	RunTest(TestGuestAddressSpaceHasNoFixedFallback);
+	RunTest(TestGuestFreeRangeSearchDoesNotUnderflow);
+	RunTest(TestFlexibleMemoryCapacityIsBootFixed);
+	RunTest(TestFlexibleMemoryUsesSharedBacking);
+	RunTest(TestFlexibleDmemCompatAndAlignmentFlags);
+	RunTest(TestFlexibleNoCoalescePreservesBoundaries);
+	RunTest(TestFlexibleMemoryReuseIsZeroFilled);
+	RunTest(TestGuestStackUsesPrivateOwnerMemoryAndCache);
+	RunTest(TestMainEntryUsesGuestStackAndDisablesHostChecks);
+	RunTest(TestFragmentedBackingUnmapRollback);
+	RunTest(TestRuntimeMemoryOwnerLifecycle);
 	RunTest(TestFlexibleMapQueryAndWholeMunmap);
 	RunTest(TestPartialFlexibleMunmapAndFindNext);
 	RunTest(TestReserveMapFixedAndNoOverwrite);
@@ -1467,7 +2143,12 @@ int main() {
 	RunTest(TestReleasedReserveCanBeReused);
 	RunTest(TestMunmapAcrossAdjacentFlexibleMappings);
 	RunTest(TestDirectMapQueryOffsetAndPartialMunmap);
+	RunTest(TestDirectPartialProtectUnmapPreservesNeighbors);
+	RunTest(TestDirectMapValidationBeforeOwnerMutation);
+	RunTest(TestDirectReleaseRollbackRestoresOwnerMapping);
+	RunTest(TestDirectReleaseContracts);
 	RunTest(TestNonzeroDirectOffsetAliasesSharedBacking);
+	RunTest(TestDirectMapAcrossContiguousAllocations);
 	RunTest(TestDirectPhysicalFreeRangeReuseAndCoalescing);
 	RunTest(TestDirectAlignmentStaysWithinSearchRange);
 	RunTest(TestDefaultDirectMapUsesSystemAddressRange);
@@ -1476,7 +2157,6 @@ int main() {
 	RunTest(TestFixedReserveReplacesPartialDirectMapping);
 	RunTest(TestFixedReserveRollbackConsumesRestoredPlaceholder);
 	RunTest(TestFixedReserveRollbackSkipsUntouchedChunks);
-	RunTest(TestFixedReserveRollbackRestoresDecommittedHostPages);
 	RunTest(TestFixedReserveRangeAddRollbackKeepsPlaceholder);
 	RunTest(TestLargeHintedReserveHostsSmallDirectMap);
 	RunTest(TestMemoryPoolAlignmentContracts);
@@ -1484,7 +2164,8 @@ int main() {
 	RunTest(TestFragmentedMemoryPoolBacking);
 	RunTest(TestMemoryPoolMultiRangeDecommit);
 	RunTest(TestMemoryPoolCommitDecommitQueryFlags);
-	RunTest(TestProgramMemoryRegistrationAndProtection);
+	RunTest(TestProgramMemoryAllocationAndProtection);
+	RunTest(TestModuleRelocationUsesWritableHostMapping);
 
 	if (g_failed_tests != 0) {
 		std::printf("VirtualMemoryAllocationTests: %d case(s) failed\n", g_failed_tests);

tests/shaderCfgTests.cpp

@@ -3020,6 +3020,37 @@ void TestNewShaderRecompilerImageQueryLowering() {
 	CheckSpirvBinaryValidates(result.spirv);
 }
 
+void TestNewShaderRecompilerCubeSampleCoordinates() {
+	constexpr uint32_t MimgDimCube = 3;
+	const uint32_t shader[] = {
+	    EncodeMimg0(0x20, 0xf, false, MimgDimCube),
+	    EncodeMimg1(0, 0, 1, 0), // image_sample cube
+	    EncodeMimg0(0x60, 0x3, false, MimgDimCube),
+	    EncodeMimg1(8, 0, 1, 4), // image_get_lod cube
+	    0xbf810000u,
+	};
+
+	auto user_data = ImageTestUserData(Prospero::ImageType::kCube);
+	ShaderRecompiler::CompileOptions options;
+	options.stage     = ShaderType::Compute;
+	options.user_data = user_data.data();
+
+	ShaderRecompiler::CompileResult result;
+	std::string                     error;
+	Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str());
+	Check(result.program.info.images.size() == 1 && result.program.info.images[0].cube,
+	      "cube descriptor identity was not preserved through compilation");
+	Check(SpirvInstructionOpcodeCount(result.spirv, 131) == 4,
+	      "cube sample/get-lod did not remove the RDNA2 S/T bias");
+	Check(SpirvInstructionOpcodeCount(result.spirv, 109) == 1 &&
+	          SpirvInstructionOpcodeCount(result.spirv, 112) == 1 &&
+	          SpirvInstructionOpcodeCount(result.spirv, 194) == 1 &&
+	          SpirvInstructionOpcodeCount(result.spirv, 196) == 1 &&
+	          SpirvInstructionOpcodeCount(result.spirv, 130) == 1,
+	      "cube sample did not repack its face ID exactly once, or get-lod used an array layer");
+	CheckSpirvBinaryValidates(result.spirv);
+}
+
 void TestNewShaderRecompilerImageSampleVariants() {
 	const uint32_t shader[] = {
 	    EncodeMimg0(0x24, 0xf),
@@ -6957,6 +6988,7 @@ int main() {
 	TestNewShaderRecompilerScalarBitfieldAlu();
 	TestNewShaderRecompilerMemoryFamilyLowering();
 	TestNewShaderRecompilerImageQueryLowering();
+	TestNewShaderRecompilerCubeSampleCoordinates();
 	TestNewShaderRecompilerImageSampleVariants();
 	TestNewShaderRecompilerImageSampleA16SamplerCoords();
 	TestNewShaderRecompilerImageSampleOpcodeAliases();