docs: add macOS build and run instructions (#137)

The README had macOS badges and an experimental-support note but no build,
run, or system-requirement information for the platform. Document the
Rosetta 2 / MoltenVK setup, the x86-64 configure invocation, the Qt
universal-build requirement, MoltenVK installation and signing, and the
SDL_VULKAN_LIBRARY variable needed at run time.

.github/workflows/build.yml

@@ -83,7 +83,12 @@ jobs:
       - name: Build
         shell: cmd
         run: |
-          cmake --build _Build/windows --target launcher --parallel
+          cmake --build _Build/windows --target launcher virtual_memory_allocation_tests --parallel
+
+      - name: Test
+        shell: cmd
+        run: |
+          ctest --test-dir _Build/windows --output-on-failure -R "^virtual_memory_allocation$"
 
       - name: Install
         shell: cmd
@@ -153,7 +158,15 @@ jobs:
       - name: Build
         shell: bash
         run: |
-          cmake --build _Build/macos --target launcher --parallel
+          cmake --build _Build/macos \
+            --target launcher virtual_memory_allocation_tests \
+            --parallel
+
+      - name: Test
+        shell: bash
+        run: |
+          ctest --test-dir _Build/macos --output-on-failure \
+            -R '^virtual_memory_allocation$'
 
       - name: Install
         shell: bash
@@ -284,13 +297,14 @@ jobs:
         run: |
           cmake --build _Build/linux \
             --target launcher page_manager_tests memory_tracker_tests \
+              virtual_memory_allocation_tests \
             --parallel
 
       - name: Test
         shell: bash
         run: |
           ctest --test-dir _Build/linux --output-on-failure \
-            -R '^(page_manager|memory_tracker)$'
+            -R '^(page_manager|memory_tracker|virtual_memory_allocation)$'
 
       - name: Install
         shell: bash

README.md

@@ -27,8 +27,9 @@ Development is focused on compatibility and boot reliability.
 Windows is the primary platform and receives the most testing. Linux builds and runs; see
 [Building on Linux](#building-on-linux).
 
-macOS support is experimental. Compatibility with the same games on Windows and macOS has not yet
-been tested.
+macOS support is experimental. The emulator is built for x86-64 and runs on Apple Silicon under
+Rosetta 2, with Vulkan provided by MoltenVK. A small number of titles have been verified in-game
+on Apple Silicon hardware; see [Building on macOS](#building-on-macos).
 
 ## Bugs and Issues
 
@@ -50,7 +51,7 @@ graphical glitches, low compatibility, and poor performance.
   </tr>
   <tr>
     <td align="center">
-      <strong>Minecraft Legends</strong><br>
+      <strong>Neptunia ReVerse</strong><br>
       <img src="docs/screenshots/ps5-04.png" width="300" alt="Minecraft Legends running in KytyPS5">
     </td>
     <td align="center">
@@ -58,8 +59,20 @@ graphical glitches, low compatibility, and poor performance.
       <img src="docs/screenshots/ps5-05.png" width="300" alt="SILENT HILL: The Short Message running in KytyPS5">
     </td>
   </tr>
+  <tr>
+    <td align="center">
+      <strong>Hellboy</strong><br>
+      <img src="docs/screenshots/ps5-02.png" width="300" alt="Disgaea 6 running in KytyPS5">
+    </td>
+    <td align="center">
+      <strong>Paleo Pines</strong><br>
+      <img src="docs/screenshots/ps5-06.png" width="300" alt="Dreaming Sarah running in KytyPS5">
+    </td>
+  </tr>
 </table>
 
+<p align="center"><em>And many more...</em></p>
+
 ## Contributing
 
 Testing games and submitting detailed bug reports are useful ways to contribute. Search existing
@@ -102,9 +115,10 @@ the Vulkan/SPIR-V validation rules.
 
 ### System requirements
 
-- Windows 10 version 1803, or a current Linux distribution
-- A 64-bit x86 processor
-- A Vulkan 1.3-capable GPU with current drivers
+- Windows 10 version 1803, a current Linux distribution, or macOS on Apple Silicon
+- A 64-bit x86 processor (on macOS, an Apple Silicon processor with Rosetta 2)
+- A Vulkan 1.3-capable GPU with current drivers (on macOS, Vulkan is provided by the bundled
+  MoltenVK)
 
 ### Build requirements (Windows)
 
@@ -176,6 +190,45 @@ time.
 
 Note that the CMake source root is `src`, not the repository root.
 
+### Building on macOS
+
+macOS builds target x86-64 and run under Rosetta 2 on Apple Silicon, so the PS5's x86-64 game
+code executes through the same translation layer as the emulator itself. Prebuilt archives are
+attached to releases; the steps below are for building from source.
+
+Requirements:
+
+- An Apple Silicon Mac with Rosetta 2 installed (`softwareupdate --install-rosetta`)
+- Xcode (or the Command Line Tools)
+- Homebrew packages: `brew install cmake ninja glslang`
+- Qt 6 (Concurrent, Network, Widgets) with x86-64 support. The official Qt installation is
+  universal and works; Homebrew's Qt is arm64-only and will not link
+
+```bash
+git submodule update --init --recursive
+
+cmake -S src -B _Build/macos -G Ninja -DCMAKE_BUILD_TYPE=Release \
+  -DCMAKE_OSX_ARCHITECTURES=x86_64 \
+  -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ \
+  -DCMAKE_PREFIX_PATH="$Qt6_DIR"
+
+cmake --build _Build/macos --target launcher --parallel
+cmake --install _Build/macos --prefix _Build/macos/install
+```
+
+The build re-signs `kyty_emulator` with the JIT entitlements it needs to execute translated
+guest code; no manual signing step is required.
+
+Vulkan comes from MoltenVK. Download `MoltenVK-macos.tar` from the
+[MoltenVK releases](https://github.com/KhronosGroup/MoltenVK/releases), then copy
+`MoltenVK/dynamic/dylib/macOS/libMoltenVK.dylib` next to `kyty_emulator` and ad-hoc sign it:
+
+```bash
+codesign --force --sign - _Build/macos/install/libMoltenVK.dylib
+```
+
+Release archives already include a signed `libMoltenVK.dylib`.
+
 ### Visual Studio Code
 
 A ready-made Visual Studio Code setup is included in [`.vscode`](.vscode). It configures CMake
@@ -221,6 +274,14 @@ The emulator can also be started directly with a legally obtained game directory
 ./_Build/linux/install/kyty_emulator --game "/games/ExampleGame"
 ```
 
+On macOS, point SDL at the MoltenVK library explicitly; the hardened runtime prevents it from
+being picked up from the executable's directory:
+
+```bash
+cd _Build/macos/install
+SDL_VULKAN_LIBRARY="$PWD/libMoltenVK.dylib" ./kyty_emulator --game "/games/ExampleGame"
+```
+
 Run `kyty_emulator --help` to see the available graphics, logging, validation, profiling, and
 debugging options.
 

src/CMakeLists.txt

@@ -314,6 +314,16 @@ function(add_kyty_full_emulator_test target source)
 	endif()
 endfunction()
 
+function(configure_macos_guest_address_space target)
+	if(APPLE AND (CMAKE_OSX_ARCHITECTURES STREQUAL "x86_64" OR
+			(NOT CMAKE_OSX_ARCHITECTURES AND CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|AMD64)$")))
+		target_sources(${target} PRIVATE kernel/macosGuestAddressSpace.cpp)
+		target_compile_definitions(${target} PRIVATE KYTY_LINKED_GUEST_ADDRESS_SPACE=1)
+		target_link_options(${target} PRIVATE
+			-Wl,-ld_classic,-no_pie,-no_fixup_chains,-no_huge,-pagezero_size,0x40000,-segaddr,SYSTEM_MANAGED,0x40000,-segaddr,SYSTEM_RESERVED,0x7ffffc000,-segaddr,USER_AREA,0x7000000000,-image_base,0x700000000000)
+	endif()
+endfunction()
+
 add_kyty_full_emulator_test(shader_cfg_tests ../tests/shaderCfgTests.cpp)
 
 add_executable(scalar_provenance_tests EXCLUDE_FROM_ALL
@@ -338,7 +348,6 @@ add_executable(memory_tracker_tests EXCLUDE_FROM_ALL
 )
 target_link_libraries(memory_tracker_tests fmt::fmt common)
 target_include_directories(memory_tracker_tests PRIVATE ${inc_headers})
-target_compile_definitions(memory_tracker_tests PRIVATE KYTY_MEMORY_TRACKER_TESTS=1)
 
 add_executable(shader_vertex_metadata_tests EXCLUDE_FROM_ALL
 	../tests/ShaderVertexMetadataTests.cpp
@@ -421,6 +430,7 @@ target_sources(shader_recompiler_compute_tests PRIVATE
 add_kyty_full_emulator_test(virtual_memory_allocation_tests ../tests/VirtualMemoryAllocationTests.cpp)
 target_compile_definitions(virtual_memory_allocation_tests PRIVATE
 	KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS=1)
+configure_macos_guest_address_space(virtual_memory_allocation_tests)
 
 # These tests use exceptions.
 if(NOT KYTY_CLANG_CL)
@@ -437,6 +447,8 @@ if(BUILD_TESTING)
 	add_test(NAME resource_mutex COMMAND $<TARGET_FILE:resource_mutex_tests>)
 	add_test(NAME event_queue_lifetime COMMAND $<TARGET_FILE:event_queue_lifetime_tests>)
 	add_test(NAME shader_recompiler_compute COMMAND $<TARGET_FILE:shader_recompiler_compute_tests>)
+	add_test(NAME virtual_memory_allocation
+		COMMAND $<TARGET_FILE:virtual_memory_allocation_tests>)
 	add_test(NAME command_scheduler_timeline
 		COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --scheduler-only)
 	add_test(NAME stream_buffer_ring
@@ -470,6 +482,7 @@ endif()
 
 
 add_executable(kyty_emulator main.cpp ${kyty_emulator_src})
+configure_macos_guest_address_space(kyty_emulator)
 
 target_link_libraries(kyty_emulator ${kyty_emulator_link_libraries})
 if (WIN32)

src/common/hostException.cpp

@@ -324,6 +324,10 @@ bool InstallHandler(Handler handler) {
 	sa.sa_sigaction = SignalHandler;
 	sa.sa_flags     = SA_SIGINFO;
 	sigemptyset(&sa.sa_mask);
+	// The guest signal-dispatch path (KernelRaiseException) interrupts threads with
+	// SIGUSR1; block it while a fault is being resolved so a stop-the-world request
+	// cannot preempt the handler between the protection fix and the retry.
+	sigaddset(&sa.sa_mask, SIGUSR1);
 
 	// macOS raises SIGBUS for protection faults on some paths and SIGSEGV on others;
 	// SIGILL covers instructions the host cannot execute (routed to the x64 emulator).

src/common/platform/sysLinuxVirtual.cpp

@@ -432,11 +432,6 @@ uint64_t SysVirtualReserveAligned(uint64_t address, uint64_t size, uint64_t alig
 
 	pthread_mutex_lock(&g_virtual_mutex);
 	record_alloc(ret_addr, size);
-	uintptr_t page_start  = ret_addr >> 12u;
-	uintptr_t page_end    = (ret_addr + size - 1) >> 12u;
-	for (uintptr_t page = page_start; page <= page_end; page++) {
-		(*g_protects)[page] = PROT_NONE;
-	}
 	pthread_mutex_unlock(&g_virtual_mutex);
 
 	return ret_addr;
@@ -470,11 +465,6 @@ bool SysVirtualReserveFixed(uint64_t address, uint64_t size) {
 	if (ptr != MAP_FAILED) {
 		pthread_mutex_lock(&g_virtual_mutex);
 		record_alloc(ret_addr, size);
-		uintptr_t page_start  = ret_addr >> 12u;
-		uintptr_t page_end    = (ret_addr + size - 1) >> 12u;
-		for (uintptr_t page = page_start; page <= page_end; page++) {
-			(*g_protects)[page] = PROT_NONE;
-		}
 		pthread_mutex_unlock(&g_virtual_mutex);
 
 		return true;

src/common/virtualMemory.cpp

@@ -58,25 +58,6 @@ bool FlushInstructionCache(uint64_t address, uint64_t size) {
 	return SysVirtualFlushInstructionCache(address, size);
 }
 
-bool PatchReplace(uint64_t vaddr, uint64_t value) {
-	Mode old_mode {};
-	Protect(vaddr, 8, Mode::ReadWrite, &old_mode);
-
-	auto* ptr = reinterpret_cast<uint64_t*>(vaddr);
-
-	bool ret = (*ptr != value);
-
-	*ptr = value;
-
-	Protect(vaddr, 8, old_mode);
-
-	if (IsExecute(old_mode)) {
-		FlushInstructionCache(vaddr, 8);
-	}
-
-	return ret;
-}
-
 } // namespace VirtualMemory
 
 } // namespace Common

src/common/virtualMemory.h

@@ -37,7 +37,6 @@ bool     Free(uint64_t address);
 bool     FreeRange(uint64_t address, uint64_t size);
 bool     Protect(uint64_t address, uint64_t size, Mode mode, Mode* old_mode = nullptr);
 bool     FlushInstructionCache(uint64_t address, uint64_t size);
-bool     PatchReplace(uint64_t vaddr, uint64_t value);
 
 } // namespace VirtualMemory
 

src/emulator.cpp

@@ -105,7 +105,7 @@ static void ClearDebugTextureFolder() {
 	}
 }
 
-static void Init(const Config::ConfigOptions& cfg) {
+static void Init(const Config::ConfigOptions& cfg, const std::filesystem::path& param_json) {
 	EXIT_IF(!Common::Thread::IsMainThread());
 
 	auto* slist = Common::SubsystemsList::Instance();
@@ -127,12 +127,21 @@ static void Init(const Config::ConfigOptions& cfg) {
 	slist->InitAll(true);
 
 	Config::Load(cfg);
+	slist->Add(log, {core, config});
+	slist->InitAll(true);
+
+	if (Common::File::IsFileExisting(param_json)) {
+		Loader::SystemContentLoadParamSfo(param_json);
+		if (const auto flexible_memory_size = Loader::SystemContentGetFlexibleMemorySize();
+		    flexible_memory_size != 0) {
+			Libs::LibKernel::Memory::SetFlexibleMemorySize(flexible_memory_size);
+		}
+	}
 
 	slist->Add(audio, {core, log, pthread, memory});
 	slist->Add(controller, {core, log, config});
 	slist->Add(file_system, {core, log, pthread});
 	slist->Add(graphics, {core, log, pthread, memory, config, profiler, controller});
-	slist->Add(log, {core, config});
 	slist->Add(memory, {core, log});
 	slist->Add(network, {core, log, pthread});
 	slist->Add(profiler, {core, config});
@@ -180,7 +189,8 @@ void Run(const RunOptions& options) {
 		EXIT("ELF is required\n");
 	}
 
-	Init(options.config);
+	const auto param_json = options.app0_dir / "sce_sys" / "param.json";
+	Init(options.config, param_json);
 
 	ClearDebugTextureFolder();
 
@@ -192,15 +202,6 @@ void Run(const RunOptions& options) {
 	Libs::LibKernel::FileSystem::Mount(options.app0_dir, "/app0");
 	Libs::LibKernel::FileSystem::Mount(options.app0_dir, "/hostapp");
 
-	auto param_json = options.app0_dir / "sce_sys" / "param.json";
-	if (Common::File::IsFileExisting(param_json)) {
-		Loader::SystemContentLoadParamSfo(param_json);
-		if (auto flexible_memory_size = Loader::SystemContentGetFlexibleMemorySize();
-		    flexible_memory_size != 0) {
-			Libs::LibKernel::Memory::SetFlexibleMemorySize(flexible_memory_size);
-		}
-	}
-
 	MountSandboxDirs();
 
 	auto* rt = Common::Singleton<Loader::RuntimeLinker>::Instance();

src/graphics/host_gpu/memoryTracker.cpp

@@ -4,16 +4,6 @@
 
 namespace Libs::Graphics {
 
-#if defined(KYTY_MEMORY_TRACKER_TESTS)
-namespace {
-std::atomic<MemoryTracker::UnmapContentionHook> g_unmap_contention_hook {nullptr};
-}
-
-void MemoryTracker::SetUnmapContentionHook(UnmapContentionHook hook) noexcept {
-	g_unmap_contention_hook.store(hook, std::memory_order_release);
-}
-#endif
-
 static_assert(std::atomic<void*>::is_always_lock_free);
 
 MemoryTracker::MemoryTracker(PageManager& page_manager, PageWatchMode gpu_watch_mode)
@@ -94,7 +84,6 @@ RegionManager* MemoryTracker::GetOrCreateRegion(uint64_t index) {
 bool MemoryTracker::IsRegionCpuModified(uint64_t vaddr, uint64_t size) {
 	CheckNotInUploadCallback();
 	std::lock_guard access(m_access_mutex);
-	RequireMapped(vaddr, size);
 	return Iterate<true>(vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) {
 		std::scoped_lock lock(manager->lock);
 		return manager->IsModified<DirtySource::Cpu>(offset, bytes);
@@ -104,7 +93,6 @@ bool MemoryTracker::IsRegionCpuModified(uint64_t vaddr, uint64_t size) {
 bool MemoryTracker::IsRegionGpuModified(uint64_t vaddr, uint64_t size) {
 	CheckNotInUploadCallback();
 	std::lock_guard access(m_access_mutex);
-	RequireMapped(vaddr, size);
 	return Iterate<false>(vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) {
 		std::scoped_lock lock(manager->lock);
 		return manager->IsModified<DirtySource::Gpu>(offset, bytes);
@@ -114,7 +102,6 @@ bool MemoryTracker::IsRegionGpuModified(uint64_t vaddr, uint64_t size) {
 void MemoryTracker::MarkRegionAsCpuModified(uint64_t vaddr, uint64_t size) {
 	CheckNotInUploadCallback();
 	std::lock_guard access(m_access_mutex);
-	RequireMapped(vaddr, size);
 	Iterate<true>(vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) {
 		std::scoped_lock lock(manager->lock);
 		const auto       changed =
@@ -126,7 +113,6 @@ void MemoryTracker::MarkRegionAsCpuModified(uint64_t vaddr, uint64_t size) {
 void MemoryTracker::MarkRegionAsGpuModified(uint64_t vaddr, uint64_t size) {
 	CheckNotInUploadCallback();
 	std::lock_guard access(m_access_mutex);
-	RequireMapped(vaddr, size);
 	Iterate<true>(vaddr, size, [this](RegionManager* manager, uint64_t offset, uint64_t bytes) {
 		std::scoped_lock lock(manager->lock);
 		const auto       changed =
@@ -138,7 +124,6 @@ void MemoryTracker::MarkRegionAsGpuModified(uint64_t vaddr, uint64_t size) {
 void MemoryTracker::UnmarkRegionAsGpuModified(uint64_t vaddr, uint64_t size) {
 	CheckNotInUploadCallback();
 	std::lock_guard access(m_access_mutex);
-	RequireMapped(vaddr, size);
 	Iterate<true>(vaddr, size, [this](RegionManager* manager, uint64_t offset, uint64_t bytes) {
 		std::scoped_lock lock(manager->lock);
 		if (!manager->IsFullyModified<DirtySource::Gpu>(offset, bytes)) {
@@ -151,8 +136,6 @@ void MemoryTracker::UnmarkRegionAsGpuModified(uint64_t vaddr, uint64_t size) {
 }
 
 void MemoryTracker::UntrackMemoryLocked(uint64_t vaddr, uint64_t size) {
-	RequireMapped(vaddr, size);
-
 	std::vector<RegionManager*> managers;
 	managers.reserve((vaddr % TRACKER_REGION_SIZE + size + TRACKER_REGION_SIZE - 1) /
 	                 TRACKER_REGION_SIZE);
@@ -185,22 +168,6 @@ void MemoryTracker::UntrackMemory(uint64_t vaddr, uint64_t size) {
 	UntrackMemoryLocked(vaddr, size);
 }
 
-void MemoryTracker::UnmapMemory(uint64_t vaddr, uint64_t size) {
-	CheckNotInUploadCallback();
-	std::unique_lock access(m_access_mutex, std::try_to_lock);
-	if (!access.owns_lock()) {
-#if defined(KYTY_MEMORY_TRACKER_TESTS)
-		if (const auto hook = g_unmap_contention_hook.load(std::memory_order_acquire);
-		    hook != nullptr) {
-			hook();
-		}
-#endif
-		access.lock();
-	}
-	UntrackMemoryLocked(vaddr, size);
-	m_page_manager.OnGpuUnmap(vaddr, size);
-}
-
 bool MemoryTracker::InvalidateRegion(uint64_t vaddr, uint64_t size, PageFaultPhase phase) noexcept {
 	switch (phase) {
 		case PageFaultPhase::Release: return true;

src/graphics/host_gpu/memoryTracker.h

@@ -30,7 +30,6 @@ public:
 	void               MarkRegionAsGpuModified(uint64_t vaddr, uint64_t size);
 	void               UnmarkRegionAsGpuModified(uint64_t vaddr, uint64_t size);
 	void               UntrackMemory(uint64_t vaddr, uint64_t size);
-	void               UnmapMemory(uint64_t vaddr, uint64_t size);
 	[[nodiscard]] CpuFaultAction
 	                   BeginCpuFault(uint64_t vaddr, uint64_t size,
 	                                 PageFaultAccess access = PageFaultAccess::Write) noexcept;
@@ -38,10 +37,51 @@ public:
 	                                    bool downloaded) noexcept;
 	[[nodiscard]] bool InvalidateRegion(uint64_t vaddr, uint64_t size,
 	                                    PageFaultPhase phase) noexcept;
+	template <typename Flush>
+	void InvalidateRegion(uint64_t vaddr, uint64_t size, Flush&& on_flush) {
+		static_assert(std::is_invocable_v<Flush&>);
+		CheckNotInUploadCallback();
+		ValidateRange(vaddr, size);
+
+		const auto update_cpu_state = [this, vaddr, size] {
+			std::lock_guard             access(m_access_mutex);
+			std::vector<RegionManager*> managers;
+			Iterate<false>(vaddr, size, [&](RegionManager* manager, uint64_t, uint64_t) {
+				managers.push_back(manager);
+			});
+			std::vector<std::unique_lock<TrackingSpinLock>> locks;
+			locks.reserve(managers.size());
+			for (auto* manager: managers) {
+				locks.emplace_back(manager->lock);
+			}
+			const bool gpu_modified = Iterate<false>(
+			    vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) {
+				    return manager->IsModified<DirtySource::Gpu>(offset, bytes);
+			    });
+			if (gpu_modified) {
+				return true;
+			}
+			Iterate<false>(vaddr, size,
+			               [](RegionManager* manager, uint64_t offset, uint64_t bytes) {
+				               const auto changed = manager->ChangeState<DirtySource::Cpu, true>(
+				                   manager->GetCpuAddr() + offset, bytes);
+				               manager->ApplyProtection(changed, false);
+			               });
+			return false;
+		};
+
+		if (!update_cpu_state()) {
+			return;
+		}
+		std::forward<Flush>(on_flush)();
+		if (update_cpu_state()) {
+			EXIT("memory invalidation retained GPU-owned pages\n");
+		}
+	}
 	[[nodiscard]] bool InvalidateVirtualGpuWrite(PageFaultAccess access, uint64_t vaddr,
 	                                             uint64_t size, PageFaultPhase phase) noexcept;
-	void ValidateGpuDirtyPages(const RangeSet& dirty, uint64_t vaddr, uint64_t size,
-	                           const char* operation) const noexcept;
+	void               ValidateGpuDirtyPages(const RangeSet& dirty, uint64_t vaddr, uint64_t size,
+	                                         const char* operation) const noexcept;
 	void ValidateGpuDirtyOwnership(const RangeSet& dirty, uint64_t vaddr, uint64_t size,
 	                               const char* operation);
 
@@ -50,8 +90,7 @@ public:
 		static_assert(std::is_nothrow_invocable_v<Preflight&, uint64_t, uint64_t>);
 		static_assert(std::is_nothrow_invocable_v<Func&, uint64_t, uint64_t>);
 		CheckNotInUploadCallback();
-		std::lock_guard access(m_access_mutex);
-		RequireMapped(vaddr, size);
+		std::lock_guard             access(m_access_mutex);
 		std::vector<RegionManager*> managers;
 		Iterate<false>(vaddr, size, [&](RegionManager* manager, uint64_t, uint64_t) {
 			managers.push_back(manager);
@@ -91,11 +130,6 @@ public:
 		    vaddr, size, [](uint64_t, uint64_t) noexcept {}, std::forward<Func>(func));
 	}
 
-#if defined(KYTY_MEMORY_TRACKER_TESTS)
-	using UnmapContentionHook = void (*)() noexcept;
-	static void SetUnmapContentionHook(UnmapContentionHook hook) noexcept;
-#endif
-
 	template <typename RangeFunc, typename UploadFunc>
 	void ForEachUploadRange(uint64_t vaddr, uint64_t size, bool is_written, RangeFunc&& range_func,
 	                        UploadFunc&& upload_func) {
@@ -103,7 +137,6 @@ public:
 		static_assert(std::is_nothrow_invocable_v<UploadFunc&>);
 		CheckNotInUploadCallback();
 		std::unique_lock access(m_access_mutex);
-		RequireMapped(vaddr, size);
 		Iterate<true>(vaddr, size, [](RegionManager*, uint64_t, uint64_t) {});
 		const auto* previous_upload_owner = std::exchange(s_upload_owner, this);
 		Iterate<false>(vaddr, size, [&](RegionManager* manager, uint64_t offset, uint64_t bytes) {
@@ -168,16 +201,8 @@ private:
 		return false;
 	}
 
-	static void ValidateRange(uint64_t vaddr, uint64_t size);
-	void        UntrackMemoryLocked(uint64_t vaddr, uint64_t size);
-	void        RequireMapped(uint64_t vaddr, uint64_t size) const {
-		ValidateRange(vaddr, size);
-		if (!m_page_manager.IsMapped(vaddr, size)) {
-			EXIT("memory tracker range [0x%llx, 0x%llx) is not mapped\n",
-			     static_cast<unsigned long long>(vaddr),
-			     static_cast<unsigned long long>(vaddr + size));
-		}
-	}
+	static void    ValidateRange(uint64_t vaddr, uint64_t size);
+	void           UntrackMemoryLocked(uint64_t vaddr, uint64_t size);
 	RegionManager* GetOrCreateRegion(uint64_t index);
 
 	std::unique_ptr<std::atomic<RegionManager*>[]> m_regions;

src/graphics/host_gpu/pageManager.cpp

@@ -1,7 +1,9 @@
 #include "graphics/host_gpu/pageManager.h"
 
 #include "graphics/host_gpu/regionDefinitions.h"
+#include "kernel/memory.h"
 
+#include <algorithm>
 #include <array>
 #include <atomic>
 #include <cstdarg>
@@ -20,16 +22,11 @@
 #undef min
 #undef max
 #elif defined(__APPLE__)
-#include <mach/mach.h>
-#include <mach/mach_vm.h>
 #include <pthread.h>
 #include <sys/mman.h>
 #include <unistd.h>
 #else
-#include <cerrno>
-#include <cstring>
 #include <execinfo.h>
-#include <fcntl.h>
 #include <sys/mman.h>
 #include <sys/syscall.h>
 #include <unistd.h>
@@ -56,45 +53,8 @@ constexpr uint64_t REGION_PAGES = REGION_SIZE / PAGE_SIZE;
 constexpr uint32_t NO_ACCESS_PROTECTION  = PAGE_NOACCESS;
 constexpr uint32_t READ_ONLY_PROTECTION  = PAGE_READONLY;
 constexpr uint32_t READ_WRITE_PROTECTION = PAGE_READWRITE;
-
-#if defined(__APPLE__)
-// Map the tracker's Win32-style protection tags to POSIX mprotect flags.
-static int PageProtToPosix(uint32_t protection) {
-	switch (protection) {
-		case PAGE_NOACCESS: return PROT_NONE;
-		case PAGE_READONLY: return PROT_READ;
-		case PAGE_READWRITE: return PROT_READ | PROT_WRITE;
-		default: return PROT_NONE;
-	}
-}
-
-// Query the current protection of the page containing vaddr via the Mach VM map and
-// collapse it to the tracker's read/write tags (execute is irrelevant to write tracking).
-static uint32_t MachQueryPageProt(uint64_t vaddr) {
-	auto              region_addr = static_cast<mach_vm_address_t>(vaddr);
-	mach_vm_size_t    region_size = 0;
-	vm_region_basic_info_data_64_t info {};
-	mach_msg_type_number_t         count       = VM_REGION_BASIC_INFO_COUNT_64;
-	mach_port_t                    object_name = MACH_PORT_NULL;
-
-	kern_return_t kr =
-	    mach_vm_region(mach_task_self(), &region_addr, &region_size, VM_REGION_BASIC_INFO_64,
-	                   reinterpret_cast<vm_region_info_t>(&info), &count, &object_name);
-	if (kr != KERN_SUCCESS || region_addr > vaddr) {
-		return PAGE_NOACCESS; // no region covering vaddr
-	}
-	if ((info.protection & VM_PROT_WRITE) != 0) {
-		return PAGE_READWRITE;
-	}
-	if ((info.protection & VM_PROT_READ) != 0) {
-		return PAGE_READONLY;
-	}
-	return PAGE_NOACCESS;
-}
-#elif defined(__linux__)
 // Zero is the unknown protection sentinel.
 constexpr uint32_t UNKNOWN_PROTECTION = 0;
-#endif
 
 thread_local bool g_in_fault_resolution = false;
 
@@ -135,6 +95,15 @@ thread_local bool g_in_fault_resolution = false;
 	std::_Exit(322);
 }
 
+Common::VirtualMemory::Mode ToMemoryMode(uint32_t protection) {
+	switch (protection) {
+		case NO_ACCESS_PROTECTION: return Common::VirtualMemory::Mode::NoAccess;
+		case READ_ONLY_PROTECTION: return Common::VirtualMemory::Mode::Read;
+		case READ_WRITE_PROTECTION: return Common::VirtualMemory::Mode::ReadWrite;
+		default: Fatal("unmappable protection 0x%08" PRIx32, protection);
+	}
+}
+
 uint32_t CurrentThread() noexcept {
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
 	return GetCurrentThreadId();
@@ -154,120 +123,6 @@ uint32_t CurrentThread() noexcept {
 #endif
 }
 
-#if defined(__linux__)
-int ToHostProtection(uint32_t protection) {
-	switch (protection) {
-		case NO_ACCESS_PROTECTION: return PROT_NONE;
-		case READ_ONLY_PROTECTION: return PROT_READ;
-		case READ_WRITE_PROTECTION: return PROT_READ | PROT_WRITE;
-		default: Fatal("unmappable protection 0x%08" PRIx32, protection);
-	}
-}
-
-// Async-signal-safe lookup in the address-ordered /proc/self/maps.
-uint32_t QueryHostProtection(uint64_t vaddr) noexcept {
-	int fd = ::open("/proc/self/maps", O_RDONLY | O_CLOEXEC); // NOLINT
-	if (fd < 0) {
-		return UNKNOWN_PROTECTION;
-	}
-
-	enum class Field { Start, End, Perms, Rest };
-
-	uint32_t result     = UNKNOWN_PROTECTION;
-	auto     field      = Field::Start;
-	uint64_t start      = 0;
-	uint64_t end        = 0;
-	char     perms[4]   = {};
-	uint32_t perms_len  = 0;
-	bool     line_valid = true;
-
-	char buffer[8192];
-
-	for (bool done = false; !done;) {
-		const auto got = ::read(fd, buffer, sizeof(buffer));
-		if (got < 0) {
-			if (errno == EINTR) {
-				continue;
-			}
-			break;
-		}
-		if (got == 0) {
-			break;
-		}
-
-		for (ssize_t i = 0; i < got && !done; i++) {
-			const char c = buffer[i];
-
-			if (c == '\n') {
-				field      = Field::Start;
-				start      = 0;
-				end        = 0;
-				perms_len  = 0;
-				line_valid = true;
-				continue;
-			}
-
-			if (!line_valid) {
-				continue;
-			}
-
-			switch (field) {
-				case Field::Start:
-				case Field::End: {
-					uint64_t digit = 0;
-					if (c >= '0' && c <= '9') {
-						digit = static_cast<uint64_t>(c - '0');
-					} else if (c >= 'a' && c <= 'f') {
-						digit = static_cast<uint64_t>(c - 'a') + 10;
-					} else if (c == '-' && field == Field::Start) {
-						field = Field::End;
-						break;
-					} else if (c == ' ' && field == Field::End) {
-						field     = Field::Perms;
-						perms_len = 0;
-						break;
-					} else {
-						line_valid = false;
-						break;
-					}
-
-					auto& value = (field == Field::Start ? start : end);
-					value       = (value << 4u) | digit;
-					break;
-				}
-
-				case Field::Perms: {
-					if (c != ' ') {
-						if (perms_len < sizeof(perms)) {
-							perms[perms_len] = c;
-						}
-						perms_len++;
-						break;
-					}
-
-					if (vaddr < start) {
-						done = true;
-					} else if (vaddr < end && perms_len >= 2) {
-						result = perms[1] == 'w'   ? READ_WRITE_PROTECTION
-						         : perms[0] == 'r' ? READ_ONLY_PROTECTION
-						                           : NO_ACCESS_PROTECTION;
-						done   = true;
-					} else {
-						field = Field::Rest;
-					}
-					break;
-				}
-
-				case Field::Rest: break;
-			}
-		}
-	}
-
-	::close(fd);
-	return result;
-}
-#endif
-
 class SpinGuard final {
 public:
 	explicit SpinGuard(std::atomic_flag& lock): m_lock(lock) {
@@ -301,31 +156,46 @@ uint64_t PageEnd(uint64_t vaddr, uint64_t size) {
 
 struct PageManager::Impl {
 	struct PageState {
-		std::atomic_flag lock                 = ATOMIC_FLAG_INIT;
-		uint32_t         mappings             = 0;
-		uint32_t         gpu_read_mappings    = 0;
-		uint32_t         gpu_write_mappings   = 0;
-		uint32_t         write_watchers       = 0;
-		uint32_t         access_watchers      = 0;
-		uint32_t         original_protection  = 0;
-		uint32_t         backing_writer       = 0;
-#if defined(__linux__)
+		std::atomic_flag lock                = ATOMIC_FLAG_INIT;
+		uint32_t         write_watchers      = 0;
+		uint32_t         access_watchers     = 0;
+		uint32_t         original_protection = 0;
+		uint32_t         backing_writer      = 0;
 		// Shadow the protection applied through Protect().
-		uint32_t current_protection = UNKNOWN_PROTECTION;
-#endif
-		bool             resolving            = false;
-		bool             resolving_read_write = false;
-		bool             late_read_pending    = false;
-		bool             late_write_pending   = false;
+		uint32_t current_protection   = UNKNOWN_PROTECTION;
+		bool     resolving            = false;
+		bool     resolving_read_write = false;
+		bool     late_read_pending    = false;
+		bool     late_write_pending   = false;
 	};
 
 	struct Region {
 		std::array<PageState, REGION_PAGES> pages;
 	};
 
+	class PageRangeGuard final {
+	public:
+		explicit PageRangeGuard(std::span<PageState*> pages): m_pages(pages) {
+			for (auto* page: m_pages) {
+				while (page->lock.test_and_set(std::memory_order_acquire)) {
+					std::atomic_signal_fence(std::memory_order_seq_cst);
+				}
+			}
+		}
+		~PageRangeGuard() {
+			for (auto it = m_pages.rbegin(); it != m_pages.rend(); ++it) {
+				(*it)->lock.clear(std::memory_order_release);
+			}
+		}
+		KYTY_CLASS_NO_COPY(PageRangeGuard);
+
+	private:
+		std::span<PageState*> m_pages;
+	};
+
 	Impl(PageFaultHandler handler, void* context): fault_handler(handler), fault_context(context) {
 		if (fault_handler == nullptr) {
-			Fatal("null fault handler");
+			Fatal("null page-manager fault callback");
 		}
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
 		SYSTEM_INFO info {};
@@ -337,8 +207,7 @@ struct PageManager::Impl {
 #elif defined(__APPLE__)
 		// Under Rosetta the host page size is 4 KB, matching TRACKER_PAGE_SIZE.
 		if (static_cast<uint64_t>(getpagesize()) != PAGE_SIZE) {
-			Fatal("unsupported host page size 0x%08" PRIx32,
-			      static_cast<uint32_t>(getpagesize()));
+			Fatal("unsupported host page size 0x%08" PRIx32, static_cast<uint32_t>(getpagesize()));
 		}
 #else
 		const auto host_page_size = ::sysconf(_SC_PAGESIZE);
@@ -356,9 +225,8 @@ struct PageManager::Impl {
 		for (const auto& region: region_storage) {
 			for (auto& page: region->pages) {
 				SpinGuard lock(page.lock);
-				if (page.mappings != 0 || page.gpu_read_mappings != 0 ||
-				    page.gpu_write_mappings != 0 || page.write_watchers != 0 ||
-				    page.access_watchers != 0 || page.backing_writer != 0 || page.resolving) {
+				if (page.write_watchers != 0 || page.access_watchers != 0 ||
+				    page.backing_writer != 0 || page.resolving) {
 					FailFast("PageManager destroyed with live page state");
 				}
 			}
@@ -405,130 +273,65 @@ struct PageManager::Impl {
 		if (old_protection == NO_ACCESS_PROTECTION && new_protection != NO_ACCESS_PROTECTION) {
 			page.late_read_pending = true;
 		}
-		if ((old_protection == NO_ACCESS_PROTECTION ||
-		     old_protection == READ_ONLY_PROTECTION) &&
+		if ((old_protection == NO_ACCESS_PROTECTION || old_protection == READ_ONLY_PROTECTION) &&
 		    new_protection == READ_WRITE_PROTECTION) {
 			page.late_write_pending = true;
 		}
 	}
 
-	static uint32_t QueryProtection([[maybe_unused]] PageState& page, uint64_t vaddr) {
-#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
-		MEMORY_BASIC_INFORMATION info {};
-		if (VirtualQuery(reinterpret_cast<const void*>(static_cast<uintptr_t>(vaddr)), &info,
-		                 sizeof(info)) == 0 ||
-		    info.State != MEM_COMMIT || info.Protect != PAGE_READWRITE) {
-			Fatal("basic path requires PAGE_READWRITE at 0x%016" PRIx64 " (state=0x%08" PRIx32
-			      ", protection=0x%08" PRIx32 ")",
-			      vaddr, static_cast<uint32_t>(info.State), static_cast<uint32_t>(info.Protect));
+	static void InitializeProtection(std::span<PageState*> pages) {
+		for (auto* page: pages) {
+			page->original_protection = READ_WRITE_PROTECTION;
+			page->current_protection  = READ_WRITE_PROTECTION;
 		}
-		return info.Protect;
-#elif defined(__APPLE__)
-		const uint32_t protection = MachQueryPageProt(vaddr);
-		if (protection != PAGE_READWRITE) {
-			Fatal("basic path requires PAGE_READWRITE at 0x%016" PRIx64 " (protection=0x%08" PRIx32
-			      ")",
-			      vaddr, protection);
-		}
-		return protection;
-#else
-		const auto host_protection = QueryHostProtection(vaddr);
-		if (host_protection != READ_WRITE_PROTECTION) {
-			Fatal("basic path requires a read/write mapping at 0x%016" PRIx64
-			      " (protection=0x%08" PRIx32 ")",
-			      vaddr, host_protection);
-		}
-		page.current_protection = host_protection;
-		return host_protection;
-#endif
 	}
 
-	static bool AllowsAccess([[maybe_unused]] const PageState& page, uint64_t vaddr,
+	static bool AllowsAccess(const PageState& page, [[maybe_unused]] uint64_t vaddr,
 	                         PageFaultAccess access) noexcept {
-#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
-		MEMORY_BASIC_INFORMATION info {};
-		if (VirtualQuery(reinterpret_cast<const void*>(static_cast<uintptr_t>(vaddr)), &info,
-		                 sizeof(info)) == 0 ||
-		    info.State != MEM_COMMIT) {
-			return false;
-		}
 		switch (access) {
 			case PageFaultAccess::Read:
-				return info.Protect == PAGE_READONLY || info.Protect == PAGE_READWRITE;
-			case PageFaultAccess::Write: return info.Protect == PAGE_READWRITE;
+				return page.current_protection == READ_ONLY_PROTECTION ||
+				       page.current_protection == READ_WRITE_PROTECTION;
+			case PageFaultAccess::Write: return page.current_protection == READ_WRITE_PROTECTION;
 			default: return false;
 		}
-#elif defined(__APPLE__)
-		const uint32_t protection = MachQueryPageProt(vaddr);
-		switch (access) {
-			case PageFaultAccess::Read:
-				return protection == PAGE_READONLY || protection == PAGE_READWRITE;
-			case PageFaultAccess::Write: return protection == PAGE_READWRITE;
-			default: return false;
-		}
-#else
-		const auto permitted = [](uint32_t protection, PageFaultAccess wanted) {
-			switch (wanted) {
-				case PageFaultAccess::Read:
-					return protection == READ_ONLY_PROTECTION || protection == READ_WRITE_PROTECTION;
-				case PageFaultAccess::Write: return protection == READ_WRITE_PROTECTION;
-				default: return false;
-			}
-		};
-
-		if (!permitted(page.current_protection, access)) {
-			return false;
-		}
-		return permitted(QueryHostProtection(vaddr), access);
-#endif
 	}
 
-	static void Protect([[maybe_unused]] PageState& page, uint64_t vaddr, uint32_t protection,
-	                    uint32_t expected_old, bool fault_path) noexcept {
-#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
-		DWORD old_protection = 0;
-		if (VirtualProtect(reinterpret_cast<void*>(static_cast<uintptr_t>(vaddr)), PAGE_SIZE,
-		                   protection, &old_protection) == 0 ||
-		    old_protection != expected_old) {
-			if (fault_path) {
-				FailFast("VirtualProtect fault transition did not match expected protection");
-			}
-			Fatal("invalid protection transition at 0x%016" PRIx64 ", old=0x%08" PRIx32
-			      ", expected=0x%08" PRIx32 ", new=0x%08" PRIx32,
-			      vaddr, static_cast<uint32_t>(old_protection), expected_old, protection);
+	void ProtectRange(std::span<PageState*> pages, uint64_t vaddr, uint32_t protection,
+	                  std::span<const uint32_t> expected_old, bool fault_path) noexcept {
+		const auto size = pages.size() * PAGE_SIZE;
+		if (pages.size() != expected_old.size()) {
+			FailFast("protection range state size mismatch");
 		}
-#elif defined(__APPLE__)
-		// mprotect cannot report the previous protection, so the expected_old comparison
-		// is dropped; the tracker is the sole mutator of these pages and drives the
-		// transition from its own shadow state.
-		(void)expected_old;
-		if (mprotect(reinterpret_cast<void*>(static_cast<uintptr_t>(vaddr)), PAGE_SIZE,
-		             PageProtToPosix(protection)) != 0) {
-			if (fault_path) {
-				FailFast("mprotect fault transition failed");
+		for (size_t i = 0; i < pages.size(); i++) {
+			const auto actual = pages[i]->current_protection;
+			if (actual != UNKNOWN_PROTECTION && actual != expected_old[i]) {
+				if (fault_path) {
+					FailFast("mprotect fault transition did not match expected protection");
+				}
+				Fatal("invalid protection transition at 0x%016" PRIx64 ", old=0x%08" PRIx32
+				      ", expected=0x%08" PRIx32 ", new=0x%08" PRIx32,
+				      vaddr + i * PAGE_SIZE, actual, expected_old[i], protection);
 			}
-			Fatal("mprotect failed at 0x%016" PRIx64 ", new=0x%08" PRIx32, vaddr, protection);
 		}
-#else
-		if (page.current_protection != UNKNOWN_PROTECTION &&
-		    page.current_protection != expected_old) {
+		if (!Libs::LibKernel::Memory::ProtectGuestHostMemory(vaddr, size,
+		                                                     ToMemoryMode(protection))) {
 			if (fault_path) {
-				FailFast("mprotect fault transition did not match expected protection");
+				FailFast("address-space fault protection transition failed");
 			}
-			Fatal("invalid protection transition at 0x%016" PRIx64 ", old=0x%08" PRIx32
-			      ", expected=0x%08" PRIx32 ", new=0x%08" PRIx32,
-			      vaddr, page.current_protection, expected_old, protection);
+			Fatal("address-space protection failed at 0x%016" PRIx64 ", new=0x%08" PRIx32, vaddr,
+			      protection);
 		}
-		if (::mprotect(reinterpret_cast<void*>(static_cast<uintptr_t>(vaddr)), PAGE_SIZE,
-		               ToHostProtection(protection)) != 0) {
-			if (fault_path) {
-				FailFast("mprotect failed on the fault path");
-			}
-			Fatal("mprotect failed at 0x%016" PRIx64 ", new=0x%08" PRIx32 " (%s)", vaddr,
-			      protection, std::strerror(errno));
+		for (auto* page: pages) {
+			page->current_protection = protection;
 		}
-		page.current_protection = protection;
-#endif
+	}
+
+	void Protect(PageState& page, uint64_t vaddr, uint32_t protection, uint32_t expected_old,
+	             bool fault_path) noexcept {
+		PageState* pages[]    = {&page};
+		uint32_t   expected[] = {expected_old};
+		ProtectRange(pages, vaddr, protection, expected, fault_path);
 	}
 
 	std::unique_ptr<std::atomic<Region*>[]> regions;
@@ -565,201 +368,141 @@ bool PageManager::IsTracked(uint64_t vaddr) const noexcept {
 	return page.write_watchers != 0 || page.access_watchers != 0;
 }
 
-bool PageManager::IsMapped(uint64_t vaddr, uint64_t size) const noexcept {
-	if (vaddr == 0 || size == 0 || vaddr >= ADDRESS_SIZE || size > ADDRESS_SIZE - vaddr) {
-		return false;
-	}
-	const auto end = PageStart(vaddr + size - 1) + PAGE_SIZE;
-	for (auto page_vaddr = PageStart(vaddr); page_vaddr < end; page_vaddr += PAGE_SIZE) {
-		auto* region = m_impl->FindRegion(page_vaddr);
-		if (region == nullptr) {
-			return false;
-		}
-		auto&     page = m_impl->GetPage(*region, page_vaddr);
-		SpinGuard lock(page.lock);
-		if (page.mappings == 0) {
-			return false;
-		}
-	}
-	return true;
-}
-
-bool PageManager::HasAnyMapping(uint64_t vaddr, uint64_t size) const noexcept {
-	if (g_in_fault_resolution || vaddr == 0 || size == 0 || vaddr >= ADDRESS_SIZE ||
-	    size > ADDRESS_SIZE - vaddr) {
-		return false;
-	}
-	const auto end = PageEnd(vaddr, size);
-	for (auto page_vaddr = PageStart(vaddr); page_vaddr < end; page_vaddr += PAGE_SIZE) {
-		auto* region = m_impl->FindRegion(page_vaddr);
-		if (region == nullptr) {
-			continue;
-		}
-		auto&     page = m_impl->GetPage(*region, page_vaddr);
-		SpinGuard lock(page.lock);
-		if (page.mappings != 0) {
-			return true;
-		}
-	}
-	return false;
-}
-
-bool PageManager::HasGpuAccess(uint64_t vaddr, uint64_t size, GpuAccess access) const noexcept {
-	if (access != GpuAccess::Read && access != GpuAccess::Write && access != GpuAccess::ReadWrite) {
-		FailFast("HasGpuAccess received an invalid GPU access mode");
-	}
-	const bool need_read  = access == GpuAccess::Read || access == GpuAccess::ReadWrite;
-	const bool need_write = access == GpuAccess::Write || access == GpuAccess::ReadWrite;
-	if (vaddr == 0 || size == 0 || vaddr >= ADDRESS_SIZE || size > ADDRESS_SIZE - vaddr) {
-		return false;
-	}
-	const auto end = PageEnd(vaddr, size);
-	for (auto addr = PageStart(vaddr); addr < end; addr += PAGE_SIZE) {
-		auto* region = m_impl->FindRegion(addr);
-		if (region == nullptr) {
-			return false;
-		}
-		auto&     page = m_impl->GetPage(*region, addr);
-		SpinGuard lock(page.lock);
-		if ((need_read && page.gpu_read_mappings == 0) ||
-		    (need_write && page.gpu_write_mappings == 0)) {
-			return false;
-		}
-	}
-	return true;
-}
-
 void PageManager::UpdatePageWatchers(bool track, uint64_t vaddr, uint64_t size,
                                      PageWatchMode mode) {
 	if (mode != PageWatchMode::Write && mode != PageWatchMode::ReadWrite) {
 		Fatal("invalid watcher mode");
 	}
-	const auto end = PageEnd(vaddr, size);
-	for (auto page_vaddr = PageStart(vaddr); page_vaddr < end; page_vaddr += PAGE_SIZE) {
-		auto* region =
-		    track ? m_impl->GetOrCreateRegion(page_vaddr) : m_impl->FindRegion(page_vaddr);
+	const auto begin = PageStart(vaddr);
+	const auto end   = PageEnd(vaddr, size);
+	for (auto chunk_begin = begin; chunk_begin < end;) {
+		const auto chunk_end = std::min(end, (chunk_begin / REGION_SIZE + 1) * REGION_SIZE);
+		auto*      region =
+		    track ? m_impl->GetOrCreateRegion(chunk_begin) : m_impl->FindRegion(chunk_begin);
 		if (region == nullptr) {
-			Fatal("untracking unknown page 0x%016" PRIx64, page_vaddr);
+			Fatal("untracking unknown page 0x%016" PRIx64, chunk_begin);
 		}
-		auto&     page = m_impl->GetPage(*region, page_vaddr);
-		SpinGuard lock(page.lock);
-		if (page.resolving && track) {
-			FailFast("new page watcher raced active fault resolution");
+
+		const auto page_count = static_cast<size_t>((chunk_end - chunk_begin) / PAGE_SIZE);
+		std::vector<Impl::PageState*> pages;
+		pages.reserve(page_count);
+		for (auto address = chunk_begin; address < chunk_end; address += PAGE_SIZE) {
+			pages.push_back(&m_impl->GetPage(*region, address));
 		}
-		if (page.mappings == 0) {
-			Fatal("watching unmapped page 0x%016" PRIx64, page_vaddr);
+		Impl::PageRangeGuard lock(pages);
+
+		std::vector<uint8_t> first_watchers(page_count);
+		for (size_t i = 0; i < page_count; i++) {
+			auto&      page    = *pages[i];
+			const auto address = chunk_begin + i * PAGE_SIZE;
+			if (page.resolving && track) {
+				FailFast("new page watcher raced active fault resolution");
+			}
+			auto& watchers =
+			    (mode == PageWatchMode::ReadWrite ? page.access_watchers : page.write_watchers);
+			if (track) {
+				if (watchers == std::numeric_limits<uint32_t>::max()) {
+					Fatal("watcher overflow at 0x%016" PRIx64, address);
+				}
+				first_watchers[i] = page.write_watchers == 0 && page.access_watchers == 0;
+			} else {
+				if (watchers == 0) {
+					Fatal("watcher underflow at 0x%016" PRIx64, address);
+				}
+				if (page.backing_writer != 0 && page.backing_writer != CurrentThread()) {
+					Fatal("backing write ownership changed at 0x%016" PRIx64, address);
+				}
+			}
 		}
-		auto& watchers =
-		    (mode == PageWatchMode::ReadWrite ? page.access_watchers : page.write_watchers);
+
 		if (track) {
-			if (watchers == std::numeric_limits<uint32_t>::max()) {
-				Fatal("watcher overflow at 0x%016" PRIx64, page_vaddr);
-			}
-			const bool first_watcher = page.write_watchers == 0 && page.access_watchers == 0;
-			if (first_watcher) {
-				page.original_protection = Impl::QueryProtection(page, page_vaddr);
+			for (size_t first = 0; first < page_count;) {
+				while (first < page_count && first_watchers[first] == 0) {
+					first++;
+				}
+				auto last = first;
+				while (last < page_count && first_watchers[last] != 0) {
+					last++;
+				}
+				if (first != last) {
+					Impl::InitializeProtection(std::span {pages}.subspan(first, last - first));
+				}
+				first = last;
 			}
+		}
+
+		std::vector<uint32_t> old_protections(page_count);
+		std::vector<uint32_t> new_protections(page_count);
+		std::vector<uint8_t>  transitions(page_count);
+		for (size_t i = 0; i < page_count; i++) {
+			auto& page = *pages[i];
+			auto& watchers =
+			    (mode == PageWatchMode::ReadWrite ? page.access_watchers : page.write_watchers);
 			const auto old_protection = Impl::WatcherProtection(page);
-			watchers++;
-			const auto new_protection = Impl::WatcherProtection(page);
-			if (new_protection != old_protection) {
-				Impl::Protect(page, page_vaddr, new_protection, old_protection, false);
+			if (track) {
+				watchers++;
+			} else {
+				watchers--;
 			}
-			switch (new_protection) {
-				case NO_ACCESS_PROTECTION:
-					page.late_read_pending  = false;
-					page.late_write_pending = false;
+			const auto new_protection = Impl::WatcherProtection(page);
+			old_protections[i]        = old_protection;
+			new_protections[i]        = new_protection;
+			if (new_protection != old_protection && (track || page.backing_writer == 0)) {
+				transitions[i] = 1;
+			}
+		}
+
+		for (size_t first = 0; first < page_count;) {
+			while (first < page_count && transitions[first] == 0) {
+				first++;
+			}
+			if (first == page_count) {
+				break;
+			}
+			const auto protection = new_protections[first];
+			auto       current    = first + 1;
+			auto       last       = current;
+			for (; current < page_count && new_protections[current] == protection; current++) {
+				if (old_protections[current] != new_protections[current] &&
+				    transitions[current] == 0) {
 					break;
-				case READ_ONLY_PROTECTION: page.late_write_pending = false; break;
-				default: break;
+				}
+				if (transitions[current] != 0) {
+					last = current + 1;
+				}
 			}
-		} else {
-			if (watchers == 0) {
-				Fatal("watcher underflow at 0x%016" PRIx64, page_vaddr);
-			}
-			if (page.backing_writer != 0 && page.backing_writer != CurrentThread()) {
-				Fatal("backing write ownership changed at 0x%016" PRIx64, page_vaddr);
-			}
-			const auto old_protection = Impl::WatcherProtection(page);
-			watchers--;
-			const auto new_protection = Impl::WatcherProtection(page);
-			if (page.backing_writer == 0 && new_protection != old_protection) {
-				Impl::Protect(page, page_vaddr, new_protection, old_protection, false);
-			}
-			if (page.backing_writer == 0) {
-				Impl::PublishDelayedFaults(page, old_protection, new_protection);
-			}
-			if (page.backing_writer == 0 && page.write_watchers == 0 && page.access_watchers == 0) {
-				page.original_protection = 0;
+			m_impl->ProtectRange(std::span {pages}.subspan(first, last - first),
+			                     chunk_begin + first * PAGE_SIZE, protection,
+			                     std::span {old_protections}.subspan(first, last - first), false);
+			first = current;
+		}
+
+		for (size_t i = 0; i < page_count; i++) {
+			auto&      page       = *pages[i];
+			const auto protection = new_protections[i];
+			if (track) {
+				switch (protection) {
+					case NO_ACCESS_PROTECTION:
+						page.late_read_pending  = false;
+						page.late_write_pending = false;
+						break;
+					case READ_ONLY_PROTECTION: page.late_write_pending = false; break;
+					default: break;
+				}
+			} else if (page.backing_writer == 0) {
+				Impl::PublishDelayedFaults(page, old_protections[i], protection);
+				if (page.write_watchers == 0 && page.access_watchers == 0) {
+					page.original_protection = 0;
+				}
 			}
 		}
+		chunk_begin = chunk_end;
 	}
 }
 
-void PageManager::OnGpuMap(uint64_t vaddr, uint64_t size, GpuAccess access) {
-	if (g_in_fault_resolution) {
-		FailFast("GPU mapping changed during fault resolution");
-	}
-	if (access != GpuAccess::Read && access != GpuAccess::Write && access != GpuAccess::ReadWrite) {
-		FailFast("GPU map received an invalid access mode");
-	}
-	const bool gpu_read  = access == GpuAccess::Read || access == GpuAccess::ReadWrite;
-	const bool gpu_write = access == GpuAccess::Write || access == GpuAccess::ReadWrite;
-	const auto end       = PageEnd(vaddr, size);
-	for (auto addr = PageStart(vaddr); addr < end; addr += PAGE_SIZE) {
-		auto&     page = m_impl->GetPage(*m_impl->GetOrCreateRegion(addr), addr);
-		SpinGuard lock(page.lock);
-		if (page.resolving || page.mappings == std::numeric_limits<uint32_t>::max() ||
-		    (gpu_read && page.gpu_read_mappings == std::numeric_limits<uint32_t>::max()) ||
-		    (gpu_write && page.gpu_write_mappings == std::numeric_limits<uint32_t>::max())) {
-			Fatal("invalid map state at 0x%016" PRIx64, addr);
-		}
-		page.mappings++;
-		page.gpu_read_mappings += gpu_read ? 1u : 0u;
-		page.gpu_write_mappings += gpu_write ? 1u : 0u;
-#if defined(__linux__)
-		// New guest mappings start read/write.
-		if (page.current_protection == UNKNOWN_PROTECTION) {
-			page.current_protection = READ_WRITE_PROTECTION;
-		}
-#endif
-	}
-}
+void PageManager::OnGpuMap(uint64_t, uint64_t) {}
 
-void PageManager::OnGpuUnmap(uint64_t vaddr, uint64_t size, GpuAccess access) {
-	if (g_in_fault_resolution) {
-		FailFast("GPU unmapping changed during fault resolution");
-	}
-	if (access != GpuAccess::Read && access != GpuAccess::Write && access != GpuAccess::ReadWrite) {
-		FailFast("GPU unmap received an invalid access mode");
-	}
-	const bool gpu_read  = access == GpuAccess::Read || access == GpuAccess::ReadWrite;
-	const bool gpu_write = access == GpuAccess::Write || access == GpuAccess::ReadWrite;
-	const auto end       = PageEnd(vaddr, size);
-	for (auto page_vaddr = PageStart(vaddr); page_vaddr < end; page_vaddr += PAGE_SIZE) {
-		auto* region = m_impl->FindRegion(page_vaddr);
-		if (region == nullptr) {
-			Fatal("unmapping unknown page 0x%016" PRIx64, page_vaddr);
-		}
-		auto&     page = m_impl->GetPage(*region, page_vaddr);
-		SpinGuard lock(page.lock);
-		if (page.resolving || page.mappings == 0 || (gpu_read && page.gpu_read_mappings == 0) ||
-		    (gpu_write && page.gpu_write_mappings == 0) ||
-		    (page.mappings == 1 && (page.write_watchers != 0 || page.access_watchers != 0))) {
-			Fatal("invalid unmap state at 0x%016" PRIx64, page_vaddr);
-		}
-		page.mappings--;
-		page.gpu_read_mappings -= gpu_read ? 1u : 0u;
-		page.gpu_write_mappings -= gpu_write ? 1u : 0u;
-		if (page.mappings == 0) {
-			if (page.gpu_read_mappings != 0 || page.gpu_write_mappings != 0) {
-				FailFast("GPU unmap left nonzero GPU mapping counts");
-			}
-			page.late_read_pending  = false;
-			page.late_write_pending = false;
-		}
-	}
-}
+void PageManager::OnGpuUnmap(uint64_t, uint64_t) {}
 
 PageManager::BackingWrite::BackingWrite(PageManager& manager, uint64_t vaddr,
                                         uint64_t size) noexcept
@@ -815,8 +558,7 @@ void PageManager::BeginBackingWrite(uint64_t vaddr, uint64_t size) noexcept {
 		}
 		auto&     page = m_impl->GetPage(*region, address);
 		SpinGuard lock(page.lock);
-		if (page.mappings == 0 || page.resolving || page.backing_writer != 0 ||
-		    page.access_watchers == 0) {
+		if (page.resolving || page.backing_writer != 0 || page.access_watchers == 0) {
 			Fatal("backing write races page resolution at 0x%016" PRIx64, address);
 		}
 		page.resolving            = true;
@@ -844,7 +586,7 @@ void PageManager::EndBackingWrite(uint64_t vaddr, uint64_t size) noexcept {
 		const auto old_protection = NO_ACCESS_PROTECTION;
 		const auto new_protection = Impl::WatcherProtection(page);
 		if (new_protection != old_protection) {
-			Impl::Protect(page, address, new_protection, old_protection, false);
+			m_impl->Protect(page, address, new_protection, old_protection, false);
 		}
 		Impl::PublishDelayedFaults(page, old_protection, new_protection);
 		if (page.write_watchers == 0 && page.access_watchers == 0) {
@@ -945,7 +687,8 @@ bool PageManager::HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noex
 				page.write_watchers = 0;
 			}
 			const auto restored_protection = Impl::WatcherProtection(page);
-			Impl::Protect(page, PageStart(fault_vaddr), restored_protection, old_protection, true);
+			m_impl->Protect(page, PageStart(fault_vaddr), restored_protection, old_protection,
+			                true);
 			if (page.write_watchers == 0) {
 				page.original_protection = 0;
 			}
@@ -966,22 +709,4 @@ bool PageManager::HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noex
 	return true;
 }
 
-bool PageManager::HandleWriteRange(uint64_t vaddr, uint64_t size) noexcept {
-	if (g_in_fault_resolution || vaddr == 0 || size == 0 || vaddr >= ADDRESS_SIZE ||
-	    size > ADDRESS_SIZE - vaddr) {
-		return false;
-	}
-	const auto end = PageEnd(vaddr, size);
-	for (auto page_vaddr = PageStart(vaddr); page_vaddr < end; page_vaddr += PAGE_SIZE) {
-		if (!IsMapped(page_vaddr, 1)) {
-			continue;
-		}
-		const auto fault_vaddr = std::max(page_vaddr, vaddr);
-		if (!HandleFault(PageFaultAccess::Write, fault_vaddr)) {
-			return false;
-		}
-	}
-	return true;
-}
-
 } // namespace Libs::Graphics

src/graphics/host_gpu/pageManager.h

@@ -13,11 +13,9 @@ namespace Libs::Graphics {
 enum class PageFaultAccess { Read, Write, Execute, Unknown };
 enum class PageFaultPhase { Invalidate, Complete, Release };
 enum class PageWatchMode { Write, ReadWrite };
-enum class GpuAccess { Read, Write, ReadWrite };
 
 using PageFaultHandler = bool (*)(void* context, PageFaultAccess access, uint64_t vaddr,
                                   uint64_t size, PageFaultPhase phase) noexcept;
-
 class PageManager final {
 public:
 	class BackingWrite final {
@@ -40,19 +38,15 @@ public:
 
 	[[nodiscard]] uint64_t GetPageSize() const;
 	[[nodiscard]] bool     IsTracked(uint64_t vaddr) const noexcept;
-	[[nodiscard]] bool     IsMapped(uint64_t vaddr, uint64_t size) const noexcept;
-	[[nodiscard]] bool     HasAnyMapping(uint64_t vaddr, uint64_t size) const noexcept;
-	[[nodiscard]] bool HasGpuAccess(uint64_t vaddr, uint64_t size, GpuAccess access) const noexcept;
 
 	void UpdatePageWatchers(bool track, uint64_t vaddr, uint64_t size,
 	                        PageWatchMode mode = PageWatchMode::Write);
-	void OnGpuMap(uint64_t vaddr, uint64_t size, GpuAccess access = GpuAccess::ReadWrite);
-	void OnGpuUnmap(uint64_t vaddr, uint64_t size, GpuAccess access = GpuAccess::ReadWrite);
+	void OnGpuMap(uint64_t vaddr, uint64_t size);
+	void OnGpuUnmap(uint64_t vaddr, uint64_t size);
 
 	[[nodiscard]] bool HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noexcept;
-	[[nodiscard]] bool HandleWriteRange(uint64_t vaddr, uint64_t size) noexcept;
 	[[nodiscard]] std::vector<std::unique_ptr<BackingWrite>>
-	     ReserveBackingWrites(std::span<const RangeSet::Range> ranges);
+	ReserveBackingWrites(std::span<const RangeSet::Range> ranges);
 
 private:
 	void BeginBackingWrite(uint64_t vaddr, uint64_t size) noexcept;

src/graphics/host_gpu/rangeSet.h

@@ -59,6 +59,16 @@ public:
 		return result;
 	}
 
+	[[nodiscard]] bool Contains(uint64_t address, uint64_t size) const {
+		const auto end = End(address, size);
+		auto       it  = m_ranges.upper_bound(address);
+		if (it == m_ranges.begin()) {
+			return false;
+		}
+		--it;
+		return it->first <= address && it->second >= end;
+	}
+
 	template <typename Func>
 	void ForEachIntersection(uint64_t address, uint64_t size, Func&& func) const {
 		const auto end = End(address, size);

src/graphics/host_gpu/renderer/cache/bufferCache.cpp

@@ -4,10 +4,10 @@
 #include "common/logging/log.h"
 #include "common/profiler.h"
 #include "graphics/host_gpu/graphicContext.h"
-#include "graphics/host_gpu/renderer/commandScheduler.h"
-#include "graphics/host_gpu/renderer/render.h"
 #include "graphics/host_gpu/renderer/cache/resourceMutex.h"
 #include "graphics/host_gpu/renderer/cache/textureCache.h"
+#include "graphics/host_gpu/renderer/commandScheduler.h"
+#include "graphics/host_gpu/renderer/render.h"
 #include "kernel/memory.h"
 
 #include <algorithm>
@@ -183,9 +183,8 @@ BufferCache::RecordDownloads(std::span<const DownloadCopy> copies) {
 		return {};
 	}
 
-	auto& download = m_download_buffer;
-	const auto [mapped, base_offset] =
-	    download.Map(reservation_size, DOWNLOAD_ALIGNMENT);
+	auto& download                   = m_download_buffer;
+	const auto [mapped, base_offset] = download.Map(reservation_size, DOWNLOAD_ALIGNMENT);
 	if (mapped == nullptr) {
 		EXIT("BufferCache: download batch could not reserve the shared stream\n");
 	}
@@ -215,40 +214,38 @@ void BufferCache::PublishDownloads(std::span<const DownloadRange> downloads) {
 	}
 }
 
-void BufferCache::QueueGarbageDownload(std::span<const DownloadCopy> copies,
-                                       RetiredBuffer retire) {
+void BufferCache::QueueGarbageDownload(std::span<const DownloadCopy> copies, RetiredBuffer retire) {
 	if (copies.empty()) {
 		return;
 	}
-	auto downloads  = RecordDownloads(copies);
-	const auto tick = m_scheduler.CurrentTick();
+	auto       downloads = RecordDownloads(copies);
+	const auto tick      = m_scheduler.CurrentTick();
 	BeginBackingPublication(retire.address, retire.size, tick);
-	m_scheduler.DeferOperation(
-	    [this, downloads = std::move(downloads), retire = std::move(retire), tick]() mutable {
-		    PublishDownloads(downloads);
-		    {
-			    FaultSafeCacheLock lock(this, m_mutex);
-			    if (m_memory_tracker.IsRegionGpuModified(retire.address, retire.size)) {
-				    m_memory_tracker.ForEachDownloadRange<true>(
-				        retire.address, retire.size,
-				        [&](uint64_t address, uint64_t size) noexcept {
-					        m_memory_tracker.ValidateGpuDirtyPages(
-					            m_gpu_modified_ranges, address, size,
-					            "asynchronous garbage retirement");
-				        },
-				        [](uint64_t, uint64_t) noexcept {});
-			    }
-			    for (const auto& range: downloads) {
-				    m_gpu_modified_ranges.Subtract(range.address, range.size);
-			    }
-			    if (m_memory_tracker.IsRegionGpuModified(retire.address, retire.size) ||
-			        !m_gpu_modified_ranges.Intersections(retire.address, retire.size).empty()) {
-				    EXIT("BufferCache: asynchronous garbage collection retained GPU ownership\n");
-			    }
-			    m_memory_tracker.UntrackMemory(retire.address, retire.size);
-		    }
-		    CompleteBackingPublication(retire.address, retire.size, tick);
-	    });
+	m_scheduler.DeferOperation([this, downloads = std::move(downloads), retire = std::move(retire),
+	                            tick]() mutable {
+		PublishDownloads(downloads);
+		{
+			FaultSafeCacheLock lock(this, m_mutex);
+			if (m_memory_tracker.IsRegionGpuModified(retire.address, retire.size)) {
+				m_memory_tracker.ForEachDownloadRange<true>(
+				    retire.address, retire.size,
+				    [&](uint64_t address, uint64_t size) noexcept {
+					    m_memory_tracker.ValidateGpuDirtyPages(m_gpu_modified_ranges, address, size,
+					                                           "asynchronous garbage retirement");
+				    },
+				    [](uint64_t, uint64_t) noexcept {});
+			}
+			for (const auto& range: downloads) {
+				m_gpu_modified_ranges.Subtract(range.address, range.size);
+			}
+			if (m_memory_tracker.IsRegionGpuModified(retire.address, retire.size) ||
+			    !m_gpu_modified_ranges.Intersections(retire.address, retire.size).empty()) {
+				EXIT("BufferCache: asynchronous garbage collection retained GPU ownership\n");
+			}
+			m_memory_tracker.UntrackMemory(retire.address, retire.size);
+		}
+		CompleteBackingPublication(retire.address, retire.size, tick);
+	});
 }
 
 BufferCache::BufferCache(GraphicContext& graphics, CommandScheduler& scheduler,
@@ -301,14 +298,13 @@ BufferCache::~BufferCache() {
 bool BufferCache::SynchronizeBacking(uint64_t vaddr, uint64_t size) {
 	bool waited = false;
 	for (;;) {
-		uint64_t tick = 0;
+		uint64_t   tick       = 0;
 		const auto page_begin = vaddr & ~(TRACKER_PAGE_SIZE - 1);
-		const auto page_end =
-		    (vaddr + size + TRACKER_PAGE_SIZE - 1) & ~(TRACKER_PAGE_SIZE - 1);
+		const auto page_end   = (vaddr + size + TRACKER_PAGE_SIZE - 1) & ~(TRACKER_PAGE_SIZE - 1);
 		CacheRange affected {.address = page_begin, .size = page_end - page_begin};
 		{
 			FaultSafeCacheLock lock(this, m_mutex);
-			bool changed = true;
+			bool               changed = true;
 			while (changed) {
 				changed = false;
 				for (const auto& [address, cached]: m_buffers) {
@@ -384,6 +380,73 @@ BufferBinding BufferCache::UploadTransient(const void* data, uint64_t size, uint
 	return {owner, owner->Handle(), 0};
 }
 
+void BufferCache::InvalidateMemory(uint64_t vaddr, uint64_t size) {
+	if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE ||
+	    size > TRACKER_ADDRESS_SIZE - vaddr) {
+		EXIT("BufferCache: invalid memory-invalidation range\n");
+	}
+	(void)SynchronizeBacking(vaddr, size);
+	if (!HasPageOverlap(vaddr, size)) {
+		return;
+	}
+	m_memory_tracker.InvalidateRegion(vaddr, size,
+	                                  [this, vaddr, size] { ReadMemory(vaddr, size); });
+}
+
+void BufferCache::ReadMemory(uint64_t vaddr, uint64_t size) {
+	(void)SynchronizeBacking(vaddr, size);
+	std::vector<DownloadCopy> copies;
+	{
+		FaultSafeCacheLock lock(this, m_mutex);
+		m_memory_tracker.ForEachDownloadRange<false>(
+		    vaddr, size,
+		    [&](uint64_t address, uint64_t bytes) noexcept {
+			    m_memory_tracker.ValidateGpuDirtyPages(m_gpu_modified_ranges, address, bytes,
+			                                           "memory invalidation");
+		    },
+		    [&](uint64_t address, uint64_t bytes) noexcept {
+			    for (const auto range: m_gpu_modified_ranges.Intersections(address, bytes)) {
+				    for (uint64_t copied = 0; copied < range.size;) {
+					    const auto copy_address = range.address + copied;
+					    auto       owner        = m_buffers.upper_bound(copy_address);
+					    if (owner == m_buffers.begin()) {
+						    EXIT("BufferCache: invalidation readback has no buffer owner\n");
+					    }
+					    auto& cached = *std::prev(owner)->second;
+					    if (!cached.buffer->IsInBounds(copy_address, 1)) {
+						    EXIT(
+						        "BufferCache: invalidation readback is outside its buffer owner\n");
+					    }
+					    const auto copy_size = std::min(range.size - copied,
+					                                    cached.vaddr + cached.size - copy_address);
+					    copies.push_back({cached.buffer, cached.buffer->Offset(copy_address),
+					                      copy_address, copy_size});
+					    copied += copy_size;
+				    }
+			    }
+		    });
+	}
+	if (copies.empty()) {
+		return;
+	}
+	auto downloads = RecordDownloads(copies);
+	m_scheduler.FinishCurrent();
+	PublishDownloads(downloads);
+	{
+		FaultSafeCacheLock lock(this, m_mutex);
+		m_memory_tracker.ForEachDownloadRange<true>(
+		    vaddr, size,
+		    [&](uint64_t address, uint64_t bytes) noexcept {
+			    m_memory_tracker.ValidateGpuDirtyPages(m_gpu_modified_ranges, address, bytes,
+			                                           "memory invalidation completion");
+		    },
+		    [](uint64_t, uint64_t) noexcept {});
+		for (const auto& range: downloads) {
+			m_gpu_modified_ranges.Subtract(range.address, range.size);
+		}
+	}
+}
+
 bool BufferCache::InvalidateMemory(PageFaultAccess access, uint64_t vaddr, uint64_t size,
                                    PageFaultPhase phase) noexcept {
 	const auto page = vaddr & ~(TRACKER_PAGE_SIZE - 1);
@@ -544,8 +607,8 @@ void BufferCache::UnmapMemory(uint64_t vaddr, uint64_t size) {
 			m_memory_tracker.ForEachDownloadRange<true>(
 			    begin, bytes,
 			    [&](uint64_t address, uint64_t download_size) noexcept {
-				    m_memory_tracker.ValidateGpuDirtyPages(
-				        m_gpu_modified_ranges, address, download_size, "unmap retirement");
+				    m_memory_tracker.ValidateGpuDirtyPages(m_gpu_modified_ranges, address,
+				                                           download_size, "unmap retirement");
 			    },
 			    [](uint64_t, uint64_t) noexcept {});
 		}
@@ -651,7 +714,6 @@ BufferBinding BufferCache::ObtainBuffer(CommandBuffer& command, uint64_t vaddr,
 	if (command.IsInvalid() || command.IsExecute()) {
 		EXIT("BufferCache: buffer request requires a recording command buffer\n");
 	}
-	ValidateGpuAccess(vaddr, size, is_read, is_written);
 	std::lock_guard transaction(m_resource_mutex);
 	(void)SynchronizeBacking(vaddr, size);
 
@@ -722,12 +784,11 @@ ImageBufferSource BufferCache::ObtainBufferForImage(uint64_t vaddr, uint64_t siz
 
 	{
 		FaultSafeCacheLock lock(this, m_mutex);
-		const bool         cpu_modified        = m_memory_tracker.IsRegionCpuModified(vaddr, size);
-		const bool         gpu_modified        = m_memory_tracker.IsRegionGpuModified(vaddr, size);
-		const auto         dirty               = m_gpu_modified_ranges.Intersections(vaddr, size);
-		const bool invalidated =
-		    !m_image_invalidated_ranges.Intersections(vaddr, size).empty();
-		const bool         requested_gpu_owned = !dirty.empty();
+		const bool         cpu_modified = m_memory_tracker.IsRegionCpuModified(vaddr, size);
+		const bool         gpu_modified = m_memory_tracker.IsRegionGpuModified(vaddr, size);
+		const auto         dirty        = m_gpu_modified_ranges.Intersections(vaddr, size);
+		const bool invalidated = !m_image_invalidated_ranges.Intersections(vaddr, size).empty();
+		const bool requested_gpu_owned = !dirty.empty();
 		m_memory_tracker.ValidateGpuDirtyOwnership(m_gpu_modified_ranges, vaddr, size,
 		                                           "image source");
 
@@ -807,8 +868,8 @@ ImageBufferSource BufferCache::ObtainBufferForImage(uint64_t vaddr, uint64_t siz
 	}
 
 	FaultSafeCacheLock lock(this, m_mutex);
-	const auto         dirty               = m_gpu_modified_ranges.Intersections(vaddr, size);
-	const bool invalidated = !m_image_invalidated_ranges.Intersections(vaddr, size).empty();
+	const auto         dirty       = m_gpu_modified_ranges.Intersections(vaddr, size);
+	const bool         invalidated = !m_image_invalidated_ranges.Intersections(vaddr, size).empty();
 	const bool         requested_gpu_owned = !dirty.empty();
 	auto               owner               = find_owner();
 	if (requested_gpu_owned && owner == m_buffers.end()) {
@@ -831,9 +892,8 @@ ImageBufferSource BufferCache::ObtainBufferForImage(uint64_t vaddr, uint64_t siz
 	    [&]() noexcept {
 		    for (const auto& [address, upload_size]: uploads) {
 			    cached.buffer->CopyFrom(
-			        m_scheduler.Current(), m_staging_buffer,
-			        stage_offset + address - stage_address, cached.buffer->Offset(address),
-			        upload_size, vk::AccessFlagBits::eHostWrite);
+			        m_scheduler.Current(), m_staging_buffer, stage_offset + address - stage_address,
+			        cached.buffer->Offset(address), upload_size, vk::AccessFlagBits::eHostWrite);
 		    }
 	    });
 	DiscardGpuDirtyBytesLocked(vaddr, size, "staged image source transfer");
@@ -917,9 +977,8 @@ std::pair<std::shared_ptr<Buffer>, uint64_t> BufferCache::ObtainBufferForImageWr
 	    [&]() noexcept {
 		    for (const auto& [address, upload_size]: uploads) {
 			    cached.buffer->CopyFrom(
-			        m_scheduler.Current(), m_staging_buffer,
-			        stage_offset + address - stage_address, cached.buffer->Offset(address),
-			        upload_size, vk::AccessFlagBits::eHostWrite);
+			        m_scheduler.Current(), m_staging_buffer, stage_offset + address - stage_address,
+			        cached.buffer->Offset(address), upload_size, vk::AccessFlagBits::eHostWrite);
 		    }
 	    });
 	return {cached.buffer, cached.buffer->Offset(vaddr)};
@@ -939,19 +998,18 @@ void BufferCache::FillBuffer(uint64_t vaddr, uint64_t size, uint32_t value, bool
 	if (vaddr == 0) {
 		EXIT("BufferCache: invalid fill memory address\n");
 	}
-	ValidateGpuAccess(vaddr, size, false, true);
 	(void)m_texture_cache.ClearMeta(vaddr);
 	{
 		std::lock_guard transaction(m_resource_mutex);
 		const auto      region = m_texture_cache.QueryRegion(vaddr, size);
 		if (!HasGpuDirtyBytes(vaddr, size) && !region.gpu_image_bytes) {
 			if (region.image_bytes) {
-				m_texture_cache.PrepareHostWrite(vaddr, size);
+				m_texture_cache.InvalidateMemory(vaddr, size);
 			}
 			std::array<uint32_t, 4096> values;
 			values.fill(value);
-			const std::span<const uint8_t> bytes {
-			    reinterpret_cast<const uint8_t*>(values.data()), sizeof(values)};
+			const std::span<const uint8_t> bytes {reinterpret_cast<const uint8_t*>(values.data()),
+			                                      sizeof(values)};
 			for (uint64_t offset = 0; offset < size;) {
 				const auto chunk = std::min<uint64_t>(size - offset, bytes.size());
 				WriteHostMemory(vaddr + offset, bytes.first(chunk));
@@ -981,18 +1039,12 @@ void BufferCache::CopyBuffer(uint64_t dst_vaddr, uint64_t src_vaddr, uint64_t si
 	    (src_gds && (src_vaddr > m_gds_buffer.Size() || size > m_gds_buffer.Size() - src_vaddr))) {
 		EXIT("BufferCache: invalid or overlapping copy range\n");
 	}
-	if (src_memory) {
-		ValidateGpuAccess(src_vaddr, size, true, false);
-	}
-	if (dst_memory) {
-		ValidateGpuAccess(dst_vaddr, size, false, true);
-	}
 	if (src_memory || dst_memory) {
 		std::lock_guard transaction(m_resource_mutex);
 		if (src_memory) {
 			(void)SynchronizeBacking(src_vaddr, size);
 		}
-		const auto      src_region =
+		const auto src_region =
 		    src_memory ? m_texture_cache.QueryRegion(src_vaddr, size) : TextureCache::RegionInfo {};
 		const auto dst_region =
 		    dst_memory ? m_texture_cache.QueryRegion(dst_vaddr, size) : TextureCache::RegionInfo {};
@@ -1004,7 +1056,7 @@ void BufferCache::CopyBuffer(uint64_t dst_vaddr, uint64_t src_vaddr, uint64_t si
 		    !HasGpuDirtyBytes(dst_vaddr, size) && !src_region.gpu_image_bytes &&
 		    !dst_region.gpu_image_bytes) {
 			if (dst_region.image_bytes) {
-				m_texture_cache.PrepareHostWrite(dst_vaddr, size);
+				m_texture_cache.InvalidateMemory(dst_vaddr, size);
 			}
 			std::array<uint8_t, 64 * 1024> bytes;
 			for (uint64_t offset = 0; offset < size;) {
@@ -1037,10 +1089,10 @@ void BufferCache::CopyBuffer(uint64_t dst_vaddr, uint64_t src_vaddr, uint64_t si
 		EXIT("BufferCache: resolved Vulkan copy ranges overlap\n");
 	}
 	auto& source = src.owner != nullptr ? *std::static_pointer_cast<Buffer>(src.owner)
-	               : src_gds           ? m_gds_buffer
-	                                   : m_stream_buffer;
-	auto& destination = dst.owner != nullptr ? *std::static_pointer_cast<Buffer>(dst.owner)
-	                                         : m_gds_buffer;
+	               : src_gds            ? m_gds_buffer
+	                                    : m_stream_buffer;
+	auto& destination =
+	    dst.owner != nullptr ? *std::static_pointer_cast<Buffer>(dst.owner) : m_gds_buffer;
 	if (source.Handle() != src.buffer || destination.Handle() != dst.buffer) {
 		EXIT("BufferCache: resolved copy owner does not match its Vulkan handle\n");
 	}
@@ -1107,7 +1159,7 @@ void BufferCache::BeginBackingPublication(uint64_t vaddr, uint64_t size, uint64_
 
 void BufferCache::CompleteBackingPublication(uint64_t vaddr, uint64_t size, uint64_t tick) {
 	std::lock_guard lock(m_publication_mutex);
-	const auto publication =
+	const auto      publication =
 	    std::ranges::find_if(m_pending_backing_publications, [&](const auto& pending) {
 		    return pending.address == vaddr && pending.size == size && pending.tick == tick;
 	    });
@@ -1143,19 +1195,6 @@ void BufferCache::PublishImageBuffer(uint64_t vaddr, uint64_t size) {
 	owner->second->tick_accessed_last = m_gc_tick;
 }
 
-void BufferCache::ValidateGpuAccess(uint64_t vaddr, uint64_t size, bool is_read,
-                                    bool is_written) const {
-	if ((!is_read && !is_written) || vaddr == 0 || size == 0 || size > UINT64_MAX - vaddr) {
-		EXIT("BufferCache: invalid GPU access request\n");
-	}
-	if (is_read && !m_page_manager.HasGpuAccess(vaddr, size, GpuAccess::Read)) {
-		EXIT("BufferCache: GPU-read access denied\n");
-	}
-	if (is_written && !m_page_manager.HasGpuAccess(vaddr, size, GpuAccess::Write)) {
-		EXIT("BufferCache: GPU-write access denied\n");
-	}
-}
-
 void BufferCache::RunGarbageCollector() {
 	std::lock_guard transaction(m_resource_mutex);
 	const auto      tick = m_gc_tick++;
@@ -1170,7 +1209,7 @@ void BufferCache::RunGarbageCollector() {
 	const uint64_t age        = std::min<uint64_t>(aggressive ? 80 : 160, tick);
 	const size_t   limit      = aggressive ? 64 : 32;
 
-	std::vector<RetiredBuffer> retires;
+	std::vector<RetiredBuffer>                                       retires;
 	std::vector<std::pair<RetiredBuffer, std::vector<DownloadCopy>>> dirty_retires;
 	{
 		FaultSafeCacheLock    lock(this, m_mutex);
@@ -1190,8 +1229,8 @@ void BufferCache::RunGarbageCollector() {
 		}
 		for (const auto address: candidates) {
 			auto& cached = *m_buffers.at(address);
-			m_memory_tracker.ValidateGpuDirtyOwnership(
-			    m_gpu_modified_ranges, cached.vaddr, cached.size, "garbage collection");
+			m_memory_tracker.ValidateGpuDirtyOwnership(m_gpu_modified_ranges, cached.vaddr,
+			                                           cached.size, "garbage collection");
 			retires.push_back({address, cached.size, cached.buffer});
 			// GC runs immediately before submission. Preserve every source referenced by commands
 			// already recorded in the active batch.
@@ -1205,13 +1244,13 @@ void BufferCache::RunGarbageCollector() {
 			m_memory_tracker.ForEachDownloadRange<false>(
 			    retire.address, retire.size,
 			    [&](uint64_t address, uint64_t size) noexcept {
-				    m_memory_tracker.ValidateGpuDirtyPages(
-				        m_gpu_modified_ranges, address, size, "garbage collection");
+				    m_memory_tracker.ValidateGpuDirtyPages(m_gpu_modified_ranges, address, size,
+				                                           "garbage collection");
 			    },
 			    [&](uint64_t address, uint64_t size) noexcept {
 				    for (const auto range: m_gpu_modified_ranges.Intersections(address, size)) {
-					    copies.push_back({retire.owner, range.address - retire.address, range.address,
-					                      range.size});
+					    copies.push_back({retire.owner, range.address - retire.address,
+					                      range.address, range.size});
 				    }
 			    });
 		}

src/graphics/host_gpu/renderer/cache/bufferCache.h

@@ -49,6 +49,8 @@ public:
 
 	[[nodiscard]] bool InvalidateMemory(PageFaultAccess access, uint64_t vaddr, uint64_t size,
 	                                    PageFaultPhase phase) noexcept;
+	void               InvalidateMemory(uint64_t vaddr, uint64_t size);
+	void               ReadMemory(uint64_t vaddr, uint64_t size);
 	void               UnmapMemory(uint64_t vaddr, uint64_t size);
 	[[nodiscard]] BufferBinding ObtainBuffer(CommandBuffer& command, uint64_t vaddr, uint64_t size,
 	                                         bool is_written = false, bool is_read = true,
@@ -71,12 +73,11 @@ public:
 	[[nodiscard]] bool IsRegionCpuModified(uint64_t vaddr, uint64_t size);
 	[[nodiscard]] bool IsRegionGpuModified(uint64_t vaddr, uint64_t size);
 	void               InvalidateImageAliases(uint64_t vaddr, uint64_t size);
-	void BeginBackingPublication(uint64_t vaddr, uint64_t size, uint64_t tick);
-	void CompleteBackingPublication(uint64_t vaddr, uint64_t size, uint64_t tick);
+	void               BeginBackingPublication(uint64_t vaddr, uint64_t size, uint64_t tick);
+	void               CompleteBackingPublication(uint64_t vaddr, uint64_t size, uint64_t tick);
 	[[nodiscard]] bool SynchronizeBacking(uint64_t vaddr, uint64_t size);
 	void               PublishImageBuffer(uint64_t vaddr, uint64_t size);
-	void ValidateGpuAccess(uint64_t vaddr, uint64_t size, bool is_read, bool is_written) const;
-	void RunGarbageCollector();
+	void               RunGarbageCollector();
 
 private:
 	friend struct BufferCacheTestAccess;
@@ -91,23 +92,21 @@ private:
 	struct RetiredBuffer;
 	struct FaultReadback;
 	struct PendingBackingPublication;
-	static constexpr uint64_t DOWNLOAD_ALIGNMENT = 64;
-	[[nodiscard]] static uint64_t AlignDown(uint64_t value) noexcept;
-	[[nodiscard]] static uint64_t AlignUp(uint64_t value);
+	static constexpr uint64_t               DOWNLOAD_ALIGNMENT = 64;
+	[[nodiscard]] static uint64_t           AlignDown(uint64_t value) noexcept;
+	[[nodiscard]] static uint64_t           AlignUp(uint64_t value);
 	[[nodiscard]] static constexpr uint64_t AlignDownload(uint64_t size) noexcept {
 		return (size + DOWNLOAD_ALIGNMENT - 1) & ~(DOWNLOAD_ALIGNMENT - 1);
 	}
 	[[nodiscard]] static bool PageOverlaps(uint64_t left, uint64_t left_size, uint64_t right,
-	                                      uint64_t right_size) noexcept;
-	[[nodiscard]] static std::pair<uint64_t, uint64_t>
-	     DownloadEnvelope(const DownloadCopy& copy);
-	[[nodiscard]] static bool   ResolveOverlap(CacheRange& merged, CacheRange candidate) noexcept;
+	                                       uint64_t right_size) noexcept;
+	[[nodiscard]] static std::pair<uint64_t, uint64_t> DownloadEnvelope(const DownloadCopy& copy);
+	[[nodiscard]] static bool ResolveOverlap(CacheRange& merged, CacheRange candidate) noexcept;
 	void Upload(CommandBuffer& command, Buffer& destination, uint64_t destination_offset,
 	            const void* source, uint64_t size);
 	[[nodiscard]] CachedBuffer& GetOrCreateBuffer(CommandBuffer& command, uint64_t vaddr,
 	                                              uint64_t size);
-	[[nodiscard]] std::vector<DownloadRange>
-	     RecordDownloads(std::span<const DownloadCopy> copies);
+	[[nodiscard]] std::vector<DownloadRange> RecordDownloads(std::span<const DownloadCopy> copies);
 	void PublishDownloads(std::span<const DownloadRange> downloads);
 	void QueueGarbageDownload(std::span<const DownloadCopy> copies, RetiredBuffer retire);
 	void RefreshInvalidatedRanges(CommandBuffer& command, CachedBuffer& cached, uint64_t vaddr,

src/graphics/host_gpu/renderer/cache/gpuResourceManager.cpp

@@ -4,7 +4,6 @@
 #include "graphics/guest_gpu/command_processor/commandProcessor.h"
 #include "graphics/guest_gpu/graphicsRun.h"
 #include "graphics/host_gpu/renderer/commandScheduler.h"
-
 namespace Libs::Graphics {
 
 GpuResourceManager::GpuResourceManager(GraphicContext& graphics, CommandScheduler& scheduler)
@@ -36,7 +35,8 @@ bool GpuResourceManager::InvalidateMemory(PageFaultAccess access, uint64_t vaddr
 }
 
 bool GpuResourceManager::HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noexcept {
-	if (!m_page_manager.IsMapped(fault_vaddr, 1)) {
+	constexpr uint64_t fault_size = 8;
+	if (!IsMapped(fault_vaddr, fault_size)) {
 		return false;
 	}
 	if (CommandScheduler::InDeferredOperation()) {
@@ -47,10 +47,15 @@ bool GpuResourceManager::HandleFault(PageFaultAccess access, uint64_t fault_vadd
 	bool       handled = false;
 	const auto resolve = [this, access, fault_vaddr, &handled](CommandProcessor& cp) {
 		cp.BeginReadbackTransaction();
-		(void)m_buffer_cache.SynchronizeBacking(fault_vaddr, 1);
 		{
 			ResourceMutex::FaultScope fault(m_resource_mutex);
-			handled = m_page_manager.HandleFault(access, fault_vaddr);
+			if (access == PageFaultAccess::Write) {
+				m_buffer_cache.InvalidateMemory(fault_vaddr, fault_size);
+				m_texture_cache.InvalidateMemory(fault_vaddr, fault_size);
+			} else {
+				m_buffer_cache.ReadMemory(fault_vaddr, fault_size);
+			}
+			handled = true;
 		}
 		cp.EndReadbackTransaction();
 	};
@@ -68,58 +73,62 @@ bool GpuResourceManager::HandleFault(PageFaultAccess access, uint64_t fault_vadd
 	return handled;
 }
 
-void GpuResourceManager::PrepareHostWrite(uint64_t vaddr, uint64_t size) {
-	if (!m_page_manager.HasAnyMapping(vaddr, size)) {
-		return;
+bool GpuResourceManager::InvalidateMemory(uint64_t vaddr, uint64_t size) {
+	if (!IsMapped(vaddr, size)) {
+		return false;
 	}
 	if (CommandScheduler::InDeferredOperation()) {
-		EXIT("unsupported host write from an asynchronous GPU completion, addr=0x%016" PRIx64
-		     " size=0x%016" PRIx64 "\n",
+		EXIT("unsupported memory invalidation from an asynchronous GPU completion, "
+		     "addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n",
 		     vaddr, size);
 	}
-	const auto handle_range = [this, vaddr, size] {
-		if (!m_page_manager.HandleWriteRange(vaddr, size)) {
-			EXIT("failed to prepare host write, addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n",
-			     vaddr, size);
-		}
-	};
-	const auto resolve = [this, &handle_range](CommandProcessor& cp) {
+	const auto resolve = [this, vaddr, size](CommandProcessor& cp) {
 		cp.BeginReadbackTransaction();
 		{
 			ResourceMutex::FaultScope fault(m_resource_mutex);
-			handle_range();
+			m_buffer_cache.InvalidateMemory(vaddr, size);
+			m_texture_cache.InvalidateMemory(vaddr, size);
 		}
 		cp.EndReadbackTransaction();
 	};
 	if (auto* cp = Gpu::CurrentCommandProcessor(); cp != nullptr) {
 		resolve(*cp);
-		return;
+		return true;
 	}
 	if (m_resource_mutex.IsOwnedByCurrentThread()) {
-		EXIT("unsupported host write from a pre-owned resource transaction, addr=0x%016" PRIx64
-		     " size=0x%016" PRIx64 "\n",
+		EXIT("unsupported memory invalidation from a pre-owned resource transaction, "
+		     "addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n",
 		     vaddr, size);
 	}
 	EXIT_IF(m_gpu == nullptr);
 	m_gpu->SendCommandSyncWithProcessor(resolve);
+	return true;
 }
 
 bool GpuResourceManager::IsMapped(uint64_t vaddr, uint64_t size) const noexcept {
-	return m_page_manager.IsMapped(vaddr, size);
-}
-
-void GpuResourceManager::MapMemory(uint64_t vaddr, uint64_t size, GpuAccess access) {
-	m_page_manager.OnGpuMap(vaddr, size, access);
-}
-
-void GpuResourceManager::UnmapMemory(uint64_t vaddr, uint64_t size, GpuAccess access) {
-	if (!IsMapped(vaddr, size)) {
-		EXIT("cannot unmap an unmapped GPU resource range\n");
+	if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE ||
+	    size > TRACKER_ADDRESS_SIZE - vaddr) {
+		return false;
 	}
-	const auto unmap = [this, vaddr, size, access] {
-		m_texture_cache.UnmapMemory(vaddr, size);
+	std::shared_lock lock(m_mapped_ranges_mutex);
+	return m_mapped_ranges.Contains(vaddr, size);
+}
+
+void GpuResourceManager::MapMemory(uint64_t vaddr, uint64_t size) {
+	{
+		std::lock_guard lock(m_mapped_ranges_mutex);
+		m_mapped_ranges.Add(vaddr, size);
+	}
+	m_page_manager.OnGpuMap(vaddr, size);
+}
+
+void GpuResourceManager::UnmapMemory(uint64_t vaddr, uint64_t size) {
+	const auto unmap = [this, vaddr, size] {
 		m_buffer_cache.UnmapMemory(vaddr, size);
-		m_page_manager.OnGpuUnmap(vaddr, size, access);
+		m_texture_cache.UnmapMemory(vaddr, size);
+		m_page_manager.OnGpuUnmap(vaddr, size);
+		std::lock_guard lock(m_mapped_ranges_mutex);
+		m_mapped_ranges.Subtract(vaddr, size);
 	};
 	if (m_gpu == nullptr) {
 		if (m_resource_mutex.IsOwnedByCurrentThread()) {

src/graphics/host_gpu/renderer/cache/gpuResourceManager.h

@@ -9,6 +9,7 @@
 #include "graphics/host_gpu/renderer/cache/textureCache.h"
 
 #include <cstdint>
+#include <shared_mutex>
 
 namespace Libs::Graphics {
 
@@ -26,10 +27,10 @@ public:
 	void                        SetGpu(Gpu* gpu) noexcept { m_gpu = gpu; }
 
 	[[nodiscard]] bool HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noexcept;
-	void               PrepareHostWrite(uint64_t vaddr, uint64_t size);
+	[[nodiscard]] bool InvalidateMemory(uint64_t vaddr, uint64_t size);
 	[[nodiscard]] bool IsMapped(uint64_t vaddr, uint64_t size) const noexcept;
-	void               MapMemory(uint64_t vaddr, uint64_t size, GpuAccess access);
-	void               UnmapMemory(uint64_t vaddr, uint64_t size, GpuAccess access);
+	void               MapMemory(uint64_t vaddr, uint64_t size);
+	void               UnmapMemory(uint64_t vaddr, uint64_t size);
 	void               RunGarbageCollector();
 
 private:
@@ -38,11 +39,13 @@ private:
 	[[nodiscard]] bool InvalidateMemory(PageFaultAccess access, uint64_t vaddr, uint64_t size,
 	                                    PageFaultPhase phase) noexcept;
 
-	PageManager   m_page_manager;
-	ResourceMutex m_resource_mutex;
-	BufferCache   m_buffer_cache;
-	TextureCache  m_texture_cache;
-	Gpu*          m_gpu = nullptr;
+	PageManager               m_page_manager;
+	ResourceMutex             m_resource_mutex;
+	BufferCache               m_buffer_cache;
+	TextureCache              m_texture_cache;
+	mutable std::shared_mutex m_mapped_ranges_mutex;
+	RangeSet                  m_mapped_ranges;
+	Gpu*                      m_gpu = nullptr;
 };
 
 } // namespace Libs::Graphics

src/graphics/host_gpu/renderer/cache/textureCache.cpp

@@ -7,13 +7,13 @@
 #include "graphics/guest_gpu/gpu_format.h"
 #include "graphics/guest_gpu/tile.h"
 #include "graphics/host_gpu/graphicContext.h"
-#include "graphics/host_gpu/renderer/image/textureCommon.h"
 #include "graphics/host_gpu/renderer/cache/bufferCache.h"
+#include "graphics/host_gpu/renderer/cache/resourceMutex.h"
 #include "graphics/host_gpu/renderer/commandScheduler.h"
 #include "graphics/host_gpu/renderer/image/imageView.h"
-#include "graphics/host_gpu/renderer/render.h"
-#include "graphics/host_gpu/renderer/cache/resourceMutex.h"
+#include "graphics/host_gpu/renderer/image/textureCommon.h"
 #include "graphics/host_gpu/renderer/image/tiler.h"
+#include "graphics/host_gpu/renderer/render.h"
 #include "kernel/memory.h"
 
 #include <algorithm>
@@ -58,8 +58,8 @@ private:
 TextureCache::TextureCache(GraphicContext& graphics, CommandScheduler& scheduler,
                            PageManager& page_manager, BufferCache& buffer_cache,
                            ResourceMutex& resource_mutex)
-    : m_graphics(graphics), m_scheduler(scheduler),
-      m_memory_tracker(page_manager, PageWatchMode::Write), m_blit_helper(graphics, scheduler),
+    : m_graphics(graphics), m_scheduler(scheduler), m_page_manager(page_manager),
+      m_blit_helper(graphics, scheduler),
       m_tiler(std::make_unique<TileManager>(graphics, scheduler,
                                             buffer_cache.GetUtilityBuffer(MemoryUsage::Stream))),
       m_buffer_cache(buffer_cache), m_resource_mutex(resource_mutex),
@@ -80,7 +80,7 @@ TextureCache::TextureCache(GraphicContext& graphics, CommandScheduler& scheduler
 TextureCache::~TextureCache() {
 	for (uint32_t index = 0; index < m_slots.size(); index++) {
 		if (m_slots[index].image != nullptr && m_slots[index].image->registered) {
-			UnregisterImage({index, m_slots[index].generation}, false);
+			UnregisterImage({index, m_slots[index].generation});
 		}
 		m_slots[index].image.reset();
 	}
@@ -88,15 +88,34 @@ TextureCache::~TextureCache() {
 
 bool TextureCache::SameBacking(const ImageInfo& cached, const ImageInfo& requested,
                                bool exact_format) {
-	const bool unit_extent =
-	    requested.extent.width == 1 && requested.extent.height == 1 && requested.extent.depth == 1;
-	return cached.data == requested.data && cached.extent == requested.extent &&
-	       cached.samples == requested.samples &&
-	       cached.bytes_per_block == requested.bytes_per_block &&
-	       (cached.type == requested.type || unit_extent) &&
-	       (exact_format
-	            ? cached.pixel_format == requested.pixel_format
-	            : ImageViewOps::FormatsCompatible(cached.pixel_format, requested.pixel_format));
+	if (cached.data.address != requested.data.address) {
+		return false;
+	}
+	if (cached.data.size != requested.data.size) {
+		return false;
+	}
+	if (cached.extent != requested.extent) {
+		return false;
+	}
+	if (cached.samples != requested.samples) {
+		return false;
+	}
+	if (cached.bytes_per_block != requested.bytes_per_block) {
+		return false;
+	}
+	if (cached.tile_mode != requested.tile_mode) {
+		return false;
+	}
+	if (!ImageViewOps::FormatsCompatible(cached.pixel_format, requested.pixel_format)) {
+		return false;
+	}
+	if (cached.type != requested.type && requested.extent != vk::Extent3D {1, 1, 1}) {
+		return false;
+	}
+	if (exact_format && cached.pixel_format != requested.pixel_format) {
+		return false;
+	}
+	return true;
 }
 
 TextureCache::BindingType TextureCache::UploadBinding(const Image& image) {
@@ -117,8 +136,7 @@ bool TextureCache::SafeToDownload(const Image& image) {
 		return false;
 	}
 	const auto range = image.info.data;
-	return !m_buffer_cache.HasGpuDirtyBytes(range.address, range.size) &&
-	       !m_memory_tracker.IsRegionCpuModified(range.address, range.size);
+	return !m_buffer_cache.HasGpuDirtyBytes(range.address, range.size);
 }
 
 Image& TextureCache::ResolveImage(ImageId id) {
@@ -187,21 +205,17 @@ void TextureCache::RegisterImage(ImageId id) {
 	m_total_used_memory += image.AccountedSize();
 }
 
-void TextureCache::UnregisterImage(ImageId id, bool release_tracking) {
+void TextureCache::UnregisterImage(ImageId id) {
 	auto& image = ResolveImage(id);
 	if (!image.registered) {
 		return;
 	}
+	UntrackImage(id);
 	std::vector<ImageOwnerIndex::ByteRange> releases;
 	if (!m_image_owner_index.Unregister(id, releases)) {
 		EXIT("TextureCache: image missing from owner index\n");
 	}
 	m_lru_cache.Free(image.lru_id);
-	if (release_tracking) {
-		for (const auto& range: releases) {
-			m_memory_tracker.UntrackMemory(range.address, range.size);
-		}
-	}
 	const auto accounted = image.AccountedSize();
 	if (accounted > m_total_used_memory) {
 		EXIT("TextureCache: image accounting underflow\n");
@@ -210,7 +224,7 @@ void TextureCache::UnregisterImage(ImageId id, bool release_tracking) {
 	image.registered = false;
 }
 
-void TextureCache::DeleteImage(ImageId id, bool release_tracking) {
+void TextureCache::DeleteImage(ImageId id) {
 	auto owner = ResolveOwner(id);
 	if (owner == nullptr || !owner->registered) {
 		return;
@@ -224,7 +238,7 @@ void TextureCache::DeleteImage(ImageId id, bool release_tracking) {
 			}
 		}
 		for (const auto association: associations) {
-			ReleaseGpuTracking(association);
+			ClearGpuModified(association);
 			DeleteImage(association);
 		}
 	}
@@ -235,7 +249,7 @@ void TextureCache::DeleteImage(ImageId id, bool release_tracking) {
 	if (owner->info.metadata.kind == ImageMetadataKind::Htile) {
 		m_surface_metas.erase(owner->info.metadata.range.address);
 	}
-	UnregisterImage(id, release_tracking);
+	UnregisterImage(id);
 	const auto erase_slot = [this, id, retained = owner] {
 		auto& slot = m_slots[id.index];
 		if (slot.generation != id.generation || slot.image != retained) {
@@ -266,10 +280,10 @@ void TextureCache::DeleteImages(std::span<const ImageId> ids,
 			continue;
 		}
 		if (native_source == id) {
-			ReleaseGpuTracking(id);
+			ClearGpuModified(id);
 		} else if (owner->IsGpuModified()) {
 			DownloadImage(id);
-			ReleaseGpuTracking(id);
+			ClearGpuModified(id);
 		}
 		DeleteImage(id);
 	}
@@ -296,6 +310,121 @@ void TextureCache::TouchImage(Image& image) {
 	}
 }
 
+void TextureCache::TrackImage(ImageId id) {
+	auto& image = ResolveImage(id);
+	if (!image.registered) {
+		return;
+	}
+	const auto image_begin = image.info.data.address;
+	const auto image_end   = image.info.data.End();
+	if (image_begin == image.track_addr && image_end == image.track_addr_end) {
+		return;
+	}
+	if (!image.IsTracked()) {
+		image.track_addr     = image_begin;
+		image.track_addr_end = image_end;
+		m_page_manager.UpdatePageWatchers(true, image_begin, image.info.data.size);
+		return;
+	}
+	if (image_begin < image.track_addr) {
+		TrackImageHead(id);
+	}
+	if (image.track_addr_end < image_end) {
+		TrackImageTail(id);
+	}
+}
+
+void TextureCache::TrackImageHead(ImageId id) {
+	auto& image = ResolveImage(id);
+	if (!image.registered) {
+		return;
+	}
+	const auto image_begin = image.info.data.address;
+	if (image_begin == image.track_addr) {
+		return;
+	}
+	if (!image.IsTracked() || image_begin > image.track_addr) {
+		EXIT("TextureCache: invalid image head tracking range\n");
+	}
+	const auto size  = image.track_addr - image_begin;
+	image.track_addr = image_begin;
+	m_page_manager.UpdatePageWatchers(true, image_begin, size);
+}
+
+void TextureCache::TrackImageTail(ImageId id) {
+	auto& image = ResolveImage(id);
+	if (!image.registered) {
+		return;
+	}
+	const auto image_end = image.info.data.End();
+	if (image_end == image.track_addr_end) {
+		return;
+	}
+	if (!image.IsTracked() || image.track_addr_end > image_end) {
+		EXIT("TextureCache: invalid image tail tracking range\n");
+	}
+	const auto address   = image.track_addr_end;
+	const auto size      = image_end - address;
+	image.track_addr_end = image_end;
+	m_page_manager.UpdatePageWatchers(true, address, size);
+}
+
+void TextureCache::UntrackImage(ImageId id) {
+	auto& image = ResolveImage(id);
+	if (!image.IsTracked()) {
+		return;
+	}
+	const auto address   = image.track_addr;
+	const auto size      = image.track_addr_end - image.track_addr;
+	image.track_addr     = 0;
+	image.track_addr_end = 0;
+	if (size != 0) {
+		m_page_manager.UpdatePageWatchers(false, address, size);
+	}
+}
+
+void TextureCache::UntrackImageHead(ImageId id) {
+	auto&      image = ResolveImage(id);
+	const auto begin = image.info.data.address;
+	if (!image.IsTracked() || begin < image.track_addr) {
+		return;
+	}
+	const auto address = (begin + TRACKER_PAGE_SIZE) & ~(TRACKER_PAGE_SIZE - 1);
+	const auto size    = address - begin;
+	image.track_addr   = address;
+	if (image.track_addr == image.track_addr_end) {
+		image.MarkMaybeCpuDirty();
+		if (image.NeedsMaybeCpuHash()) {
+			image.SetMaybeCpuHash(image.HashGuestEdges());
+		}
+		UntrackImage(id);
+	}
+	if (size != 0) {
+		m_page_manager.UpdatePageWatchers(false, begin, size);
+	}
+}
+
+void TextureCache::UntrackImageTail(ImageId id) {
+	auto&      image = ResolveImage(id);
+	const auto end   = image.info.data.End();
+	if (!image.IsTracked() || image.track_addr_end < end) {
+		return;
+	}
+	const auto address   = end & ~(TRACKER_PAGE_SIZE - 1);
+	const auto size      = end - address;
+	image.track_addr_end = address;
+	if (image.track_addr == image.track_addr_end) {
+		image.MarkMaybeCpuDirty();
+		if (image.NeedsMaybeCpuHash()) {
+			image.SetMaybeCpuHash(image.HashGuestEdges());
+		}
+		UntrackImage(id);
+	}
+	if (size != 0) {
+		m_page_manager.UpdatePageWatchers(false, address, size);
+	}
+}
+
 void TextureCache::TrackImageDownload(ImageId id) {
 	std::lock_guard transaction(m_resource_mutex);
 	CacheLock       lock(*this, m_lock);
@@ -376,9 +505,6 @@ void TextureCache::ValidateImageDesc(const ImageDesc& desc) const {
 }
 
 void TextureCache::PrepareImageCopy(Image& image) {
-	const auto range = image.info.data;
-	m_memory_tracker.ForEachUploadRange(
-	    range.address, range.size, false, [](uint64_t, uint64_t) noexcept {}, []() noexcept {});
 	if (image.IsCpuDirty()) {
 		image.RefreshComplete();
 	}
@@ -471,6 +597,7 @@ void TextureCache::CopyImage(ImageId destination_id, ImageId source_id) {
 	RefreshCopySource(source_id);
 	auto& destination = ResolveImage(destination_id);
 	auto& source      = ResolveImage(source_id);
+	TrackImage(destination_id);
 	if (source.backing.samples != destination.backing.samples) {
 		EXIT("TextureCache: cannot issue an unequal-sample image copy\n");
 	}
@@ -479,7 +606,6 @@ void TextureCache::CopyImage(ImageId destination_id, ImageId source_id) {
 		if (source.info.data == destination.info.data) {
 			destination.MarkBufferModified();
 		}
-		RestoreGpuTracking(destination);
 		return;
 	}
 	const bool source_depth = source.info.IsDepth();
@@ -503,7 +629,6 @@ void TextureCache::CopyImage(ImageId destination_id, ImageId source_id) {
 		destination.MarkGpuModified();
 	}
 	destination.ClearBufferModified();
-	RestoreGpuTracking(destination);
 }
 
 void TextureCache::CopyImageMip(ImageId destination_id, ImageId source_id, uint32_t mip,
@@ -511,6 +636,7 @@ void TextureCache::CopyImageMip(ImageId destination_id, ImageId source_id, uint3
 	RefreshCopySource(source_id);
 	auto& destination = ResolveImage(destination_id);
 	auto& source      = ResolveImage(source_id);
+	TrackImage(destination_id);
 	if (source.IsBufferModified() || source.backing.samples != destination.backing.samples) {
 		EXIT("TextureCache: invalid mip-copy ownership or sample count\n");
 	}
@@ -520,7 +646,6 @@ void TextureCache::CopyImageMip(ImageId destination_id, ImageId source_id, uint3
 	if (source.IsGpuModified()) {
 		destination.MarkGpuModified();
 	}
-	RestoreGpuTracking(destination);
 }
 
 ImageId TextureCache::ResolveDepthOverlap(const ImageInfo& requested, BindingType binding,
@@ -590,7 +715,7 @@ ImageId TextureCache::ResolveDepthOverlap(const ImageInfo& requested, BindingTyp
 	if (copied) {
 		DeleteImages(std::array {cached_id}, cached_id);
 	} else {
-		ReleaseGpuTracking(cached_id);
+		ClearGpuModified(cached_id);
 		DeleteImage(cached_id);
 	}
 	return replacement_id;
@@ -629,6 +754,12 @@ TextureCache::OverlapResult TextureCache::ResolveOverlap(const ImageInfo& reques
 		    (requested.IsVolume() || cached.info.IsVolume())) {
 			return {ExpandImage(requested, cached_id)};
 		}
+		if (requested.tile_mode != cached.info.tile_mode) {
+			if (safe_to_delete) {
+				DeleteImages(std::array {cached_id}, cached_id);
+			}
+			return {merged_id};
+		}
 		if (requested.pixel_format != cached.info.pixel_format ||
 		    requested.data.size <= cached.info.data.size) {
 			const auto result_id = merged_id ? merged_id : cached_id;
@@ -641,12 +772,6 @@ TextureCache::OverlapResult TextureCache::ResolveOverlap(const ImageInfo& reques
 		if (requested.type == cached.info.type && requested.resources > cached.info.resources) {
 			return {ExpandImage(requested, cached_id)};
 		}
-		if (requested.tile_mode != cached.info.tile_mode) {
-			if (safe_to_delete) {
-				DeleteImages(std::array {cached_id}, cached_id);
-			}
-			return {merged_id};
-		}
 		EXIT("TextureCache: unresolvable equal-address image overlap, address=0x%016" PRIx64
 		     " requested=%ux%u "
 		     "cached=%ux%u requested_size=0x%016" PRIx64 " cached_size=0x%016" PRIx64
@@ -903,39 +1028,29 @@ void TextureCache::InitializeImage(ImageId id, const ImageDesc& desc) {
 	if (image.info.data.Empty()) {
 		return;
 	}
+	TrackImage(id);
 	if (image.info.metadata.compression != VideoOutCompression::Uncompressed) {
-		m_memory_tracker.ForEachUploadRange(
-		    image.info.data.address, image.info.data.size, false,
-		    [](uint64_t, uint64_t) noexcept {}, []() noexcept {});
 		if (image.IsCpuDirty()) {
 			image.RefreshComplete();
 		}
 		return;
 	}
 	if (image.info.samples > 1) {
-		RestoreGpuTracking(image);
 		return;
 	}
-	bool data_gpu_owned = false;
-	bool data_imported  = false;
-	bool uploaded       = false;
-	m_memory_tracker.ForEachUploadRange(
-	    image.info.data.address, image.info.data.size, false,
-	    [&](uint64_t, uint64_t) noexcept { uploaded = true; },
-	    [&]() noexcept {
-		    uploaded |= image.IsBufferModified() || image.IsDefinitelyCpuDirty();
-		    if (!uploaded) {
-			    return;
-		    }
-		    const auto source =
-		        m_buffer_cache.ObtainBufferForImage(image.info.data.address, image.info.data.size);
-		    if (source.buffer == nullptr) {
-			    EXIT("TextureCache: failed to obtain image upload source\n");
-		    }
-		    data_gpu_owned |= source.gpu_owned;
-		    data_imported = true;
-		    UploadImage(image, desc, *source.buffer, source.offset);
-	    });
+	bool       data_gpu_owned = false;
+	bool       data_imported  = false;
+	const bool upload         = image.IsBufferModified() || image.IsCpuDirty();
+	if (upload) {
+		const auto source =
+		    m_buffer_cache.ObtainBufferForImage(image.info.data.address, image.info.data.size);
+		if (source.buffer == nullptr) {
+			EXIT("TextureCache: failed to obtain image upload source\n");
+		}
+		data_gpu_owned |= source.gpu_owned;
+		data_imported = true;
+		UploadImage(image, desc, *source.buffer, source.offset);
+	}
 	if (data_imported) {
 		image.ClearBufferModified();
 	}
@@ -945,39 +1060,27 @@ void TextureCache::InitializeImage(ImageId id, const ImageDesc& desc) {
 	if (image.IsCpuDirty()) {
 		image.RefreshComplete();
 	}
-	RestoreGpuTracking(image);
 }
 
 void TextureCache::RefreshImage(ImageId id, const ImageDesc& desc) {
-	auto& image           = ResolveImage(id);
-	bool  unchanged_maybe = false;
+	TrackImage(id);
+	auto& image = ResolveImage(id);
 	if (image.IsMaybeCpuDirty()) {
 		const auto hash = image.HashGuestEdges();
 		if (image.NeedsMaybeCpuHash()) {
 			image.SetMaybeCpuHash(hash);
 			return;
 		}
-		unchanged_maybe = !image.ResolveMaybeCpuHash(hash);
-		if (unchanged_maybe) {
-			m_memory_tracker.ForEachUploadRange(
-			    image.info.data.address, image.info.data.size, false,
-			    [](uint64_t, uint64_t) noexcept {}, []() noexcept {});
-		}
+		(void)image.ResolveMaybeCpuHash(hash);
 	}
 	bool cpu_dirty = image.IsBufferModified() || image.IsDefinitelyCpuDirty();
-	if (!unchanged_maybe) {
-		cpu_dirty |=
-		    m_memory_tracker.IsRegionCpuModified(image.info.data.address, image.info.data.size);
-	}
 	if (image.info.metadata.compression != VideoOutCompression::Uncompressed) {
 		if (cpu_dirty) {
 			EXIT("TextureCache: compressed guest image refresh is unsupported\n");
 		}
-		RestoreGpuTracking(image);
 		return;
 	}
 	if (!cpu_dirty) {
-		RestoreGpuTracking(image);
 		return;
 	}
 	InitializeImage(id, desc);
@@ -1029,7 +1132,7 @@ ImageId TextureCache::FindImage(ImageDesc& desc, bool exact_format) {
 
 	ImageId result {};
 	bool    replacement_buffer = false;
-	bool    replacing_image    = false;
+	bool    inserted_new       = false;
 	{
 		std::lock_guard            transaction(m_resource_mutex);
 		CacheLock                  lock(*this, m_lock);
@@ -1038,7 +1141,7 @@ ImageId TextureCache::FindImage(ImageDesc& desc, bool exact_format) {
 
 		for (const auto id: candidates) {
 			const auto owner = ResolveOwner(id);
-			if (owner == nullptr || owner->info.data != desc.info.data) {
+			if (owner == nullptr) {
 				continue;
 			}
 			if (SameBacking(owner->info, desc.info, exact_format)) {
@@ -1079,20 +1182,19 @@ ImageId TextureCache::FindImage(ImageDesc& desc, bool exact_format) {
 				}
 				replacement_buffer = resolved.IsBufferModified();
 				DeleteImage(result);
-				result          = {};
-				replacing_image = true;
+				result = {};
 			}
 		}
 		if (!result) {
 			result         = InsertImage(desc.info);
+			inserted_new   = true;
 			auto& inserted = ResolveImage(result);
 			if (replacement_buffer || m_buffer_cache.HasGpuDirtyBytes(inserted.info.data.address,
 			                                                          inserted.info.data.size)) {
 				inserted.MarkBufferModified();
-			} else if (replacing_image) {
-				m_memory_tracker.MarkRegionAsCpuModified(inserted.info.data.address,
-				                                         inserted.info.data.size);
 			}
+		}
+		if (inserted_new) {
 			InitializeImage(result, desc);
 		} else {
 			RefreshImage(result, desc);
@@ -1101,9 +1203,7 @@ ImageId TextureCache::FindImage(ImageDesc& desc, bool exact_format) {
 		auto& image = ResolveImage(result);
 		if (desc.type == BindingType::VideoOut &&
 		    desc.info.metadata.compression != VideoOutCompression::Uncompressed) {
-			const bool guest_dirty =
-			    image.IsBufferModified() || image.IsCpuDirty() ||
-			    m_memory_tracker.IsRegionCpuModified(image.info.data.address, image.info.data.size);
+			const bool guest_dirty = image.IsBufferModified() || image.IsCpuDirty();
 			const bool native_current =
 			    (image.usage.render_target || image.IsGpuModified()) && !guest_dirty;
 			if (!native_current) {
@@ -1252,6 +1352,7 @@ void TextureCache::MarkGpuWritten(ImageId id) {
 	if (!image.registered || image.depth_id) {
 		EXIT("TextureCache: cannot mark an unavailable image GPU-written\n");
 	}
+	TrackImage(id);
 	CommitGpuWrite(image);
 }
 
@@ -1265,13 +1366,10 @@ void TextureCache::CommitGpuWrite(Image& image) {
 	}
 	m_buffer_cache.InvalidateImageAliases(range.address, range.size);
 	image.ClearBufferModified();
-	m_memory_tracker.ForEachUploadRange(
-	    range.address, range.size, true, [](uint64_t, uint64_t) noexcept {}, []() noexcept {});
 	if (image.IsCpuDirty()) {
 		image.RefreshComplete();
 	}
 	image.MarkGpuModified();
-	RestoreGpuTracking(image);
 }
 
 bool TextureCache::ClearImageFromBuffer(CommandBuffer& command, uint64_t address, uint64_t size,
@@ -1279,7 +1377,6 @@ bool TextureCache::ClearImageFromBuffer(CommandBuffer& command, uint64_t address
 	if (command.IsInvalid() || !GuestRange {address, size}.Valid()) {
 		EXIT("TextureCache: invalid image clear\n");
 	}
-	m_buffer_cache.ValidateGpuAccess(address, size, false, true);
 	std::lock_guard      transaction(m_resource_mutex);
 	CacheLock            lock(*this, m_lock);
 	ImageId              selected {};
@@ -1335,8 +1432,7 @@ bool TextureCache::ClearImageFromBuffer(CommandBuffer& command, uint64_t address
 	if (m_buffer_cache.HasGpuDirtyBytes(address, size)) {
 		m_buffer_cache.DiscardGpuDirtyBytes(address, size);
 	}
-	if (image.IsBufferModified() || image.IsCpuDirty() ||
-	    m_memory_tracker.IsRegionCpuModified(image.info.data.address, image.info.data.size)) {
+	if (image.IsBufferModified() || image.IsCpuDirty()) {
 		ImageDesc refresh {.info = image.info, .view_info = {}, .type = UploadBinding(image)};
 		InitializeImage(selected, refresh);
 		if (image.info.samples == 1 && (image.IsBufferModified() || image.IsCpuDirty())) {
@@ -1361,14 +1457,12 @@ bool TextureCache::ClearImageFromBuffer(CommandBuffer& command, uint64_t address
 	return true;
 }
 
-void TextureCache::PrepareHostWrite(uint64_t address, uint64_t size) {
+void TextureCache::InvalidateMemory(uint64_t address, uint64_t size) {
 	if (!GuestRange {address, size}.Valid()) {
-		EXIT("TextureCache: invalid host-write range\n");
+		EXIT("TextureCache: invalid memory-invalidation range\n");
 	}
 	CacheLock lock(*this, m_lock);
 	InvalidateCpuAliases(address, size);
-	m_memory_tracker.ForEachDownloadRange<true>(address, size, [](uint64_t, uint64_t) noexcept {});
-	m_memory_tracker.MarkRegionAsCpuModified(address, size);
 }
 
 void TextureCache::DownloadDepth(Image& image, Buffer& destination, uint64_t destination_offset) {
@@ -1567,18 +1661,15 @@ bool TextureCache::SynchronizeImageToBuffer(ImageId id) {
 	if (!plan.valid) {
 		return false;
 	}
-	const auto range   = image.info.data;
-	const bool refresh = image.IsDefinitelyCpuDirty() ||
-	                     m_memory_tracker.IsRegionCpuModified(range.address, range.size);
-	if (refresh) {
+	const auto range = image.info.data;
+	if (image.IsCpuDirty()) {
 		RefreshImage(id,
 		             ImageDesc {.info = image.info, .view_info = {}, .type = UploadBinding(image)});
 	}
 	if (!image.IsGpuModified()) {
 		return true;
 	}
-	if (image.IsDefinitelyCpuDirty() || image.IsBufferModified() ||
-	    m_memory_tracker.IsRegionCpuModified(range.address, range.size)) {
+	if (image.IsDefinitelyCpuDirty() || image.IsBufferModified()) {
 		EXIT("TextureCache: image mirror source is not native-current\n");
 	}
 	auto [destination, offset] =
@@ -1591,7 +1682,7 @@ bool TextureCache::SynchronizeImageToBuffer(ImageId id) {
 	m_buffer_cache.PublishImageBuffer(range.address, range.size);
 	image.MarkBufferModified();
 	RetainImage(m_scheduler.Current(), id);
-	ReleaseGpuTracking(id);
+	ClearGpuModified(id);
 	return true;
 }
 
@@ -1633,7 +1724,7 @@ bool TextureCache::InvalidateMemoryFromGPU(uint64_t address, uint64_t size,
 			if (!formatted_buffer_write) {
 				EXIT("TextureCache: buffer write aliases GPU-modified image\n");
 			}
-			ReleaseGpuTracking(id);
+			ClearGpuModified(id);
 		}
 		owner->MarkBufferModified();
 		found = true;
@@ -1660,50 +1751,40 @@ TextureCache::RegionInfo TextureCache::QueryRegion(uint64_t address, uint64_t si
 }
 
 void TextureCache::InvalidateCpuAliases(uint64_t address, uint64_t size) {
+	const auto page_begin = address & ~(TRACKER_PAGE_SIZE - 1);
+	const auto page_end   = (address + size + TRACKER_PAGE_SIZE - 1) & ~(TRACKER_PAGE_SIZE - 1);
 	for (const auto id: FindImagesInRegion(address, size, true)) {
 		auto owner = ResolveOwner(id);
 		if (owner == nullptr || owner->depth_id) {
 			continue;
 		}
-		owner->InvalidateCpuWrite(address, size);
-		if (owner->NeedsMaybeCpuHash()) {
-			owner->SetMaybeCpuHash(owner->HashGuestEdges());
+		if (owner->Overlaps(address, size)) {
+			owner->InvalidateCpuWrite(address, size);
+			UntrackImage(id);
+			continue;
+		}
+		const auto image_begin = owner->info.data.address;
+		const auto image_end   = owner->info.data.End();
+		if (page_end < image_end) {
+			UntrackImageHead(id);
+		} else if (image_begin < page_begin) {
+			UntrackImageTail(id);
+		} else {
+			owner->MarkMaybeCpuDirty();
+			if (owner->NeedsMaybeCpuHash()) {
+				owner->SetMaybeCpuHash(owner->HashGuestEdges());
+			}
+			UntrackImage(id);
 		}
 	}
 }
 
-void TextureCache::RestoreGpuTracking(const Image& image) {
-	if (!image.IsGpuModified()) {
-		return;
-	}
-	constexpr uint64_t page_mask = TRACKER_PAGE_SIZE - 1;
-	const auto         range     = image.info.data;
-	const auto         begin     = range.address & ~page_mask;
-	const auto         end       = (range.End() + page_mask) & ~page_mask;
-	for (auto page = begin; page < end; page += TRACKER_PAGE_SIZE) {
-		if (!m_memory_tracker.IsRegionGpuModified(page, TRACKER_PAGE_SIZE) &&
-		    !m_memory_tracker.IsRegionCpuModified(page, TRACKER_PAGE_SIZE)) {
-			m_memory_tracker.MarkRegionAsGpuModified(page, TRACKER_PAGE_SIZE);
-		}
-	}
-}
-
-void TextureCache::ReleaseGpuTracking(ImageId id) {
+void TextureCache::ClearGpuModified(ImageId id) {
 	auto owner = ResolveOwner(id);
 	if (owner == nullptr || !owner->IsGpuModified()) {
 		return;
 	}
-	const auto released = owner->info.data;
 	owner->ClearGpuModified();
-	m_memory_tracker.ForEachDownloadRange<true>(released.address, released.size,
-	                                            [](uint64_t, uint64_t) noexcept {});
-	for (const auto candidate: FindImagesInRegion(released.address, released.size, true)) {
-		const auto survivor = ResolveOwner(candidate);
-		if (survivor != nullptr && survivor.get() != owner.get() && !survivor->depth_id) {
-			RestoreGpuTracking(*survivor);
-		}
-	}
-	RestoreGpuTracking(*owner);
 }
 
 bool TextureCache::IsMeta(uint64_t address) {
@@ -1755,27 +1836,25 @@ bool TextureCache::InvalidateMemory(PageFaultAccess access, uint64_t address, ui
 		return false;
 	}
 	if (phase == PageFaultPhase::Invalidate) {
-		const bool gpu_image =
-		    m_memory_tracker.InvalidateVirtualGpuWrite(access, address, size, phase);
-		CpuFaultAction action = gpu_image ? CpuFaultAction::Download
-		                                  : m_memory_tracker.BeginCpuFault(address, size, access);
-		{
-			CacheLock lock(*this, m_lock);
+		CacheLock  lock(*this, m_lock);
+		const bool tracked =
+		    std::ranges::any_of(FindImagesInRegion(address, size, true), [&](ImageId id) {
+			    const auto owner = ResolveOwner(id);
+			    return owner != nullptr && !owner->depth_id && owner->IsTracked();
+		    });
+		if (tracked) {
 			InvalidateCpuAliases(address, size);
 		}
-		return action != CpuFaultAction::Untracked;
+		return tracked;
 	}
-
-	if (phase == PageFaultPhase::Complete) {
-		const bool gpu_image = m_memory_tracker.IsRegionGpuModified(address, size);
-		return gpu_image ? m_memory_tracker.InvalidateVirtualGpuWrite(access, address, size, phase)
-		                 : m_memory_tracker.CompleteCpuFault(address, size, access, false);
-	}
-	if (phase != PageFaultPhase::Release) {
+	if (phase != PageFaultPhase::Complete && phase != PageFaultPhase::Release) {
 		return false;
 	}
-	(void)m_memory_tracker.InvalidateVirtualGpuWrite(access, address, size, phase);
-	return true;
+	CacheLock lock(*this, m_lock);
+	return std::ranges::any_of(FindImagesInRegion(address, size, true), [&](ImageId id) {
+		const auto owner = ResolveOwner(id);
+		return owner != nullptr && !owner->depth_id;
+	});
 }
 
 void TextureCache::UnmapMemory(uint64_t address, uint64_t size) {
@@ -1794,16 +1873,10 @@ void TextureCache::UnmapMemory(uint64_t address, uint64_t size) {
 			continue;
 		}
 		if (owner->IsGpuModified()) {
-			ReleaseGpuTracking(id);
+			ClearGpuModified(id);
 		}
 		DeleteImage(id);
 	}
-	m_memory_tracker.UntrackMemory(address, size);
-	for (const auto id: FindImagesInRegion(address, size, true)) {
-		if (const auto survivor = ResolveOwner(id); survivor != nullptr) {
-			RestoreGpuTracking(*survivor);
-		}
-	}
 }
 
 void TextureCache::RunGarbageCollector() {
@@ -1849,7 +1922,7 @@ void TextureCache::RunGarbageCollector() {
 				if (safe && !TryDownloadImage(id)) {
 					continue;
 				}
-				ReleaseGpuTracking(id);
+				ClearGpuModified(id);
 			}
 			DeleteImage(id);
 			if (m_total_used_memory < m_critical_gc_memory && aggressive) {

src/graphics/host_gpu/renderer/cache/textureCache.h

@@ -4,10 +4,11 @@
 #include "common/abi.h"
 #include "common/common.h"
 #include "common/lruCache.h"
-#include "graphics/host_gpu/memoryTracker.h"
+#include "graphics/host_gpu/pageManager.h"
+#include "graphics/host_gpu/regionManager.h"
+#include "graphics/host_gpu/renderer/cache/multiLevelPageTable.h"
 #include "graphics/host_gpu/renderer/image/blitHelper.h"
 #include "graphics/host_gpu/renderer/image/image.h"
-#include "graphics/host_gpu/renderer/cache/multiLevelPageTable.h"
 
 #include <compare>
 #include <map>
@@ -64,7 +65,7 @@ public:
 
 	[[nodiscard]] bool ClearImageFromBuffer(CommandBuffer& command, uint64_t address, uint64_t size,
 	                                        uint32_t packed_clear);
-	void               PrepareHostWrite(uint64_t address, uint64_t size);
+	void               InvalidateMemory(uint64_t address, uint64_t size);
 	[[nodiscard]] bool SynchronizeImageToBuffer(uint64_t address, uint64_t size);
 	[[nodiscard]] bool InvalidateMemoryFromGPU(uint64_t address, uint64_t size,
 	                                           bool formatted_buffer_write = false);
@@ -109,11 +110,17 @@ private:
 	[[nodiscard]] ImageId                InsertImage(const ImageInfo& info);
 	[[nodiscard]] ImageId                GetNullImage(const ImageDesc& desc);
 	void                                 RegisterImage(ImageId id);
-	void                                 UnregisterImage(ImageId id, bool release_tracking);
-	void                                 DeleteImage(ImageId id, bool release_tracking = true);
+	void                                 UnregisterImage(ImageId id);
+	void                                 DeleteImage(ImageId id);
 	void DeleteImages(std::span<const ImageId> ids, std::optional<ImageId> native_source = {});
 	void RetainImage(CommandBuffer& command, ImageId id);
 	void TouchImage(Image& image);
+	void TrackImage(ImageId id);
+	void TrackImageHead(ImageId id);
+	void TrackImageTail(ImageId id);
+	void UntrackImage(ImageId id);
+	void UntrackImageHead(ImageId id);
+	void UntrackImageTail(ImageId id);
 	void TrackImageDownload(ImageId id);
 	void TrackImageDownloadLocked(ImageId id, Image& image);
 	[[nodiscard]] static bool SameBacking(const ImageInfo& cached, const ImageInfo& requested,
@@ -148,8 +155,7 @@ private:
 	void ValidateImageDesc(const ImageDesc& desc) const;
 
 	void InvalidateCpuAliases(uint64_t address, uint64_t size);
-	void RestoreGpuTracking(const Image& image);
-	void ReleaseGpuTracking(ImageId id);
+	void ClearGpuModified(ImageId id);
 
 	[[nodiscard]] bool                          SynchronizeImageToBuffer(ImageId id);
 	void                                        DownloadImage(ImageId id);
@@ -160,7 +166,7 @@ private:
 	GraphicContext&                                   m_graphics;
 	CommandScheduler&                                 m_scheduler;
 	TrackingSpinLock                                  m_lock;
-	MemoryTracker                                     m_memory_tracker;
+	PageManager&                                      m_page_manager;
 	BlitHelper                                        m_blit_helper;
 	std::unique_ptr<TileManager>                      m_tiler;
 	BufferCache&                                      m_buffer_cache;

src/graphics/host_gpu/renderer/image/image.cpp

@@ -2,10 +2,10 @@
 
 #include "common/assert.h"
 #include "common/profiler.h"
+#include "graphics/host_gpu/renderer/cache/streamBuffer.h"
 #include "graphics/host_gpu/renderer/commandScheduler.h"
 #include "graphics/host_gpu/renderer/image/imageView.h"
 #include "graphics/host_gpu/renderer/renderTarget.h"
-#include "graphics/host_gpu/renderer/cache/streamBuffer.h"
 #include "kernel/memory.h"
 
 #include <algorithm>
@@ -99,9 +99,9 @@ vk::ImageAspectFlags Image::FullAspectMask(vk::Format format) noexcept {
 	}
 }
 
-Image::Barriers Image::GetBarriers(vk::ImageLayout destination_layout,
-                                   vk::AccessFlags2 destination_access,
-                                   vk::PipelineStageFlags2 destination_stage,
+Image::Barriers Image::GetBarriers(vk::ImageLayout                      destination_layout,
+                                   vk::AccessFlags2                     destination_access,
+                                   vk::PipelineStageFlags2              destination_stage,
                                    std::optional<ImageSubresourceRange> range) {
 	auto& state              = backing.state;
 	auto& subresource_states = backing.subresource_states;
@@ -130,25 +130,25 @@ Image::Barriers Image::GetBarriers(vk::ImageLayout destination_layout,
 				constexpr auto write_access = vk::AccessFlagBits2::eTransferWrite |
 				                              vk::AccessFlagBits2::eShaderWrite |
 				                              vk::AccessFlagBits2::eMemoryWrite;
-				const bool repeated_write =
+				const bool     repeated_write =
 				    static_cast<bool>(subresource_state.access_mask & write_access);
 				if (subresource_state.layout != destination_layout ||
 				    subresource_state.access_mask != destination_access || repeated_write) {
 					vk::ImageMemoryBarrier2 barrier {};
-					barrier.srcStageMask                          = subresource_state.pl_stage;
-					barrier.srcAccessMask                         = subresource_state.access_mask;
-					barrier.dstStageMask                          = destination_stage;
-					barrier.dstAccessMask                         = destination_access;
-					barrier.oldLayout                             = subresource_state.layout;
-					barrier.newLayout                             = destination_layout;
-					barrier.srcQueueFamilyIndex                   = VK_QUEUE_FAMILY_IGNORED;
-					barrier.dstQueueFamilyIndex                   = VK_QUEUE_FAMILY_IGNORED;
-					barrier.image                                 = backing.image;
-					barrier.subresourceRange.aspectMask           = FullAspectMask(backing.format);
-					barrier.subresourceRange.baseMipLevel         = level;
-					barrier.subresourceRange.levelCount           = 1;
-					barrier.subresourceRange.baseArrayLayer       = layer;
-					barrier.subresourceRange.layerCount           = 1;
+					barrier.srcStageMask                    = subresource_state.pl_stage;
+					barrier.srcAccessMask                   = subresource_state.access_mask;
+					barrier.dstStageMask                    = destination_stage;
+					barrier.dstAccessMask                   = destination_access;
+					barrier.oldLayout                       = subresource_state.layout;
+					barrier.newLayout                       = destination_layout;
+					barrier.srcQueueFamilyIndex             = VK_QUEUE_FAMILY_IGNORED;
+					barrier.dstQueueFamilyIndex             = VK_QUEUE_FAMILY_IGNORED;
+					barrier.image                           = backing.image;
+					barrier.subresourceRange.aspectMask     = FullAspectMask(backing.format);
+					barrier.subresourceRange.baseMipLevel   = level;
+					barrier.subresourceRange.levelCount     = 1;
+					barrier.subresourceRange.baseArrayLayer = layer;
+					barrier.subresourceRange.layerCount     = 1;
 					barriers.push_back(barrier);
 					subresource_state = {destination_stage, destination_access, destination_layout};
 				}
@@ -159,10 +159,10 @@ Image::Barriers Image::GetBarriers(vk::ImageLayout destination_layout,
 			subresource_states.clear();
 		}
 	} else {
-		constexpr auto write_access = vk::AccessFlagBits2::eTransferWrite |
-		                              vk::AccessFlagBits2::eShaderWrite |
-		                              vk::AccessFlagBits2::eMemoryWrite;
-		const bool repeated_write = static_cast<bool>(state.access_mask & write_access);
+		constexpr auto write_access   = vk::AccessFlagBits2::eTransferWrite |
+		                                vk::AccessFlagBits2::eShaderWrite |
+		                                vk::AccessFlagBits2::eMemoryWrite;
+		const bool     repeated_write = static_cast<bool>(state.access_mask & write_access);
 		if (state.layout == destination_layout && state.access_mask == destination_access &&
 		    !repeated_write) {
 			return {};
@@ -191,8 +191,7 @@ Image::Barriers Image::GetBarriers(vk::ImageLayout destination_layout,
 }
 
 void Image::Transit(vk::ImageLayout destination_layout, vk::AccessFlags2 destination_access,
-                    std::optional<ImageSubresourceRange> range,
-                    vk::CommandBuffer command_buffer) {
+                    std::optional<ImageSubresourceRange> range, vk::CommandBuffer command_buffer) {
 	const auto transfer_access =
 	    vk::AccessFlagBits2::eTransferRead | vk::AccessFlagBits2::eTransferWrite;
 	vk::PipelineStageFlags2 destination_stage {};
@@ -201,8 +200,8 @@ void Image::Transit(vk::ImageLayout destination_layout, vk::AccessFlags2 destina
 	}
 	if (!destination_access ||
 	    static_cast<bool>(destination_access & ~vk::AccessFlags2 {transfer_access})) {
-		destination_stage |= vk::PipelineStageFlagBits2::eAllGraphics |
-		                     vk::PipelineStageFlagBits2::eComputeShader;
+		destination_stage |=
+		    vk::PipelineStageFlagBits2::eAllGraphics | vk::PipelineStageFlagBits2::eComputeShader;
 	}
 	const auto barriers =
 	    GetBarriers(destination_layout, destination_access, destination_stage, range);
@@ -218,10 +217,9 @@ void Image::Transit(vk::ImageLayout destination_layout, vk::AccessFlags2 destina
 	command_buffer.pipelineBarrier2(dependency);
 }
 
-void Image::Upload(std::span<const vk::BufferImageCopy> copies, vk::Buffer buffer,
-                   uint64_t offset, uint64_t size) {
-	EXIT_IF(m_scheduler == nullptr || copies.empty() || buffer == nullptr ||
-	        size == 0);
+void Image::Upload(std::span<const vk::BufferImageCopy> copies, vk::Buffer buffer, uint64_t offset,
+                   uint64_t size) {
+	EXIT_IF(m_scheduler == nullptr || copies.empty() || buffer == nullptr || size == 0);
 	m_scheduler->EndRendering();
 	vk::BufferMemoryBarrier2 buffer_barrier {};
 	buffer_barrier.srcStageMask        = vk::PipelineStageFlagBits2::eAllCommands;
@@ -234,16 +232,15 @@ void Image::Upload(std::span<const vk::BufferImageCopy> copies, vk::Buffer buffe
 	buffer_barrier.offset              = offset;
 	buffer_barrier.size                = size;
 	const auto image_barriers =
-	    GetBarriers(vk::ImageLayout::eTransferDstOptimal,
-	                vk::AccessFlagBits2::eTransferWrite,
+	    GetBarriers(vk::ImageLayout::eTransferDstOptimal, vk::AccessFlagBits2::eTransferWrite,
 	                vk::PipelineStageFlagBits2::eCopy, {});
 	vk::DependencyInfo dependency {};
-	dependency.dependencyFlags           = vk::DependencyFlagBits::eByRegion;
+	dependency.dependencyFlags          = vk::DependencyFlagBits::eByRegion;
 	dependency.bufferMemoryBarrierCount = 1;
 	dependency.pBufferMemoryBarriers    = &buffer_barrier;
-	dependency.imageMemoryBarrierCount = static_cast<uint32_t>(image_barriers.size());
-	dependency.pImageMemoryBarriers    = image_barriers.data();
-	auto command = m_scheduler->Current().Handle();
+	dependency.imageMemoryBarrierCount  = static_cast<uint32_t>(image_barriers.size());
+	dependency.pImageMemoryBarriers     = image_barriers.data();
+	auto command                        = m_scheduler->Current().Handle();
 	command.pipelineBarrier2(dependency);
 	command.copyBufferToImage(buffer, backing.image, vk::ImageLayout::eTransferDstOptimal,
 	                          static_cast<uint32_t>(copies.size()), copies.data());
@@ -256,8 +253,7 @@ void Image::Upload(std::span<const vk::BufferImageCopy> copies, vk::Buffer buffe
 	dependency.pImageMemoryBarriers    = nullptr;
 	command.pipelineBarrier2(dependency);
 	Transit(vk::ImageLayout::eGeneral,
-	        vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eTransferRead, {},
-	        command);
+	        vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eTransferRead, {}, command);
 }
 
 void Image::Download(std::span<const vk::BufferImageCopy> copies, vk::Buffer buffer,
@@ -265,7 +261,7 @@ void Image::Download(std::span<const vk::BufferImageCopy> copies, vk::Buffer buf
 	EXIT_IF(m_scheduler == nullptr || copies.empty() || buffer == nullptr || size == 0);
 	m_scheduler->EndRendering();
 	vk::BufferMemoryBarrier2 buffer_barrier {};
-	buffer_barrier.srcStageMask        = vk::PipelineStageFlagBits2::eAllCommands;
+	buffer_barrier.srcStageMask = vk::PipelineStageFlagBits2::eAllCommands;
 	buffer_barrier.srcAccessMask =
 	    vk::AccessFlagBits2::eMemoryRead | vk::AccessFlagBits2::eMemoryWrite;
 	buffer_barrier.dstStageMask        = vk::PipelineStageFlagBits2::eCopy;
@@ -276,16 +272,15 @@ void Image::Download(std::span<const vk::BufferImageCopy> copies, vk::Buffer buf
 	buffer_barrier.offset              = offset;
 	buffer_barrier.size                = size;
 	const auto image_barriers =
-	    GetBarriers(vk::ImageLayout::eTransferSrcOptimal,
-	                vk::AccessFlagBits2::eTransferRead,
+	    GetBarriers(vk::ImageLayout::eTransferSrcOptimal, vk::AccessFlagBits2::eTransferRead,
 	                vk::PipelineStageFlagBits2::eCopy, {});
 	vk::DependencyInfo dependency {};
-	dependency.dependencyFlags           = vk::DependencyFlagBits::eByRegion;
+	dependency.dependencyFlags          = vk::DependencyFlagBits::eByRegion;
 	dependency.bufferMemoryBarrierCount = 1;
 	dependency.pBufferMemoryBarriers    = &buffer_barrier;
-	dependency.imageMemoryBarrierCount = static_cast<uint32_t>(image_barriers.size());
-	dependency.pImageMemoryBarriers    = image_barriers.data();
-	auto command = m_scheduler->Current().Handle();
+	dependency.imageMemoryBarrierCount  = static_cast<uint32_t>(image_barriers.size());
+	dependency.pImageMemoryBarriers     = image_barriers.data();
+	auto command                        = m_scheduler->Current().Handle();
 	command.pipelineBarrier2(dependency);
 	command.copyImageToBuffer(backing.image, vk::ImageLayout::eTransferSrcOptimal, buffer,
 	                          static_cast<uint32_t>(copies.size()), copies.data());
@@ -299,11 +294,11 @@ void Image::Download(std::span<const vk::BufferImageCopy> copies, vk::Buffer buf
 	command.pipelineBarrier2(dependency);
 }
 
-std::pair<uint32_t, uint32_t>
-Image::SanitizeCopyLayers(const Image& source, const Image& destination, uint32_t depth) {
-	const auto source_type      = source.backing.image_type;
-	const auto destination_type = destination.backing.image_type;
-	uint32_t   source_layers    = source.backing.layers;
+std::pair<uint32_t, uint32_t> Image::SanitizeCopyLayers(const Image& source,
+                                                        const Image& destination, uint32_t depth) {
+	const auto source_type        = source.backing.image_type;
+	const auto destination_type   = destination.backing.image_type;
+	uint32_t   source_layers      = source.backing.layers;
 	uint32_t   destination_layers = destination.backing.layers;
 	if (source_type == vk::ImageType::e3D) {
 		source_layers = 1;
@@ -312,13 +307,10 @@ Image::SanitizeCopyLayers(const Image& source, const Image& destination, uint32_
 		destination_layers = 1;
 	}
 	if (source_type == destination_type) {
-		source_layers = destination_layers =
-		    std::min(source_layers, destination_layers);
-	} else if (source_type == vk::ImageType::e2D &&
-	           destination_type == vk::ImageType::e3D) {
+		source_layers = destination_layers = std::min(source_layers, destination_layers);
+	} else if (source_type == vk::ImageType::e2D && destination_type == vk::ImageType::e3D) {
 		source_layers = depth;
-	} else if (source_type == vk::ImageType::e3D &&
-	           destination_type == vk::ImageType::e2D) {
+	} else if (source_type == vk::ImageType::e3D && destination_type == vk::ImageType::e2D) {
 		destination_layers = depth;
 	}
 	return {source_layers, destination_layers};
@@ -327,12 +319,11 @@ Image::SanitizeCopyLayers(const Image& source, const Image& destination, uint32_
 void Image::CopyImage(Image& source) {
 	EXIT_IF(m_scheduler == nullptr || source.backing.samples != backing.samples);
 	m_scheduler->EndRendering();
-	const uint32_t levels =
-	    std::min(source.backing.mip_levels, backing.mip_levels);
+	const uint32_t levels     = std::min(source.backing.mip_levels, backing.mip_levels);
 	const uint32_t base_depth = backing.image_type == vk::ImageType::e3D
 	                                ? backing.extent.depth
 	                                : source.backing.extent.depth;
-	const auto source_aspect =
+	const auto     source_aspect =
 	    FullAspectMask(source.backing.format) & ~vk::ImageAspectFlagBits::eStencil;
 	const auto destination_aspect =
 	    FullAspectMask(backing.format) & ~vk::ImageAspectFlagBits::eStencil;
@@ -342,8 +333,7 @@ void Image::CopyImage(Image& source) {
 		const auto width  = std::max(source.backing.extent.width >> level, 1u);
 		const auto height = std::max(source.backing.extent.height >> level, 1u);
 		const auto depth  = std::max(base_depth >> level, 1u);
-		const auto [source_layers, destination_layers] =
-		    SanitizeCopyLayers(source, *this, depth);
+		const auto [source_layers, destination_layers] = SanitizeCopyLayers(source, *this, depth);
 		vk::ImageCopy copy {};
 		copy.srcSubresource = {source_aspect, level, 0, 1};
 		copy.dstSubresource = {destination_aspect, level, 0, 1};
@@ -351,8 +341,7 @@ void Image::CopyImage(Image& source) {
 			if (source.backing.image_type == vk::ImageType::e3D) {
 				copy.extent = {width, height, depth};
 			} else {
-				copy.srcSubresource.layerCount =
-				    std::min(source_layers, destination_layers);
+				copy.srcSubresource.layerCount = std::min(source_layers, destination_layers);
 				copy.dstSubresource.layerCount = copy.srcSubresource.layerCount;
 				copy.extent                    = {width, height, 1};
 			}
@@ -369,34 +358,30 @@ void Image::CopyImage(Image& source) {
 		return;
 	}
 	auto command = m_scheduler->Current().Handle();
-	source.Transit(vk::ImageLayout::eTransferSrcOptimal,
-	               vk::AccessFlagBits2::eTransferRead, {}, command);
-	Transit(vk::ImageLayout::eTransferDstOptimal,
-	        vk::AccessFlagBits2::eTransferWrite, {}, command);
-	command.copyImage(source.backing.image, vk::ImageLayout::eTransferSrcOptimal,
-	                  backing.image, vk::ImageLayout::eTransferDstOptimal,
-	                  static_cast<uint32_t>(copies.size()), copies.data());
+	source.Transit(vk::ImageLayout::eTransferSrcOptimal, vk::AccessFlagBits2::eTransferRead, {},
+	               command);
+	Transit(vk::ImageLayout::eTransferDstOptimal, vk::AccessFlagBits2::eTransferWrite, {}, command);
+	command.copyImage(source.backing.image, vk::ImageLayout::eTransferSrcOptimal, backing.image,
+	                  vk::ImageLayout::eTransferDstOptimal, static_cast<uint32_t>(copies.size()),
+	                  copies.data());
 	Transit(vk::ImageLayout::eGeneral,
-	        vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eTransferRead, {},
-	        command);
+	        vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eTransferRead, {}, command);
 }
 
 void Image::Resolve(Image& source, const ImageSubresourceRange& source_range,
                     const ImageSubresourceRange& destination_range) {
 	EXIT_IF(m_scheduler == nullptr || backing.samples != 1 ||
 	        source.backing.image_type != vk::ImageType::e2D ||
-	        backing.image_type != vk::ImageType::e2D ||
-	        source_range.level_count != 1 || destination_range.level_count != 1 ||
+	        backing.image_type != vk::ImageType::e2D || source_range.level_count != 1 ||
+	        destination_range.level_count != 1 ||
 	        source_range.base_level >= source.backing.mip_levels ||
 	        destination_range.base_level >= backing.mip_levels ||
 	        source_range.base_layer >= source.backing.layers ||
 	        destination_range.base_layer >= backing.layers);
-	const auto layers = std::min(
-	    {source_range.layer_count, destination_range.layer_count,
-	     source.backing.layers - source_range.base_layer,
-	     backing.layers - destination_range.base_layer});
-	const auto source_width =
-	    std::max(source.backing.extent.width >> source_range.base_level, 1u);
+	const auto layers       = std::min({source_range.layer_count, destination_range.layer_count,
+	                                    source.backing.layers - source_range.base_layer,
+	                                    backing.layers - destination_range.base_layer});
+	const auto source_width = std::max(source.backing.extent.width >> source_range.base_level, 1u);
 	const auto source_height =
 	    std::max(source.backing.extent.height >> source_range.base_level, 1u);
 	const auto destination_width =
@@ -404,43 +389,40 @@ void Image::Resolve(Image& source, const ImageSubresourceRange& source_range,
 	const auto destination_height =
 	    std::max(backing.extent.height >> destination_range.base_level, 1u);
 	const bool copy = source.backing.samples == 1;
-	EXIT_IF(layers == 0 || info.extent.width > source_width ||
-	        info.extent.height > source_height || info.extent.width > destination_width ||
-	        info.extent.height > destination_height ||
+	EXIT_IF(layers == 0 || info.extent.width > source_width || info.extent.height > source_height ||
+	        info.extent.width > destination_width || info.extent.height > destination_height ||
 	        (copy ? !ImageViewOps::FormatsCompatible(source.backing.format, backing.format)
 	              : source.backing.format != backing.format));
-	auto resolved_source_range      = source_range;
-	auto resolved_destination_range = destination_range;
+	auto resolved_source_range             = source_range;
+	auto resolved_destination_range        = destination_range;
 	resolved_source_range.layer_count      = layers;
 	resolved_destination_range.layer_count = layers;
 	const vk::Extent3D resolve_extent {info.extent.width, info.extent.height, 1};
 
 	m_scheduler->EndRendering();
 	auto command = m_scheduler->Current().Handle();
-	source.Transit(vk::ImageLayout::eTransferSrcOptimal,
-	               vk::AccessFlagBits2::eTransferRead, resolved_source_range, command);
-	Transit(vk::ImageLayout::eTransferDstOptimal,
-	        vk::AccessFlagBits2::eTransferWrite, resolved_destination_range, command);
+	source.Transit(vk::ImageLayout::eTransferSrcOptimal, vk::AccessFlagBits2::eTransferRead,
+	               resolved_source_range, command);
+	Transit(vk::ImageLayout::eTransferDstOptimal, vk::AccessFlagBits2::eTransferWrite,
+	        resolved_destination_range, command);
 	if (copy) {
 		vk::ImageCopy region {};
-		region.srcSubresource = {vk::ImageAspectFlagBits::eColor,
-		                         resolved_source_range.base_level,
+		region.srcSubresource = {vk::ImageAspectFlagBits::eColor, resolved_source_range.base_level,
 		                         resolved_source_range.base_layer, layers};
 		region.dstSubresource = {vk::ImageAspectFlagBits::eColor,
 		                         resolved_destination_range.base_level,
 		                         resolved_destination_range.base_layer, layers};
-		region.extent = resolve_extent;
-		command.copyImage(source.backing.image, vk::ImageLayout::eTransferSrcOptimal,
-		                  backing.image, vk::ImageLayout::eTransferDstOptimal, region);
+		region.extent         = resolve_extent;
+		command.copyImage(source.backing.image, vk::ImageLayout::eTransferSrcOptimal, backing.image,
+		                  vk::ImageLayout::eTransferDstOptimal, region);
 	} else {
 		vk::ImageResolve region {};
-		region.srcSubresource = {vk::ImageAspectFlagBits::eColor,
-		                         resolved_source_range.base_level,
+		region.srcSubresource = {vk::ImageAspectFlagBits::eColor, resolved_source_range.base_level,
 		                         resolved_source_range.base_layer, layers};
 		region.dstSubresource = {vk::ImageAspectFlagBits::eColor,
 		                         resolved_destination_range.base_level,
 		                         resolved_destination_range.base_layer, layers};
-		region.extent = resolve_extent;
+		region.extent         = resolve_extent;
 		command.resolveImage(source.backing.image, vk::ImageLayout::eTransferSrcOptimal,
 		                     backing.image, vk::ImageLayout::eTransferDstOptimal, region);
 	}
@@ -454,22 +436,21 @@ uint32_t Image::CopyRows(uint64_t row_size, uint32_t rows, uint64_t capacity) no
 }
 
 void Image::CopyImageWithBuffer(Image& source, Buffer& buffer) {
-	EXIT_IF(m_scheduler == nullptr || buffer.Handle() == nullptr ||
-	        source.backing.samples != 1 || backing.samples != 1);
+	EXIT_IF(m_scheduler == nullptr || buffer.Handle() == nullptr || source.backing.samples != 1 ||
+	        backing.samples != 1);
 	m_scheduler->EndRendering();
-	const uint32_t levels =
-	    std::min(source.backing.mip_levels, backing.mip_levels);
-	const auto source_aspect =
+	const uint32_t levels = std::min(source.backing.mip_levels, backing.mip_levels);
+	const auto     source_aspect =
 	    FullAspectMask(source.backing.format) & ~vk::ImageAspectFlagBits::eStencil;
 	const auto destination_aspect =
 	    FullAspectMask(backing.format) & ~vk::ImageAspectFlagBits::eStencil;
-	const auto source_bytes = DepthAspectTransferBytes(source.backing.format) != 0
-	                              ? DepthAspectTransferBytes(source.backing.format)
-	                              : source.info.bytes_per_block;
-	const auto destination_bytes = DepthAspectTransferBytes(backing.format) != 0
-	                                   ? DepthAspectTransferBytes(backing.format)
-	                                   : info.bytes_per_block;
-	const uint32_t source_block = source.info.IsBlock() ? 4u : 1u;
+	const auto     source_bytes      = DepthAspectTransferBytes(source.backing.format) != 0
+	                                       ? DepthAspectTransferBytes(source.backing.format)
+	                                       : source.info.bytes_per_block;
+	const auto     destination_bytes = DepthAspectTransferBytes(backing.format) != 0
+	                                       ? DepthAspectTransferBytes(backing.format)
+	                                       : info.bytes_per_block;
+	const uint32_t source_block      = source.info.IsBlock() ? 4u : 1u;
 	const uint32_t destination_block = info.IsBlock() ? 4u : 1u;
 	EXIT_IF(levels == 0 || source_bytes == 0 || source_bytes != destination_bytes ||
 	        source_block != destination_block);
@@ -484,74 +465,66 @@ void Image::CopyImageWithBuffer(Image& source, Buffer& buffer) {
 	barrier.buffer              = buffer.Handle();
 	barrier.offset              = 0;
 	vk::DependencyInfo dependency {};
-	dependency.dependencyFlags           = vk::DependencyFlagBits::eByRegion;
+	dependency.dependencyFlags          = vk::DependencyFlagBits::eByRegion;
 	dependency.bufferMemoryBarrierCount = 1;
 	dependency.pBufferMemoryBarriers    = &barrier;
-	auto command = m_scheduler->Current().Handle();
-	source.Transit(vk::ImageLayout::eTransferSrcOptimal,
-	               vk::AccessFlagBits2::eTransferRead, {}, command);
-	Transit(vk::ImageLayout::eTransferDstOptimal,
-	        vk::AccessFlagBits2::eTransferWrite, {}, command);
+	auto command                        = m_scheduler->Current().Handle();
+	source.Transit(vk::ImageLayout::eTransferSrcOptimal, vk::AccessFlagBits2::eTransferRead, {},
+	               command);
+	Transit(vk::ImageLayout::eTransferDstOptimal, vk::AccessFlagBits2::eTransferWrite, {}, command);
 	for (uint32_t level = 0; level < levels; level++) {
-		const auto width  = std::max(source.backing.extent.width >> level, 1u);
-		const auto height = std::max(source.backing.extent.height >> level, 1u);
-		const auto source_depth = source.backing.image_type == vk::ImageType::e3D
-		                              ? std::max(source.backing.extent.depth >> level, 1u)
-		                              : source.backing.layers;
+		const auto width             = std::max(source.backing.extent.width >> level, 1u);
+		const auto height            = std::max(source.backing.extent.height >> level, 1u);
+		const auto source_depth      = source.backing.image_type == vk::ImageType::e3D
+		                                   ? std::max(source.backing.extent.depth >> level, 1u)
+		                                   : source.backing.layers;
 		const auto destination_depth = backing.image_type == vk::ImageType::e3D
 		                                   ? std::max(backing.extent.depth >> level, 1u)
 		                                   : backing.layers;
-		const auto slices    = std::min(source_depth, destination_depth);
-		const auto block_rows = (height + source_block - 1) / source_block;
+		const auto slices            = std::min(source_depth, destination_depth);
+		const auto block_rows        = (height + source_block - 1) / source_block;
 		const auto row_size =
 		    static_cast<uint64_t>((width + source_block - 1) / source_block) * source_bytes;
 		const auto rows_per_copy = CopyRows(row_size, block_rows, buffer.Size());
 		EXIT_IF(slices == 0 || rows_per_copy == 0);
 		for (uint32_t slice = 0; slice < slices; slice++) {
-			for (uint32_t block_row = 0; block_row < block_rows;
-			     block_row += rows_per_copy) {
-				const auto copy_rows = std::min(rows_per_copy, block_rows - block_row);
-				const auto y         = block_row * source_block;
-				const auto copy_height =
-				    std::min(copy_rows * source_block, height - y);
-				const auto copy_size = row_size * copy_rows;
+			for (uint32_t block_row = 0; block_row < block_rows; block_row += rows_per_copy) {
+				const auto          copy_rows   = std::min(rows_per_copy, block_rows - block_row);
+				const auto          y           = block_row * source_block;
+				const auto          copy_height = std::min(copy_rows * source_block, height - y);
+				const auto          copy_size   = row_size * copy_rows;
 				vk::BufferImageCopy source_copy {};
 				source_copy.imageSubresource = {
 				    source_aspect, level,
 				    source.backing.image_type == vk::ImageType::e3D ? 0u : slice, 1};
-				source_copy.imageOffset = {
-				    0, static_cast<int32_t>(y),
-				    source.backing.image_type == vk::ImageType::e3D
-				        ? static_cast<int32_t>(slice)
-				        : 0};
-				source_copy.imageExtent = {width, copy_height, 1};
-				auto destination_copy = source_copy;
+				source_copy.imageOffset           = {0, static_cast<int32_t>(y),
+				                                     source.backing.image_type == vk::ImageType::e3D
+				                                         ? static_cast<int32_t>(slice)
+				                                         : 0};
+				source_copy.imageExtent           = {width, copy_height, 1};
+				auto destination_copy             = source_copy;
 				destination_copy.imageSubresource = {
 				    destination_aspect, level,
 				    backing.image_type == vk::ImageType::e3D ? 0u : slice, 1};
 				destination_copy.imageOffset.z =
-				    backing.image_type == vk::ImageType::e3D
-				        ? static_cast<int32_t>(slice)
-				        : 0;
+				    backing.image_type == vk::ImageType::e3D ? static_cast<int32_t>(slice) : 0;
 				barrier.size          = copy_size;
 				barrier.srcAccessMask = vk::AccessFlagBits2::eTransferRead;
 				barrier.dstAccessMask = vk::AccessFlagBits2::eTransferWrite;
 				command.pipelineBarrier2(dependency);
 				command.copyImageToBuffer(source.backing.image,
-				                          vk::ImageLayout::eTransferSrcOptimal,
-				                          buffer.Handle(), source_copy);
+				                          vk::ImageLayout::eTransferSrcOptimal, buffer.Handle(),
+				                          source_copy);
 				barrier.srcAccessMask = vk::AccessFlagBits2::eTransferWrite;
 				barrier.dstAccessMask = vk::AccessFlagBits2::eTransferRead;
 				command.pipelineBarrier2(dependency);
 				command.copyBufferToImage(buffer.Handle(), backing.image,
-				                          vk::ImageLayout::eTransferDstOptimal,
-				                          destination_copy);
+				                          vk::ImageLayout::eTransferDstOptimal, destination_copy);
 			}
 		}
 	}
 	Transit(vk::ImageLayout::eGeneral,
-	        vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eTransferRead, {},
-	        command);
+	        vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eTransferRead, {}, command);
 }
 
 void Image::CopyMip(Image& source, uint32_t mip, uint32_t layer) {
@@ -561,11 +534,9 @@ void Image::CopyMip(Image& source, uint32_t mip, uint32_t layer) {
 	const auto width  = std::max(backing.extent.width >> mip, 1u);
 	const auto height = std::max(backing.extent.height >> mip, 1u);
 	const auto depth  = std::max(backing.extent.depth >> mip, 1u);
-	EXIT_IF(width != source.backing.extent.width ||
-	        height != source.backing.extent.height);
-	const auto [source_layers, destination_layers] =
-	    SanitizeCopyLayers(source, *this, depth);
-	const auto aspects = FullAspectMask(source.backing.format);
+	EXIT_IF(width != source.backing.extent.width || height != source.backing.extent.height);
+	const auto [source_layers, destination_layers] = SanitizeCopyLayers(source, *this, depth);
+	const auto aspects                             = FullAspectMask(source.backing.format);
 	EXIT_IF(aspects != FullAspectMask(backing.format));
 	std::array<vk::ImageCopy, 2> copies {};
 	uint32_t                     copy_count = 0;
@@ -580,16 +551,13 @@ void Image::CopyMip(Image& source, uint32_t mip, uint32_t layer) {
 		copy.extent         = {width, height, depth};
 	}
 	auto command = m_scheduler->Current().Handle();
-	Transit(vk::ImageLayout::eTransferDstOptimal,
-	        vk::AccessFlagBits2::eTransferWrite, {}, command);
-	source.Transit(vk::ImageLayout::eTransferSrcOptimal,
-	               vk::AccessFlagBits2::eTransferRead, {}, command);
-	command.copyImage(source.backing.image, vk::ImageLayout::eTransferSrcOptimal,
-	                  backing.image, vk::ImageLayout::eTransferDstOptimal, copy_count,
-	                  copies.data());
+	Transit(vk::ImageLayout::eTransferDstOptimal, vk::AccessFlagBits2::eTransferWrite, {}, command);
+	source.Transit(vk::ImageLayout::eTransferSrcOptimal, vk::AccessFlagBits2::eTransferRead, {},
+	               command);
+	command.copyImage(source.backing.image, vk::ImageLayout::eTransferSrcOptimal, backing.image,
+	                  vk::ImageLayout::eTransferDstOptimal, copy_count, copies.data());
 	Transit(vk::ImageLayout::eGeneral,
-	        vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eTransferRead, {},
-	        command);
+	        vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eTransferRead, {}, command);
 }
 
 namespace ImageOps {
@@ -601,8 +569,7 @@ void Validate(const ImageInfo& info) {
 	if (info.pixel_format == vk::Format::eUndefined) {
 		const bool metadata_empty =
 		    info.metadata.range.address == 0 && info.metadata.range.size == 0 &&
-		    info.metadata.kind == ImageMetadataKind::None &&
-		    info.metadata.control == 0 &&
+		    info.metadata.kind == ImageMetadataKind::None && info.metadata.control == 0 &&
 		    info.metadata.compression == VideoOutCompression::Uncompressed &&
 		    !info.metadata.stencil_compressed;
 		if (info.data.Empty() || info.HasStencil() || !metadata_empty || info.extent.width == 0 ||
@@ -615,9 +582,9 @@ void Validate(const ImageInfo& info) {
 	}
 
 	if (info.extent.width == 0 || info.extent.height == 0 || info.extent.depth == 0 ||
-	    info.resources.levels == 0 ||
-	    info.resources.levels > info.mip_layout.size() || info.resources.layers == 0 ||
-	    info.samples == 0 || vulkan_sample_count(info.samples) == vk::SampleCountFlagBits {} ||
+	    info.resources.levels == 0 || info.resources.levels > info.mip_layout.size() ||
+	    info.resources.layers == 0 || info.samples == 0 ||
+	    vulkan_sample_count(info.samples) == vk::SampleCountFlagBits {} ||
 	    info.bytes_per_block == 0 || (info.data.address != 0 && info.pitch == 0)) {
 		EXIT("invalid image geometry or format\n");
 	}
@@ -688,11 +655,11 @@ uint32_t RenderTargetTransferFormat(uint32_t bytes_per_element) {
 
 } // namespace ImageOps
 
-Image::Image(GraphicContext& graphics, CommandScheduler& scheduler,
-             const ImageInfo& image_info)
+Image::Image(GraphicContext& graphics, CommandScheduler& scheduler, const ImageInfo& image_info)
     : info(image_info), m_graphics(&graphics), m_scheduler(&scheduler) {
 	KYTY_PROFILER_FUNCTION();
 	ImageOps::Validate(info);
+	m_cpu_dirty = !info.data.Empty();
 	if (info.pixel_format == vk::Format::eUndefined) {
 		return;
 	}
@@ -742,9 +709,9 @@ Image::Image(GraphicContext& graphics, CommandScheduler& scheduler,
 }
 
 uint64_t Image::HashGuestEdges() const {
-	constexpr uint64_t page_mask = TRACKER_PAGE_SIZE - 1;
+	constexpr uint64_t                         page_mask = TRACKER_PAGE_SIZE - 1;
 	std::array<uint8_t, TRACKER_PAGE_SIZE * 2> bytes {};
-	const auto     range        = info.data;
+	const auto                                 range = info.data;
 	const uint64_t head_end     = std::min(range.End(), (range.address + page_mask) & ~page_mask);
 	const uint64_t tail_begin   = std::max(range.address, range.End() & ~page_mask);
 	const uint64_t head_size    = head_end - range.address;

src/graphics/host_gpu/renderer/image/image.h

@@ -66,16 +66,16 @@ public:
 	[[nodiscard]] vk::ImageView FindView(const ImageViewInfo& view_info);
 	void                        AssociateDepth(ImageId image_id) { depth_id = image_id; }
 	using Barriers = std::vector<vk::ImageMemoryBarrier2>;
-	[[nodiscard]] Barriers
-	GetBarriers(vk::ImageLayout destination_layout, vk::AccessFlags2 destination_access,
-	            vk::PipelineStageFlags2 destination_stage,
-	            std::optional<ImageSubresourceRange> range);
+	[[nodiscard]] Barriers GetBarriers(vk::ImageLayout                      destination_layout,
+	                                   vk::AccessFlags2                     destination_access,
+	                                   vk::PipelineStageFlags2              destination_stage,
+	                                   std::optional<ImageSubresourceRange> range);
 	void Transit(vk::ImageLayout destination_layout, vk::AccessFlags2 destination_access,
 	             std::optional<ImageSubresourceRange> range, vk::CommandBuffer command_buffer);
-	void Upload(std::span<const vk::BufferImageCopy> copies, vk::Buffer buffer,
-	            uint64_t offset, uint64_t size);
-	void Download(std::span<const vk::BufferImageCopy> copies, vk::Buffer buffer,
-	              uint64_t offset, uint64_t size);
+	void Upload(std::span<const vk::BufferImageCopy> copies, vk::Buffer buffer, uint64_t offset,
+	            uint64_t size);
+	void Download(std::span<const vk::BufferImageCopy> copies, vk::Buffer buffer, uint64_t offset,
+	              uint64_t size);
 	void CopyImage(Image& source);
 	void Resolve(Image& source, const ImageSubresourceRange& source_range,
 	             const ImageSubresourceRange& destination_range);
@@ -84,8 +84,8 @@ public:
 
 	void InvalidateCpuWrite(uint64_t vaddr, uint64_t size) {
 		if (ImageRangeOverlaps(info.data.address, info.data.size, vaddr, size)) {
-			m_cpu_dirty       = true;
-			m_maybe_cpu_dirty = false;
+			m_cpu_dirty        = true;
+			m_maybe_cpu_dirty  = false;
 			m_maybe_hash_valid = false;
 		} else if (ImagePageRangesOverlap(info.data.address, info.data.size, vaddr, size)) {
 			m_maybe_cpu_dirty = true;
@@ -95,6 +95,11 @@ public:
 	[[nodiscard]] bool IsCpuDirty() const { return m_cpu_dirty || m_maybe_cpu_dirty; }
 	[[nodiscard]] bool IsDefinitelyCpuDirty() const { return m_cpu_dirty; }
 	[[nodiscard]] bool IsMaybeCpuDirty() const { return m_maybe_cpu_dirty; }
+	void               MarkMaybeCpuDirty() {
+		if (!m_cpu_dirty) {
+			m_maybe_cpu_dirty = true;
+		}
+	}
 	[[nodiscard]] bool NeedsMaybeCpuHash() const {
 		return m_maybe_cpu_dirty && !m_maybe_hash_valid;
 	}
@@ -102,14 +107,14 @@ public:
 		if (!NeedsMaybeCpuHash()) {
 			EXIT("image cannot initialize maybe-dirty hash\n");
 		}
-		m_maybe_cpu_hash = hash;
+		m_maybe_cpu_hash   = hash;
 		m_maybe_hash_valid = true;
 	}
 	[[nodiscard]] bool ResolveMaybeCpuHash(uint64_t hash) {
 		if (!m_maybe_cpu_dirty || !m_maybe_hash_valid || m_cpu_dirty) {
 			EXIT("image cannot resolve maybe-dirty hash\n");
 		}
-		m_maybe_cpu_dirty = false;
+		m_maybe_cpu_dirty  = false;
 		m_maybe_hash_valid = false;
 		m_cpu_dirty |= hash != m_maybe_cpu_hash;
 		return m_cpu_dirty;
@@ -125,14 +130,15 @@ public:
 	}
 
 	[[nodiscard]] bool IsGpuModified() const noexcept { return m_gpu_modified; }
-	void MarkGpuModified() noexcept { m_gpu_modified = true; }
-	void ClearGpuModified() noexcept { m_gpu_modified = false; }
+	void               MarkGpuModified() noexcept { m_gpu_modified = true; }
+	void               ClearGpuModified() noexcept { m_gpu_modified = false; }
 
 	[[nodiscard]] bool IsBufferModified() const noexcept { return m_buffer_modified; }
-	void MarkBufferModified() noexcept { m_buffer_modified = true; }
-	void ClearBufferModified() noexcept { m_buffer_modified = false; }
+	void               MarkBufferModified() noexcept { m_buffer_modified = true; }
+	void               ClearBufferModified() noexcept { m_buffer_modified = false; }
 
-	[[nodiscard]] bool Overlaps(uint64_t address, uint64_t size, bool pages = false) const noexcept {
+	[[nodiscard]] bool Overlaps(uint64_t address, uint64_t size,
+	                            bool pages = false) const noexcept {
 		return pages ? ImagePageRangesOverlap(info.data.address, info.data.size, address, size)
 		             : ImageRangeOverlaps(info.data.address, info.data.size, address, size);
 	}
@@ -142,38 +148,41 @@ public:
 	[[nodiscard]] bool SafeToDownload() const noexcept {
 		return IsGpuModified() && !IsBufferModified() && !IsCpuDirty();
 	}
+	[[nodiscard]] bool IsTracked() const noexcept { return track_addr != 0 && track_addr_end != 0; }
 	[[nodiscard]] uint64_t AccountedSize() const noexcept {
 		return backing.image == nullptr ? 0 : (info.data.size + 1023) & ~uint64_t {1023};
 	}
 	[[nodiscard]] uint64_t HashGuestEdges() const;
 
-	ImageInfo            info;
-	VulkanImage          backing;
-	ImageViewCache       views;
-	ImageUsage           usage;
-	ImageBinding         binding;
-	bool                 registered          = false;
-	ImageId              depth_id {};
-	uint64_t             tick_accessed_last = 0;
-	size_t               lru_id = 0;
+	ImageInfo      info;
+	VulkanImage    backing;
+	ImageViewCache views;
+	ImageUsage     usage;
+	ImageBinding   binding;
+	bool           registered     = false;
+	uint64_t       track_addr     = 0;
+	uint64_t       track_addr_end = 0;
+	ImageId        depth_id {};
+	uint64_t       tick_accessed_last = 0;
+	size_t         lru_id             = 0;
 
 private:
 	friend struct ImageTestAccess;
 
 	[[nodiscard]] static vk::ImageAspectFlags FullAspectMask(vk::Format format) noexcept;
-	[[nodiscard]] static uint32_t CopyRows(uint64_t row_size, uint32_t rows,
-	                                       uint64_t capacity) noexcept;
+	[[nodiscard]] static uint32_t             CopyRows(uint64_t row_size, uint32_t rows,
+	                                                   uint64_t capacity) noexcept;
 	[[nodiscard]] static std::pair<uint32_t, uint32_t>
 	SanitizeCopyLayers(const Image& source, const Image& destination, uint32_t depth);
 
-	GraphicContext*   m_graphics  = nullptr;
-	CommandScheduler* m_scheduler = nullptr;
-	uint64_t        m_maybe_cpu_hash = 0;
-	bool            m_cpu_dirty = false;
-	bool            m_maybe_cpu_dirty = false;
-	bool            m_maybe_hash_valid = false;
-	bool            m_gpu_modified = false;
-	bool            m_buffer_modified = false;
+	GraphicContext*   m_graphics         = nullptr;
+	CommandScheduler* m_scheduler        = nullptr;
+	uint64_t          m_maybe_cpu_hash   = 0;
+	bool              m_cpu_dirty        = false;
+	bool              m_maybe_cpu_dirty  = false;
+	bool              m_maybe_hash_valid = false;
+	bool              m_gpu_modified     = false;
+	bool              m_buffer_modified  = false;
 };
 
 namespace ImageOps {

src/graphics/host_gpu/renderer/image/imageView.h

@@ -103,10 +103,7 @@ IsSupportedSampledDepthUintResource(const ShaderRecompiler::IR::ImageResource& r
 
 inline void ValidateStorageColorView(vk::Format image_format, vk::Format view_format,
                                      uint32_t swizzle) noexcept {
-	const auto srgb_view = SrgbStorageViewFormat(image_format);
-	const bool srgb_storage_view =
-	    srgb_view != vk::Format::eUndefined && view_format == srgb_view;
-	if ((image_format != view_format && !srgb_storage_view) ||
+	if (!ImageViewOps::FormatsCompatible(image_format, view_format) ||
 	    !IsValidImageSwizzle(swizzle)) {
 		UnsupportedColorView("storage", image_format, view_format, swizzle);
 	}
@@ -122,7 +119,10 @@ IsSupportedStorageImageResource(const ShaderRecompiler::IR::ImageResource& resou
 	        resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim3D ||
 	        resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2DArray) &&
 	       resource.mip_mode == ShaderRecompiler::IR::ImageMipMode::None && resource.written &&
-	       !resource.atomic && !resource.depth_compare;
+	       (!resource.atomic ||
+	        (resource.kind == ShaderRecompiler::IR::ResourceKind::StorageImageUint &&
+	         resource.read)) &&
+	       !resource.depth_compare;
 }
 
 inline void

src/graphics/host_gpu/renderer/pipeline/descriptors.cpp

@@ -14,13 +14,13 @@
 #include "graphics/guest_gpu/tile.h"
 #include "graphics/host_gpu/graphicContext.h"
 #include "graphics/host_gpu/hostMemory.h"
-#include "graphics/host_gpu/renderer/image/textureCommon.h"
 #include "graphics/host_gpu/renderer/debug.h"
-#include "graphics/host_gpu/renderer/pipeline/descriptorCache.h"
 #include "graphics/host_gpu/renderer/image/imageView.h"
+#include "graphics/host_gpu/renderer/image/textureCommon.h"
+#include "graphics/host_gpu/renderer/pipeline/descriptorCache.h"
+#include "graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.h"
 #include "graphics/host_gpu/renderer/render.h"
 #include "graphics/host_gpu/renderer/renderContext.h"
-#include "graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.h"
 #include "graphics/host_gpu/vma.h"
 #include "graphics/host_gpu/vulkanCommon.h"
 #include "graphics/shader/recompiler/ir/BindingLayout.h"
@@ -239,11 +239,11 @@ bool IsSupportedDepthTextureEncoding(const ShaderTextureResource& descriptor, co
 	const uint32_t     field3_expected =
 	    (descriptor.Type() << 28u) | field3_common | descriptor.DstSelXYZW();
 	const uint32_t field4_expected = descriptor.Depth() | (descriptor.BaseArray5() << 16u);
-	const bool common = (descriptor.fields[1] & field1_reserved_mask) == 0 &&
-	                    (descriptor.fields[2] & field2_reserved_mask) == 0 &&
-	                    descriptor.fields[3] == field3_expected &&
-	                    descriptor.fields[4] == field4_expected &&
-	                    descriptor.fields[5] == field5_expected;
+	const bool     common          = (descriptor.fields[1] & field1_reserved_mask) == 0 &&
+	                                 (descriptor.fields[2] & field2_reserved_mask) == 0 &&
+	                                 descriptor.fields[3] == field3_expected &&
+	                                 descriptor.fields[4] == field4_expected &&
+	                                 descriptor.fields[5] == field5_expected;
 	if (!common || (descriptor.fields[6] == 0 && descriptor.fields[7] != 0)) {
 		return false;
 	}
@@ -318,8 +318,8 @@ static bool IsSupportedStorageTextureDescriptor(const ShaderRecompiler::IR::Imag
 	const bool valid_2d_slice =
 	    (is_color_2d && descriptor.Depth() == 0 && descriptor.BaseArray5() == 0) ||
 	    (is_color_2d_array && descriptor.BaseArray5() <= descriptor.Depth());
-	const bool is_2d = resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2D &&
-	                   valid_2d_slice;
+	const bool is_2d =
+	    resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2D && valid_2d_slice;
 	const bool is_2d_array =
 	    resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2DArray &&
 	    is_color_2d_array && descriptor.BaseArray5() <= descriptor.Depth();
@@ -340,13 +340,13 @@ static bool IsSupportedStorageTextureDescriptor(const ShaderRecompiler::IR::Imag
 	const bool supported_tile = tile == Prospero::GpuEnumValue(Prospero::TileMode::kLinear) ||
 	                            tile == Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget) ||
 	                            supported_depth_tile || supported_standard_tile;
+	const auto swizzle        = descriptor.DstSelXYZW();
 	const bool supported_swizzle =
-	    IsValidImageSwizzle(descriptor.DstSelXYZW()) &&
-	    (descriptor.DstSelXYZW() == DstSel(4, 5, 6, 7) || !resource.read);
+	    IsValidImageSwizzle(swizzle) &&
+	    (swizzle == DstSel(4, 5, 6, 7) || !resource.read || resource.atomic);
 	const bool supported_mip_view = descriptor.BaseLevel() == 0 || is_1d || is_2d;
 	return (is_1d || is_1d_array || is_2d || is_2d_array || is_3d) && supported_tile &&
-	       supported_mip_view &&
-	       descriptor.BaseLevel() == descriptor.LastLevel() &&
+	       supported_mip_view && descriptor.BaseLevel() == descriptor.LastLevel() &&
 	       descriptor.LastLevel() <= descriptor.MaxMip() && descriptor.MinLod() == 0 &&
 	       supported_swizzle && descriptor.BCSwizzle() == 0 && !descriptor.MsaaDepth();
 }
@@ -377,8 +377,10 @@ void ValidateStorageTexture(const ShaderRecompiler::IR::ImageResource& resource,
 	const bool encoding_ok   = IsSupportedStorageTextureEncoding(descriptor);
 	const bool uint_resource =
 	    resource.kind == ShaderRecompiler::IR::ResourceKind::StorageImageUint;
-	const bool format_ok = Prospero::IsSupportedTextureFormat(format) &&
-	                       uint_resource == Prospero::IsUintTextureFormat(format);
+	const bool format_ok =
+	    Prospero::IsSupportedTextureFormat(format) &&
+	    uint_resource == Prospero::IsUintTextureFormat(format) &&
+	    (!resource.atomic || format == Prospero::GpuEnumValue(Prospero::BufferFormat::k32UInt));
 	if (resource_ok && descriptor_ok && encoding_ok && format_ok && size != 0) {
 		return;
 	}
@@ -614,16 +616,14 @@ RenderExecutor::ResolveTexture(const ShaderRecompiler::IR::ImageResource&   reso
 	                     (address & (static_cast<uint64_t>(size.align) - 1u)) != 0);
 	if (storage) {
 		ValidateStorageTexture(resource, descriptor, size.size);
-		m_context.GetBufferCache().ValidateGpuAccess(address, size.size, resource.read,
-		                                             resource.written);
 	}
 
-	const auto              pixel_format = TextureGetFormat(format);
+	const auto              pixel_format        = TextureGetFormat(format);
 	const auto              storage_view_format = SrgbStorageViewFormat(pixel_format);
-	const auto              view_format =
-	    storage && storage_view_format != vk::Format::eUndefined ? storage_view_format
-	                                                            : pixel_format;
-	const auto              block_bytes  = Prospero::BlockCompressedBytesPerBlock(format);
+	const auto              view_format = storage && storage_view_format != vk::Format::eUndefined
+	                                          ? storage_view_format
+	                                          : pixel_format;
+	const auto              block_bytes = Prospero::BlockCompressedBytesPerBlock(format);
 	TextureCache::ImageDesc desc {};
 	desc.info.data         = {address, size.size};
 	desc.info.pixel_format = pixel_format;

src/graphics/host_gpu/renderer/renderDraw.cpp

@@ -18,15 +18,16 @@
 #include "graphics/host_gpu/renderer/depthRenderTarget.h"
 #include "graphics/host_gpu/renderer/pipeline/descriptorCache.h"
 #include "graphics/host_gpu/renderer/pipeline/pipelineCache.h"
-#include "graphics/host_gpu/renderer/render.h"
-#include "graphics/host_gpu/renderer/renderContext.h"
 #include "graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.h"
 #include "graphics/host_gpu/renderer/pipeline/shaderSubgroup.h"
+#include "graphics/host_gpu/renderer/render.h"
+#include "graphics/host_gpu/renderer/renderContext.h"
 #include "graphics/host_gpu/vulkanCommon.h"
 #include "graphics/shader/recompiler/ir/ResourceMaterialization.h"
 #include "graphics/shader/recompiler/ir/ShaderIR.h"
 #include "graphics/shader/shader.h"
 #include "kernel/eventQueue.h"
+#include "kernel/memory.h"
 #include "kernel/pthread.h"
 #include "libs/errno.h"
 
@@ -221,8 +222,7 @@ static void LogDrawTargetState(const char* draw_name, const RenderColorInfo& col
 	LogMrtState(draw_name, buffer, ps_input_info);
 }
 
-static void LogDrawInputState(const RenderCommandBuffer&   buffer,
-                               const RenderColorInfo&       color,
+static void LogDrawInputState(const RenderCommandBuffer& buffer, const RenderColorInfo& color,
                               const ShaderVertexInputInfo& vs_input_info,
                               uint32_t index_type_and_size, uint32_t index_count,
                               const void* index_addr) {
@@ -499,9 +499,9 @@ struct DrawCallInfo {
 };
 
 RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderColorInfo* colors,
-                                             uint32_t color_count, RenderDepthInfo& depth) {
+                                                 uint32_t color_count, RenderDepthInfo& depth) {
 	EXIT_IF(colors == nullptr || color_count > RENDER_COLOR_ATTACHMENTS_MAX);
-	auto& cache = m_context.GetTextureCache();
+	auto&       cache = m_context.GetTextureCache();
 	RenderState state {};
 	state.width                 = std::numeric_limits<uint32_t>::max();
 	state.height                = std::numeric_limits<uint32_t>::max();
@@ -512,8 +512,7 @@ RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderCo
 		auto& target = colors[i];
 		EXIT_IF(!target.image_id);
 		const auto old_image = cache.ResolveOwner(target.image_id);
-		if (old_image == nullptr ||
-		    (!old_image->registered && !old_image->info.data.Empty()) ||
+		if (old_image == nullptr || (!old_image->registered && !old_image->info.data.Empty()) ||
 		    old_image->binding.needs_rebind) {
 			if (old_image != nullptr) {
 				old_image->binding = {};
@@ -522,7 +521,7 @@ RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderCo
 			BindRenderTarget(target.image_id);
 		}
 		target.image_view = cache.FindRenderTarget(target.image_id, target.desc);
-		auto& image = cache.GetImage(target.image_id);
+		auto& image       = cache.GetImage(target.image_id);
 		EXIT_IF(image.backing.samples != target.samples || target.image_view == nullptr);
 		if (attachment_samples == 0) {
 			attachment_samples = target.samples;
@@ -530,20 +529,19 @@ RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderCo
 			EXIT("mixed color attachment sample counts are unsupported: %u and %u\n",
 			     attachment_samples, target.samples);
 		}
-		const auto& view = target.desc.view_info;
-		const auto layout =
-		    image.binding.is_bound ? vk::ImageLayout::eGeneral
-		                           : vk::ImageLayout::eColorAttachmentOptimal;
+		const auto& view   = target.desc.view_info;
+		const auto  layout = image.binding.is_bound ? vk::ImageLayout::eGeneral
+		                                            : vk::ImageLayout::eColorAttachmentOptimal;
 		image.Transit(layout,
 		              vk::AccessFlagBits2::eColorAttachmentRead |
 		                  vk::AccessFlagBits2::eColorAttachmentWrite,
 		              ImageSubresourceRange {view.base_level, view.level_count, view.base_layer,
 		                                     view.layer_count},
 		              buffer.Handle());
-		state.width      = std::min(state.width, target.extent.width);
-		state.height     = std::min(state.height, target.extent.height);
-		state.num_layers = std::min(state.num_layers, view.layer_count);
-		auto& attachment = state.color_attachments[i];
+		state.width             = std::min(state.width, target.extent.width);
+		state.height            = std::min(state.height, target.extent.height);
+		state.num_layers        = std::min(state.num_layers, view.layer_count);
+		auto& attachment        = state.color_attachments[i];
 		attachment.image_view   = target.image_view;
 		attachment.image_layout = layout;
 		attachment.clear_value  = target.color_clear_value.uint32;
@@ -561,8 +559,7 @@ RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderCo
 		depth.depth_meta_clear_enable =
 		    depth.htile &&
 		    cache.IsMetaCleared(depth.htile_buffer_vaddr, depth.desc.view_info.base_layer);
-		depth.depth_load_clear_enable =
-		    depth.depth_clear_enable || depth.depth_meta_clear_enable;
+		depth.depth_load_clear_enable = depth.depth_clear_enable || depth.depth_meta_clear_enable;
 		if (depth.depth_meta_clear_enable &&
 		    !cache.TouchMeta(depth.htile_buffer_vaddr, depth.desc.view_info.base_layer, false)) {
 			EXIT("failed to consume HTile clear state\n");
@@ -572,12 +569,12 @@ RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderCo
 		if (attachment_samples == 0) {
 			attachment_samples = depth.samples;
 		} else if (attachment_samples != depth.samples) {
-			EXIT("mixed color/depth sample counts are unsupported: %u and %u\n",
-			     attachment_samples, depth.samples);
+			EXIT("mixed color/depth sample counts are unsupported: %u and %u\n", attachment_samples,
+			     depth.samples);
 		}
 		const auto layout = depth_attachment_layout(depth);
 		const auto writes = depth.AttachmentWriteAspects();
-		auto access = vk::AccessFlags2 {vk::AccessFlagBits2::eDepthStencilAttachmentRead};
+		auto       access = vk::AccessFlags2 {vk::AccessFlagBits2::eDepthStencilAttachmentRead};
 		if (writes) {
 			access |= vk::AccessFlagBits2::eDepthStencilAttachmentWrite;
 		}
@@ -586,21 +583,19 @@ RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderCo
 		              ImageSubresourceRange {view.base_level, view.level_count, view.base_layer,
 		                                     view.layer_count},
 		              buffer.Handle());
-		state.width      = std::min(state.width, depth.width);
-		state.height     = std::min(state.height, depth.height);
-		state.num_layers = std::min(state.num_layers, view.layer_count);
-		const auto aspects = ImageViewOps::DepthAspectMask(depth.format);
-		auto& attachment = state.depth_stencil_attachment;
+		state.width               = std::min(state.width, depth.width);
+		state.height              = std::min(state.height, depth.height);
+		state.num_layers          = std::min(state.num_layers, view.layer_count);
+		const auto aspects        = ImageViewOps::DepthAspectMask(depth.format);
+		auto&      attachment     = state.depth_stencil_attachment;
 		attachment.image_view     = depth.image_view;
 		attachment.image_layout   = layout;
 		attachment.clear_value[0] = std::bit_cast<uint32_t>(depth.depth_clear_value);
 		attachment.clear_value[1] = depth.stencil_clear_value;
-		attachment.has_depth =
-		    static_cast<bool>(aspects & vk::ImageAspectFlagBits::eDepth);
-		attachment.depth_clear = depth.depth_load_clear_enable;
-		attachment.has_stencil =
-		    static_cast<bool>(aspects & vk::ImageAspectFlagBits::eStencil);
-		attachment.stencil_clear = depth.stencil_clear_enable;
+		attachment.has_depth      = static_cast<bool>(aspects & vk::ImageAspectFlagBits::eDepth);
+		attachment.depth_clear    = depth.depth_load_clear_enable;
+		attachment.has_stencil    = static_cast<bool>(aspects & vk::ImageAspectFlagBits::eStencil);
+		attachment.stencil_clear  = depth.stencil_clear_enable;
 	}
 	if (attachment_samples == 0 ||
 	    vulkan_sample_count(attachment_samples) == vk::SampleCountFlagBits {}) {
@@ -685,6 +680,85 @@ static uint64_t VertexBufferDescriptorSize(const ShaderVertexInputBuffer& buffer
 	                           : buffer.num_records);
 }
 
+struct VertexBufferRange {
+	uint64_t      base_address  = 0;
+	uint64_t      requested_end = 0;
+	uint64_t      acquired_end  = 0;
+	BufferBinding binding;
+
+	[[nodiscard]] uint64_t RequestedSize() const { return requested_end - base_address; }
+};
+
+static std::vector<BufferBinding> AcquireVertexBuffers(RenderCommandBuffer&         buffer,
+                                                       const ShaderVertexInputInfo& vs_input_info) {
+	// Collect the non-empty guest vertex ranges.
+	std::vector<VertexBufferRange> ranges;
+	ranges.reserve(vs_input_info.buffers_num);
+	for (int i = 0; i < vs_input_info.buffers_num; i++) {
+		const auto& vertex = vs_input_info.buffers[i];
+		const auto  size   = VertexBufferDescriptorSize(vertex);
+		if (size == 0) {
+			continue;
+		}
+		if (vertex.addr == 0 || size > UINT64_MAX - vertex.addr) {
+			EXIT("invalid vertex buffer range: addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n",
+			     vertex.addr, size);
+		}
+		ranges.push_back({vertex.addr, vertex.addr + size});
+	}
+
+	std::ranges::sort(ranges, [](const VertexBufferRange& left, const VertexBufferRange& right) {
+		return left.base_address < right.base_address;
+	});
+
+	// Merge overlapping or touching ranges before acquiring host buffers.
+	std::vector<VertexBufferRange> merged_ranges;
+	merged_ranges.reserve(ranges.size());
+	for (const auto& range: ranges) {
+		if (!merged_ranges.empty() && merged_ranges.back().requested_end >= range.base_address) {
+			merged_ranges.back().requested_end =
+			    std::max(merged_ranges.back().requested_end, range.requested_end);
+			continue;
+		}
+		merged_ranges.push_back(range);
+	}
+
+	auto& cache = buffer.GetContext().GetBufferCache();
+	for (auto& range: merged_ranges) {
+		// PPSA20298
+		const auto size =
+		    Libs::LibKernel::Memory::ClampRangeSize(range.base_address, range.RequestedSize());
+		range.acquired_end = range.base_address + size;
+		range.binding      = cache.ObtainBuffer(buffer, range.base_address, size);
+	}
+
+	// Rebuild slot bindings, offsetting non-empty slots into their acquired merged range.
+	std::vector<BufferBinding> bindings;
+	bindings.reserve(vs_input_info.buffers_num);
+	for (int i = 0; i < vs_input_info.buffers_num; i++) {
+		const auto& vertex = vs_input_info.buffers[i];
+		const auto  size   = VertexBufferDescriptorSize(vertex);
+		if (size == 0) {
+			auto owner = cache.ObtainNullBuffer();
+			bindings.push_back({owner, owner->Handle(), 0});
+			continue;
+		}
+
+		const auto range = std::ranges::find_if(merged_ranges, [&](const VertexBufferRange& value) {
+			return vertex.addr >= value.base_address && vertex.addr < value.acquired_end;
+		});
+		if (range == merged_ranges.end()) {
+			EXIT("vertex buffer address is outside the acquired range: addr=0x%016" PRIx64 "\n",
+			     vertex.addr);
+		}
+
+		auto binding = range->binding;
+		binding.offset += vertex.addr - range->base_address;
+		bindings.push_back(std::move(binding));
+	}
+	return bindings;
+}
+
 static void SetDrawDebugPhase(RenderCommandBuffer& buffer, uint64_t submit_id,
                               const DrawCallInfo& draw, uint32_t phase) {
 	EXIT_IF(draw.name == nullptr);
@@ -736,9 +810,9 @@ static bool GetDrawTopology(const HW::UserConfig& ucfg, bool auto_draw, bool use
 }
 
 bool RenderExecutor::PrepareDrawRenderState(uint64_t submit_id, RenderCommandBuffer& buffer,
-                                        const DrawCallInfo& draw,
-                                        uint32_t render_target_slice_offset,
-                                        bool log_setup_phases, DrawRenderState& state) {
+                                            const DrawCallInfo& draw,
+                                            uint32_t            render_target_slice_offset,
+                                            bool log_setup_phases, DrawRenderState& state) {
 	EXIT_IF(draw.name == nullptr);
 	auto& ctx = buffer.GetRegisters();
 
@@ -823,37 +897,13 @@ static std::vector<BufferBinding> PrepareVertexBuffers(uint64_t
 	(void)submit_id;
 
 	LogDrawPhase(draw.name, "PrepareVertexBuffers");
-	std::vector<BufferBinding> bindings;
-	bindings.reserve(vs_input_info.buffers_num);
-	for (int i = 0; i < vs_input_info.buffers_num; i++) {
-		const auto& b    = vs_input_info.buffers[i];
-		const auto  size = VertexBufferDescriptorSize(b);
-		if (size == 0) {
-			auto owner = buffer.GetContext().GetBufferCache().ObtainNullBuffer();
-			bindings.push_back({owner, owner->Handle(), 0});
-		} else {
-			bindings.push_back(
-			    buffer.GetContext().GetBufferCache().ObtainBuffer(buffer, b.addr, size));
-		}
-	}
-	return bindings;
+	return AcquireVertexBuffers(buffer, vs_input_info);
 }
 
 static void RebindVertexBuffers(RenderCommandBuffer&         buffer,
                                 const ShaderVertexInputInfo& vs_input_info,
                                 std::vector<BufferBinding>&  bindings) {
-	EXIT_IF(bindings.size() != static_cast<size_t>(vs_input_info.buffers_num));
-	for (int i = 0; i < vs_input_info.buffers_num; i++) {
-		const auto& vertex = vs_input_info.buffers[i];
-		const auto  size   = VertexBufferDescriptorSize(vertex);
-		if (size == 0) {
-			auto owner  = buffer.GetContext().GetBufferCache().ObtainNullBuffer();
-			bindings[i] = {owner, owner->Handle(), 0};
-		} else {
-			bindings[i] =
-			    buffer.GetContext().GetBufferCache().ObtainBuffer(buffer, vertex.addr, size);
-		}
-	}
+	bindings = AcquireVertexBuffers(buffer, vs_input_info);
 }
 
 static PreparedIndexBuffer PrepareIndexBuffer(RenderCommandBuffer&         buffer,
@@ -1011,17 +1061,17 @@ static void EmitDrawPrimitives(const HW::UserConfig& ucfg, vk::CommandBuffer vk_
 }
 
 void RenderExecutor::ExecutePreparedDraw(uint64_t submit_id, RenderCommandBuffer& buffer,
-                                     const DrawCallInfo& draw, DrawRenderState& state,
-                                     vk::PrimitiveTopology topology, const DrawEmitInfo& emit,
-                                     const DrawIndexBufferSource& index_source,
-                                     bool log_pipeline_phase, bool set_bind_debug,
-                                     bool set_auto_debug) {
+                                         const DrawCallInfo& draw, DrawRenderState& state,
+                                         vk::PrimitiveTopology topology, const DrawEmitInfo& emit,
+                                         const DrawIndexBufferSource& index_source,
+                                         bool log_pipeline_phase, bool set_bind_debug,
+                                         bool set_auto_debug) {
 	EXIT_IF(draw.name == nullptr);
 	auto& ucfg = buffer.GetUserConfig();
 
 	LogDrawPhase(draw.name, "PrepareBindings");
-	auto bindings = PrepareGraphicsBindings(buffer, state.vs_input_info.stage,
-	                                        state.ps_input_info.stage, state.ps_active);
+	auto bindings        = PrepareGraphicsBindings(buffer, state.vs_input_info.stage,
+	                                               state.ps_input_info.stage, state.ps_active);
 	auto vertex_bindings = PrepareVertexBuffers(submit_id, buffer, draw, state.vs_input_info);
 	auto index_binding   = PrepareIndexBuffer(buffer, index_source);
 	RebindVertexBuffers(buffer, state.vs_input_info, vertex_bindings);
@@ -1094,10 +1144,10 @@ void RenderExecutor::ExecutePreparedDraw(uint64_t submit_id, RenderCommandBuffer
 }
 
 void RenderExecutor::DrawIndex(uint64_t submit_id, RenderCommandBuffer& buffer,
-                           uint32_t index_type_and_size, uint32_t index_count,
-                           const void* index_addr, uint32_t flags, uint32_t type,
-                           uint32_t instance_count, uint32_t render_target_slice_offset,
-                           int32_t vertex_offset_add, uint32_t first_instance) {
+                               uint32_t index_type_and_size, uint32_t index_count,
+                               const void* index_addr, uint32_t flags, uint32_t type,
+                               uint32_t instance_count, uint32_t render_target_slice_offset,
+                               int32_t vertex_offset_add, uint32_t first_instance) {
 	KYTY_PROFILER_FUNCTION();
 
 	EXIT_IF(buffer.IsInvalid());
@@ -1228,11 +1278,10 @@ void RenderExecutor::DrawIndex(uint64_t submit_id, RenderCommandBuffer& buffer,
 }
 
 // NOLINTNEXTLINE(readability-function-cognitive-complexity)
-void RenderExecutor::DrawAuto(uint64_t submit_id, RenderCommandBuffer& buffer,
-                              uint32_t index_count,
-                          uint32_t flags, uint32_t render_target_slice_offset,
-                          uint32_t instance_count, uint32_t first_vertex,
-                          uint32_t first_instance) {
+void RenderExecutor::DrawAuto(uint64_t submit_id, RenderCommandBuffer& buffer, uint32_t index_count,
+                              uint32_t flags, uint32_t render_target_slice_offset,
+                              uint32_t instance_count, uint32_t first_vertex,
+                              uint32_t first_instance) {
 	KYTY_PROFILER_FUNCTION();
 
 	EXIT_IF(buffer.IsInvalid());
@@ -1290,7 +1339,8 @@ void RenderExecutor::DrawAuto(uint64_t submit_id, RenderCommandBuffer& buffer,
 	                         instance_count,  first_instance};
 
 	DrawRenderState state {};
-	if (!PrepareDrawRenderState(submit_id, buffer, draw, render_target_slice_offset, false, state)) {
+	if (!PrepareDrawRenderState(submit_id, buffer, draw, render_target_slice_offset, false,
+	                            state)) {
 		ResetBindings();
 		return;
 	}
@@ -1340,7 +1390,7 @@ void RenderExecutor::DrawAuto(uint64_t submit_id, RenderCommandBuffer& buffer,
 }
 
 bool RenderExecutor::ResolveColorTargets(uint64_t submit_id, RenderCommandBuffer& buffer,
-                                     uint32_t render_target_slice_offset) {
+                                         uint32_t render_target_slice_offset) {
 	const auto& hw = buffer.GetRegisters();
 	if (hw.GetColorControl().mode != 3) {
 		return false;
@@ -1369,8 +1419,7 @@ bool RenderExecutor::ResolveColorTargets(uint64_t submit_id, RenderCommandBuffer
 	cache.MarkGpuWritten(dst.image_id);
 	auto& source      = cache.GetImage(src.image_id);
 	auto& destination = cache.GetImage(dst.image_id);
-	destination.Resolve(source,
-	                    {src.base_mip_level, 1, src.base_array_layer, 1},
+	destination.Resolve(source, {src.base_mip_level, 1, src.base_array_layer, 1},
 	                    {dst.base_mip_level, 1, dst.base_array_layer, 1});
 	return true;
 }

src/graphics/shader/recompiler/decompiler/MemoryOps.cpp

@@ -421,13 +421,13 @@ bool DecodeDs(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
 	const uint32_t data0   = (word1 >> 8u) & 0xffu;
 	const uint32_t addr    = word1 & 0xffu;
 
-	inst.pc         = pc;
-	inst.word       = word0;
-	inst.word_count = 2;
-	inst.offset     = offset0 | (offset1 << 8u);
-	inst.gds        = ((word0 >> 17u) & 1u) != 0u;
-	inst.family     = Family::DS;
-	inst.opcode_id  = opcode;
+	inst.pc          = pc;
+	inst.word        = word0;
+	inst.word_count  = 2;
+	inst.offset      = offset0 | (offset1 << 8u);
+	inst.gds         = ((word0 >> 17u) & 1u) != 0u;
+	inst.family      = Family::DS;
+	inst.opcode_id   = opcode;
 	const auto* info = LookupMemoryOpcode(DS_OPS, static_cast<uint32_t>(std::size(DS_OPS)), opcode);
 	ApplyMemoryInfo(inst, info);
 	SetRawWords(inst, code, word_index, 2);
@@ -442,11 +442,6 @@ bool DecodeDs(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
 	     inst.opcode == Opcode::DsReadAddtidB32)) {
 		SetUnsupported(inst, Family::DS, opcode, "DS swizzle/addtid is available only for LDS");
 	}
-	if (inst.gds && (inst.opcode == Opcode::DsAppend || inst.opcode == Opcode::DsConsume) &&
-	    inst.offset != 0u) {
-		SetUnsupported(inst, Family::DS, opcode,
-		               "GDS append/consume requires a zero instruction offset");
-	}
 	if (inst.opcode == Opcode::DsWriteAddtidB32 && data1 != 0u) {
 		SetUnsupported(inst, Family::DS, opcode,
 		               "DS write addtid data1 operand is not implemented");

src/graphics/shader/recompiler/emitter/SpirvEmitter.cpp

@@ -1,7 +1,7 @@
 #include "graphics/shader/recompiler/emitter/SpirvEmitter.h"
 
-#include "graphics/shader/recompiler/ir/SrtWalker.h"
 #include "graphics/shader/recompiler/emitter/spirvEmitterInternal.h"
+#include "graphics/shader/recompiler/ir/SrtWalker.h"
 
 #include <algorithm>
 #include <array>
@@ -160,8 +160,7 @@ bool ValidateInstructionContract(const IR::Instruction& inst, std::string* error
 	                            inst.dst.kind != IR::OperandKind::Null)) ||
 	    ((inst.op == IR::Opcode::DsAppend || inst.op == IR::Opcode::DsConsume) &&
 	     (!ds_kind || !ds_resource || inst.src_count != 1 ||
-	      inst.dst.kind != IR::OperandKind::Register ||
-	      (kind == IR::ResourceKind::Gds && inst.memory.offset != 0))) ||
+	      inst.dst.kind != IR::OperandKind::Register)) ||
 	    ((inst.op == IR::Opcode::DsMinF32 || inst.op == IR::Opcode::DsMaxF32) &&
 	     (!ds_kind || !ds_resource || inst.src_count != 3 ||
 	      inst.dst.kind != IR::OperandKind::Null)) ||

src/graphics/shader/recompiler/emitter/spirvEmitterControlFlow.cpp

@@ -1025,15 +1025,47 @@ void EmitDispatcherSwitch(EmitterState& state, const IR::Program& program) {
 	EmitDispatcherExit(state);
 }
 
+size_t BufferLoadGroupSize(const IR::BasicBlock& block, size_t first_index) {
+	const auto& first = block.instructions[first_index];
+	if (first.op != IR::Opcode::BufferLoadDword || first.memory.component_index != 0u ||
+	    first.memory.component_count <= 1u) {
+		return 1u;
+	}
+
+	size_t count = 1u;
+	while (first_index + count < block.instructions.size() &&
+	       count < first.memory.component_count) {
+		const auto& next = block.instructions[first_index + count];
+		if (next.op != IR::Opcode::BufferLoadDword || next.pc != first.pc ||
+		    next.memory.component_index != count ||
+		    next.memory.component_count != first.memory.component_count) {
+			break;
+		}
+		count++;
+	}
+	return count;
+}
+
+void EmitBlockInstructions(EmitterState& state, const IR::BasicBlock& block) {
+	for (size_t i = 0; i < block.instructions.size();) {
+		const auto count = BufferLoadGroupSize(block, i);
+		if (count > 1u) {
+			EmitBufferLoadDwordGroup(state, block.instructions.data() + i,
+			                         static_cast<uint32_t>(count));
+		} else {
+			EmitInstruction(state, block.instructions[i]);
+		}
+		i += count;
+	}
+}
+
 void EmitDispatcherBlocks(EmitterState& state, const IR::Program& program) {
 	for (const auto& block: program.blocks) {
 		if (block.id >= state.reachable_blocks.size() || !state.reachable_blocks[block.id]) {
 			continue;
 		}
 		state.builder.AddFunction({OpLabel, BlockLabel(state, block.id)});
-		for (const auto& inst: block.instructions) {
-			EmitInstruction(state, inst);
-		}
+		EmitBlockInstructions(state, block);
 		EmitDispatcherTerminator(state, block.terminator);
 	}
 }
@@ -1083,9 +1115,7 @@ void EmitFunction(EmitterState& state, const IR::Program& program) {
 			continue;
 		}
 		state.builder.AddFunction({OpLabel, BlockLabel(state, block.id)});
-		for (const auto& inst: block.instructions) {
-			EmitInstruction(state, inst);
-		}
+		EmitBlockInstructions(state, block);
 		EmitTerminator(state, block.terminator);
 	}
 

src/graphics/shader/recompiler/emitter/spirvEmitterImageHelpers.cpp

@@ -2,6 +2,66 @@
 #include "graphics/shader/recompiler/emitter/spirvEmitterInternal.h"
 
 namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
+namespace {
+
+uint32_t EmitCubeAxisF32(EmitterState& state, uint32_t value) {
+	const auto normalized = state.builder.AllocateId();
+	state.builder.AddFunction(
+	    {OpFSub, state.float_type, normalized, value, ConstantF32(state, 0x3f800000u)});
+	return normalized;
+}
+
+uint32_t EmitCubeLayerF32(EmitterState& state, uint32_t face_id) {
+	// Sampled RDNA2 cubemaps encode face_id as slice * 8 + face. The native
+	// 2D-array view stores six contiguous faces per slice, so remove the two
+	// reserved face IDs from every preceding slice.
+	const auto guest_layer = state.builder.AllocateId();
+	const auto slice       = state.builder.AllocateId();
+	const auto padding     = state.builder.AllocateId();
+	const auto host_layer  = state.builder.AllocateId();
+	const auto result      = state.builder.AllocateId();
+	state.builder.AddFunction({OpConvertFToU, state.uint_type, guest_layer, face_id});
+	state.builder.AddFunction(
+	    {OpShiftRightLogical, state.uint_type, slice, guest_layer, ConstantU32(state, 3)});
+	state.builder.AddFunction(
+	    {OpShiftLeftLogical, state.uint_type, padding, slice, ConstantU32(state, 1)});
+	state.builder.AddFunction({OpISub, state.uint_type, host_layer, guest_layer, padding});
+	state.builder.AddFunction({OpConvertUToF, state.float_type, result, host_layer});
+	return result;
+}
+
+uint32_t EmitImageCoordF32Impl(EmitterState& state, const IR::Instruction& inst,
+                               const IR::Operand& address, uint32_t first_component,
+                               uint32_t components) {
+	auto x = EmitImageAddressFloatLoad(state, inst, address, first_component);
+	if (components == 1u) {
+		return x;
+	}
+	auto y = inst.memory.image_address_components > first_component + 1u
+	             ? EmitImageAddressFloatLoad(state, inst, address, first_component + 1u)
+	             : EmitZeroF32(state);
+	if (inst.memory.image_cube) {
+		// RDNA2 sampled cubemap S/T coordinates are biased by +1 relative to
+		// normalized 2D-array coordinates.
+		x = EmitCubeAxisF32(state, x);
+		y = EmitCubeAxisF32(state, y);
+	}
+	const auto coord = state.builder.AllocateId();
+	if (components == 3u) {
+		auto z = inst.memory.image_address_components > first_component + 2u
+		             ? EmitImageAddressFloatLoad(state, inst, address, first_component + 2u)
+		             : EmitZeroF32(state);
+		if (inst.memory.image_cube) {
+			z = EmitCubeLayerF32(state, z);
+		}
+		state.builder.AddFunction({OpCompositeConstruct, state.vec3_float_type, coord, x, y, z});
+	} else {
+		state.builder.AddFunction({OpCompositeConstruct, state.vec2_float_type, coord, x, y});
+	}
+	return coord;
+}
+
+} // namespace
 
 bool HasImageSampleFlag(const IR::Instruction& inst, uint32_t flag) {
 	return (inst.memory.image_sample_flags & flag) != 0;
@@ -21,7 +81,7 @@ ImageSampleLayout MakeImageSampleLayout(const IR::Instruction& inst, ImageViewKi
 	}
 	if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagDerivative)) {
 		const auto components = ImageViewSpatialComponents(view);
-		layout.grad_x = cursor;
+		layout.grad_x         = cursor;
 		cursor += components;
 		layout.grad_y = cursor;
 		cursor += components;
@@ -36,24 +96,8 @@ ImageSampleLayout MakeImageSampleLayout(const IR::Instruction& inst, ImageViewKi
 
 uint32_t EmitImageCoordF32(EmitterState& state, const IR::Instruction& inst,
                            const ImageSampleLayout& layout, ImageViewKind view) {
-	const auto x          = EmitImageAddressFloatLoad(state, inst, inst.src[0], layout.coord);
-	const auto components = ImageViewCoordinateComponents(view);
-	if (components == 1u) {
-		return x;
-	}
-	const auto y = inst.memory.image_address_components > layout.coord + 1u
-	                       ? EmitImageAddressFloatLoad(state, inst, inst.src[0], layout.coord + 1u)
-	                       : EmitZeroF32(state);
-	const auto coord = state.builder.AllocateId();
-	if (components == 3u) {
-		const auto z = inst.memory.image_address_components > layout.coord + 2u
-		                   ? EmitImageAddressFloatLoad(state, inst, inst.src[0], layout.coord + 2u)
-		                   : EmitZeroF32(state);
-		state.builder.AddFunction({OpCompositeConstruct, state.vec3_float_type, coord, x, y, z});
-	} else {
-		state.builder.AddFunction({OpCompositeConstruct, state.vec2_float_type, coord, x, y});
-	}
-	return coord;
+	return EmitImageCoordF32Impl(state, inst, inst.src[0], layout.coord,
+	                             ImageViewCoordinateComponents(view));
 }
 
 uint32_t EmitImageLodF32(EmitterState& state, const IR::Instruction& inst,
@@ -95,10 +139,10 @@ uint32_t EmitImageGradientF32(EmitterState& state, const IR::Instruction& inst,
 	                   : EmitZeroF32(state);
 	const auto grad = state.builder.AllocateId();
 	if (components == 3u) {
-		const auto z = inst.memory.image_address_components > first_component + 2u
-		                   ? EmitImageAddressFloatLoad(state, inst, inst.src[0],
-		                                               first_component + 2u)
-		                   : EmitZeroF32(state);
+		const auto z =
+		    inst.memory.image_address_components > first_component + 2u
+		        ? EmitImageAddressFloatLoad(state, inst, inst.src[0], first_component + 2u)
+		        : EmitZeroF32(state);
 		state.builder.AddFunction({OpCompositeConstruct, state.vec3_float_type, grad, x, y, z});
 	} else {
 		state.builder.AddFunction({OpCompositeConstruct, state.vec2_float_type, grad, x, y});
@@ -120,8 +164,7 @@ uint32_t EmitImagePackedOffsetI32(EmitterState& state, const IR::Instruction& in
 			state.builder.AddFunction(
 			    {OpCompositeConstruct, state.vec3_int_type, ret, zero, zero, zero});
 		} else {
-			state.builder.AddFunction(
-			    {OpCompositeConstruct, state.vec2_int_type, ret, zero, zero});
+			state.builder.AddFunction({OpCompositeConstruct, state.vec2_int_type, ret, zero, zero});
 		}
 		return ret;
 	}
@@ -131,18 +174,18 @@ uint32_t EmitImagePackedOffsetI32(EmitterState& state, const IR::Instruction& in
 	const auto offset_x    = state.builder.AllocateId();
 	state.builder.AddFunction({OpBitcast, state.int_type, packed_i32, packed_bits});
 	state.builder.AddFunction({OpBitFieldSExtract, state.int_type, offset_x, packed_i32,
-	                            ConstantI32(state, 0), ConstantI32(state, 6)});
+	                           ConstantI32(state, 0), ConstantI32(state, 6)});
 	if (components == 1u) {
 		return offset_x;
 	}
 	const auto offset_y = state.builder.AllocateId();
 	const auto offset   = state.builder.AllocateId();
 	state.builder.AddFunction({OpBitFieldSExtract, state.int_type, offset_y, packed_i32,
-	                            ConstantI32(state, 8), ConstantI32(state, 6)});
+	                           ConstantI32(state, 8), ConstantI32(state, 6)});
 	if (components == 3u) {
 		const auto offset_z = state.builder.AllocateId();
 		state.builder.AddFunction({OpBitFieldSExtract, state.int_type, offset_z, packed_i32,
-		                            ConstantI32(state, 16), ConstantI32(state, 6)});
+		                           ConstantI32(state, 16), ConstantI32(state, 6)});
 		state.builder.AddFunction(
 		    {OpCompositeConstruct, state.vec3_int_type, offset, offset_x, offset_y, offset_z});
 	} else {
@@ -158,7 +201,7 @@ uint32_t EmitImageCoordU32(EmitterState& state, const IR::Instruction& inst, Ima
 	if (components == 1u) {
 		return x;
 	}
-	const auto y = inst.memory.image_address_components > 1u
+	const auto y     = inst.memory.image_address_components > 1u
 	                       ? EmitImageAddressValueLoad(state, inst, inst.src[1], 1)
 	                       : ConstantU32(state, 0);
 	const auto coord = state.builder.AllocateId();
@@ -180,7 +223,7 @@ uint32_t EmitImageLoadCoordU32(EmitterState& state, const IR::Instruction& inst,
 	if (components == 1u) {
 		return x;
 	}
-	const auto y = inst.memory.image_address_components > 1u
+	const auto y     = inst.memory.image_address_components > 1u
 	                       ? EmitImageAddressValueLoad(state, inst, inst.src[0], 1)
 	                       : ConstantU32(state, 0);
 	const auto coord = state.builder.AllocateId();
@@ -209,24 +252,8 @@ uint32_t EmitImageMipLodU32(EmitterState& state, const IR::Instruction& inst,
 
 uint32_t EmitImageQueryCoordF32(EmitterState& state, const IR::Instruction& inst,
                                 ImageViewKind view) {
-	const auto x          = EmitImageAddressFloatLoad(state, inst, inst.src[0], 0);
-	const auto components = ImageViewCoordinateComponents(view);
-	if (components == 1u) {
-		return x;
-	}
-	const auto y = inst.memory.image_address_components > 1u
-	                       ? EmitImageAddressFloatLoad(state, inst, inst.src[0], 1)
-	                       : EmitZeroF32(state);
-	const auto coord = state.builder.AllocateId();
-	if (components == 3u) {
-		const auto z = inst.memory.image_address_components > 2u
-		                   ? EmitImageAddressFloatLoad(state, inst, inst.src[0], 2)
-		                   : EmitZeroF32(state);
-		state.builder.AddFunction({OpCompositeConstruct, state.vec3_float_type, coord, x, y, z});
-	} else {
-		state.builder.AddFunction({OpCompositeConstruct, state.vec2_float_type, coord, x, y});
-	}
-	return coord;
+	// OpImageQueryLod takes only the spatial coordinates, even for arrayed images.
+	return EmitImageCoordF32Impl(state, inst, inst.src[0], 0, ImageViewSpatialComponents(view));
 }
 
 uint32_t DmaskComponentIndex(uint32_t dmask, uint32_t component) {

src/graphics/shader/recompiler/emitter/spirvEmitterInternal.h

@@ -3,11 +3,11 @@
 
 #include "common/common.h"
 #include "common/stringUtils.h"
-#include "graphics/shader/recompiler/ir/BindingLayout.h"
 #include "graphics/shader/recompiler/BufferFormat.h"
+#include "graphics/shader/recompiler/emitter/SpirvBuilder.h"
+#include "graphics/shader/recompiler/ir/BindingLayout.h"
 #include "graphics/shader/recompiler/ir/ResourceMaterialization.h"
 #include "graphics/shader/recompiler/ir/ShaderIR.h"
-#include "graphics/shader/recompiler/emitter/SpirvBuilder.h"
 
 #include <algorithm>
 #include <array>
@@ -312,117 +312,117 @@ struct EmitterState {
 	EmitterState(const IR::Program& program_, const IR::ResourceSnapshot& resources_)
 	    : program(program_), resources(resources_) {}
 
-	Builder                                builder;
-	const IR::Program&                     program;
-	const IR::ResourceSnapshot&            resources;
-	const ShaderVertexInputInfo*           vertex_input_info            = nullptr;
-	const ShaderPixelInputInfo*            pixel_input_info             = nullptr;
-	const ShaderComputeInputInfo*          compute_input_info           = nullptr;
-	ShaderType                             stage                        = ShaderType::Unknown;
-	uint32_t                               wave_size                    = 64;
-	bool                                   exact_subgroup_operations    = false;
-	bool                                   per_invocation_masks         = false;
-	uint32_t                               void_type                    = 0;
-	uint32_t                               bool_type                    = 0;
-	uint32_t                               uint_type                    = 0;
-	uint32_t                               uint_pair_type               = 0;
-	uint32_t                               int_pair_type                = 0;
-	uint32_t                               int_type                     = 0;
-	uint32_t                               float_type                   = 0;
-	uint32_t                               vec2_uint_type               = 0;
-	uint32_t                               vec3_uint_type               = 0;
-	uint32_t                               vec4_uint_type               = 0;
-	uint32_t                               vec2_int_type                = 0;
-	uint32_t                               vec3_int_type                = 0;
-	uint32_t                               vec4_int_type                = 0;
-	uint32_t                               vec2_float_type              = 0;
-	uint32_t                               vec3_float_type              = 0;
-	uint32_t                               vec4_float_type              = 0;
-	uint32_t                               ptr_func_uint                = 0;
-	uint32_t                               ptr_input_float              = 0;
-	uint32_t                               ptr_input_bool               = 0;
-	uint32_t                               ptr_input_int                = 0;
-	uint32_t                               ptr_input_uint               = 0;
-	uint32_t                               ptr_input_vec2_float         = 0;
-	uint32_t                               ptr_input_vec3_float         = 0;
-	uint32_t                               ptr_input_vec2_int           = 0;
-	uint32_t                               ptr_input_vec3_int           = 0;
-	uint32_t                               ptr_input_vec4_int           = 0;
-	uint32_t                               ptr_input_vec2_uint          = 0;
-	uint32_t                               ptr_input_vec3_uint          = 0;
-	uint32_t                               ptr_input_vec4_uint          = 0;
-	uint32_t                               ptr_input_vec4_float         = 0;
-	uint32_t                               sample_mask_array_type       = 0;
-	uint32_t                               ptr_output_int               = 0;
-	uint32_t                               ptr_output_sample_mask_array = 0;
-	uint32_t                               ptr_output_float             = 0;
-	uint32_t                               ptr_output_vec4_float        = 0;
-	uint32_t                               per_vertex_type              = 0;
-	uint32_t                               ptr_output_per_vertex        = 0;
-	uint32_t                               storage_runtime_array_type   = 0;
-	uint32_t                               storage_buffer_type          = 0;
-	uint32_t                               ptr_storage_buffer           = 0;
-	uint32_t                               ptr_storage_buffer_uint      = 0;
-	uint32_t                               storage_buffer_array_type    = 0;
-	uint32_t                               ptr_storage_buffer_array     = 0;
-	uint32_t                               storage_buffer_variable      = 0;
+	Builder                                          builder;
+	const IR::Program&                               program;
+	const IR::ResourceSnapshot&                      resources;
+	const ShaderVertexInputInfo*                     vertex_input_info  = nullptr;
+	const ShaderPixelInputInfo*                      pixel_input_info   = nullptr;
+	const ShaderComputeInputInfo*                    compute_input_info = nullptr;
+	ShaderType                                       stage              = ShaderType::Unknown;
+	uint32_t                                         wave_size          = 64;
+	bool                                             exact_subgroup_operations    = false;
+	bool                                             per_invocation_masks         = false;
+	uint32_t                                         void_type                    = 0;
+	uint32_t                                         bool_type                    = 0;
+	uint32_t                                         uint_type                    = 0;
+	uint32_t                                         uint_pair_type               = 0;
+	uint32_t                                         int_pair_type                = 0;
+	uint32_t                                         int_type                     = 0;
+	uint32_t                                         float_type                   = 0;
+	uint32_t                                         vec2_uint_type               = 0;
+	uint32_t                                         vec3_uint_type               = 0;
+	uint32_t                                         vec4_uint_type               = 0;
+	uint32_t                                         vec2_int_type                = 0;
+	uint32_t                                         vec3_int_type                = 0;
+	uint32_t                                         vec4_int_type                = 0;
+	uint32_t                                         vec2_float_type              = 0;
+	uint32_t                                         vec3_float_type              = 0;
+	uint32_t                                         vec4_float_type              = 0;
+	uint32_t                                         ptr_func_uint                = 0;
+	uint32_t                                         ptr_input_float              = 0;
+	uint32_t                                         ptr_input_bool               = 0;
+	uint32_t                                         ptr_input_int                = 0;
+	uint32_t                                         ptr_input_uint               = 0;
+	uint32_t                                         ptr_input_vec2_float         = 0;
+	uint32_t                                         ptr_input_vec3_float         = 0;
+	uint32_t                                         ptr_input_vec2_int           = 0;
+	uint32_t                                         ptr_input_vec3_int           = 0;
+	uint32_t                                         ptr_input_vec4_int           = 0;
+	uint32_t                                         ptr_input_vec2_uint          = 0;
+	uint32_t                                         ptr_input_vec3_uint          = 0;
+	uint32_t                                         ptr_input_vec4_uint          = 0;
+	uint32_t                                         ptr_input_vec4_float         = 0;
+	uint32_t                                         sample_mask_array_type       = 0;
+	uint32_t                                         ptr_output_int               = 0;
+	uint32_t                                         ptr_output_sample_mask_array = 0;
+	uint32_t                                         ptr_output_float             = 0;
+	uint32_t                                         ptr_output_vec4_float        = 0;
+	uint32_t                                         per_vertex_type              = 0;
+	uint32_t                                         ptr_output_per_vertex        = 0;
+	uint32_t                                         storage_runtime_array_type   = 0;
+	uint32_t                                         storage_buffer_type          = 0;
+	uint32_t                                         ptr_storage_buffer           = 0;
+	uint32_t                                         ptr_storage_buffer_uint      = 0;
+	uint32_t                                         storage_buffer_array_type    = 0;
+	uint32_t                                         ptr_storage_buffer_array     = 0;
+	uint32_t                                         storage_buffer_variable      = 0;
 	std::array<uint32_t, IR::ShaderInfo::MaxBuffers> storage_buffer_offsets {};
-	uint32_t                               address_memory_array_type    = 0;
-	uint32_t                               ptr_address_memory_array     = 0;
-	uint32_t                               address_memory_variable      = 0;
-	uint32_t                               gds_variable                 = 0;
-	uint32_t                               push_constant_array_type     = 0;
-	uint32_t                               push_constant_block_type     = 0;
-	uint32_t                               ptr_push_constant_block      = 0;
-	uint32_t                               ptr_push_constant_uint       = 0;
-	uint32_t                               push_constant_variable       = 0;
-	uint32_t                               vsharp_storage_variable      = 0;
-	uint32_t                               flattened_srt_variable       = 0;
-	uint32_t                               lds_array_type               = 0;
-	uint32_t                               ptr_workgroup_array          = 0;
-	uint32_t                               ptr_workgroup_uint           = 0;
-	uint32_t                               lds_variable                 = 0;
-	std::array<SampledImageDescriptors, 10> sampled_images;
-	std::array<StorageImageDescriptors, 10> storage_images;
-	uint32_t                               sampler_type                                  = 0;
-	uint32_t                               sampler_array_type                            = 0;
-	uint32_t                               ptr_uniform_sampler                           = 0;
-	uint32_t                               ptr_uniform_sampler_array                     = 0;
-	uint32_t                               sampler_variable                              = 0;
-	uint32_t                               ptr_image_uint                                = 0;
-	uint32_t                               func_type                                     = 0;
-	uint32_t                               main_func                                     = 0;
-	uint32_t                               entry_label                                   = 0;
-	uint32_t                               pixel_valid_mask_variable                     = 0;
-	bool                                   dispatcher_fallback                           = false;
-	uint32_t                               dispatch_pc_variable                          = 0;
-	uint32_t                               dispatch_header_label                         = 0;
-	uint32_t                               dispatch_select_label                         = 0;
-	uint32_t                               dispatch_default_label                        = 0;
-	uint32_t                               dispatch_after_switch_label                   = 0;
-	uint32_t                               dispatch_continue_label                       = 0;
-	uint32_t                               dispatch_merge_label                          = 0;
-	uint32_t                               glsl_std450                                   = 0;
-	uint32_t                               subgroup_local_invocation_id_variable         = 0;
-	uint32_t                               per_vertex_variable                           = 0;
-	uint32_t                               depth_variable                                = 0;
-	uint32_t                               sample_mask_variable                          = 0;
-	bool                                   needs_subgroup_ballot                         = false;
-	bool                                   needs_subgroup_shuffle                        = false;
-	bool                                   needs_subgroup_local_invocation_id            = false;
-	bool                                   needs_compute_derivatives                     = false;
-	bool                                   needs_image_gather_extended                   = false;
-	bool                                   needs_function_lds                            = false;
-	bool                                   needs_pixel_valid_mask                        = false;
-	std::vector<RegisterBinding>           registers;
-	std::vector<InputBinding>              inputs;
-	std::vector<OutputBinding>             outputs;
-	std::vector<uint32_t>                  interface_variables;
-	std::vector<bool>                      reachable_blocks;
-	std::map<uint32_t, uint32_t>           block_labels;
-	std::map<uint32_t, uint32_t>           constants;
-	std::map<uint32_t, uint32_t>           signed_constants;
-	std::map<uint32_t, uint32_t>           float_constants;
+	uint32_t                                         address_memory_array_type = 0;
+	uint32_t                                         ptr_address_memory_array  = 0;
+	uint32_t                                         address_memory_variable   = 0;
+	uint32_t                                         gds_variable              = 0;
+	uint32_t                                         push_constant_array_type  = 0;
+	uint32_t                                         push_constant_block_type  = 0;
+	uint32_t                                         ptr_push_constant_block   = 0;
+	uint32_t                                         ptr_push_constant_uint    = 0;
+	uint32_t                                         push_constant_variable    = 0;
+	uint32_t                                         vsharp_storage_variable   = 0;
+	uint32_t                                         flattened_srt_variable    = 0;
+	uint32_t                                         lds_array_type            = 0;
+	uint32_t                                         ptr_workgroup_array       = 0;
+	uint32_t                                         ptr_workgroup_uint        = 0;
+	uint32_t                                         lds_variable              = 0;
+	std::array<SampledImageDescriptors, 10>          sampled_images;
+	std::array<StorageImageDescriptors, 10>          storage_images;
+	uint32_t                                         sampler_type                          = 0;
+	uint32_t                                         sampler_array_type                    = 0;
+	uint32_t                                         ptr_uniform_sampler                   = 0;
+	uint32_t                                         ptr_uniform_sampler_array             = 0;
+	uint32_t                                         sampler_variable                      = 0;
+	uint32_t                                         ptr_image_uint                        = 0;
+	uint32_t                                         func_type                             = 0;
+	uint32_t                                         main_func                             = 0;
+	uint32_t                                         entry_label                           = 0;
+	uint32_t                                         pixel_valid_mask_variable             = 0;
+	bool                                             dispatcher_fallback                   = false;
+	uint32_t                                         dispatch_pc_variable                  = 0;
+	uint32_t                                         dispatch_header_label                 = 0;
+	uint32_t                                         dispatch_select_label                 = 0;
+	uint32_t                                         dispatch_default_label                = 0;
+	uint32_t                                         dispatch_after_switch_label           = 0;
+	uint32_t                                         dispatch_continue_label               = 0;
+	uint32_t                                         dispatch_merge_label                  = 0;
+	uint32_t                                         glsl_std450                           = 0;
+	uint32_t                                         subgroup_local_invocation_id_variable = 0;
+	uint32_t                                         per_vertex_variable                   = 0;
+	uint32_t                                         depth_variable                        = 0;
+	uint32_t                                         sample_mask_variable                  = 0;
+	bool                                             needs_subgroup_ballot                 = false;
+	bool                                             needs_subgroup_shuffle                = false;
+	bool                                             needs_subgroup_local_invocation_id    = false;
+	bool                                             needs_compute_derivatives             = false;
+	bool                                             needs_image_gather_extended           = false;
+	bool                                             needs_function_lds                    = false;
+	bool                                             needs_pixel_valid_mask                = false;
+	std::vector<RegisterBinding>                     registers;
+	std::vector<InputBinding>                        inputs;
+	std::vector<OutputBinding>                       outputs;
+	std::vector<uint32_t>                            interface_variables;
+	std::vector<bool>                                reachable_blocks;
+	std::map<uint32_t, uint32_t>                     block_labels;
+	std::map<uint32_t, uint32_t>                     constants;
+	std::map<uint32_t, uint32_t>                     signed_constants;
+	std::map<uint32_t, uint32_t>                     float_constants;
 };
 
 constexpr uint32_t PsInputOffsetMask = 0x0000001fu;
@@ -990,11 +990,6 @@ uint32_t NormalizeFormatComponent(EmitterState& state, const Format::BufferForma
 uint32_t UnpackTBufferFormat(EmitterState& state, const IR::Instruction& inst,
                              const Format::BufferFormatInfo& info);
 
-bool EmitTypedTBufferLoad(EmitterState& state, const IR::Instruction& inst,
-                          const Format::BufferFormatInfo& info);
-
-bool EmitFormattedBufferLoad(EmitterState& state, const IR::Instruction& inst);
-
 uint32_t FormattedBufferDwordStoreComponentCount(Prospero::BufferFormat format,
                                                  uint32_t               opcode_components);
 
@@ -1020,6 +1015,9 @@ void EmitBufferLoadSshort(EmitterState& state, const IR::Instruction& inst);
 
 void EmitBufferLoadDword(EmitterState& state, const IR::Instruction& inst);
 
+void EmitBufferLoadDwordGroup(EmitterState& state, const IR::Instruction* instructions,
+                              uint32_t count);
+
 void EmitBufferStoreDword(EmitterState& state, const IR::Instruction& inst);
 
 void EmitFlatLoadUbyte(EmitterState& state, const IR::Instruction& inst);

src/graphics/shader/recompiler/emitter/spirvEmitterMemory.cpp

@@ -34,16 +34,15 @@ uint32_t EmitDppWriteActiveBool(EmitterState& state, const IR::Operand& dst) {
 	    {OpShiftLeftLogical, state.uint_type, bank_bit, ConstantU32(state, 1), bank});
 	state.builder.AddFunction(
 	    {OpShiftLeftLogical, state.uint_type, row_bit, ConstantU32(state, 1), row});
-	state.builder.AddFunction({OpBitwiseAnd, state.uint_type, bank_hit,
-	                            ConstantU32(state, dst.dpp_bank_mask), bank_bit});
+	state.builder.AddFunction(
+	    {OpBitwiseAnd, state.uint_type, bank_hit, ConstantU32(state, dst.dpp_bank_mask), bank_bit});
 	state.builder.AddFunction(
 	    {OpBitwiseAnd, state.uint_type, row_hit, ConstantU32(state, dst.dpp_row_mask), row_bit});
 	state.builder.AddFunction(
 	    {OpINotEqual, state.bool_type, bank_active, bank_hit, ConstantU32(state, 0)});
 	state.builder.AddFunction(
 	    {OpINotEqual, state.bool_type, row_active, row_hit, ConstantU32(state, 0)});
-	state.builder.AddFunction(
-	    {OpLogicalAnd, state.bool_type, dpp_active, bank_active, row_active});
+	state.builder.AddFunction({OpLogicalAnd, state.bool_type, dpp_active, bank_active, row_active});
 	uint32_t write_active = dpp_active;
 	if (!dst.dpp_bound_ctrl) {
 		const auto target  = EmitDppTargetLane(state, dst.dpp_ctrl);
@@ -102,7 +101,7 @@ void EmitStoreU32(EmitterState& state, const IR::Operand& dst, uint32_t value) {
 			const auto selected  = state.builder.AllocateId();
 			state.builder.AddFunction({OpLoad, state.uint_type, old_value, pointer});
 			state.builder.AddFunction({OpSelect, state.uint_type, selected,
-			                            EmitDppWriteActiveBool(state, dst), wave_value, old_value});
+			                           EmitDppWriteActiveBool(state, dst), wave_value, old_value});
 			state.builder.AddFunction({OpStore, pointer, selected});
 			return;
 		}
@@ -131,8 +130,7 @@ uint32_t EmitNotEqualZeroBool(EmitterState& state, uint32_t value) {
 uint32_t EmitSelectU32Value(EmitterState& state, uint32_t condition, uint32_t true_value,
                             uint32_t false_value) {
 	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpSelect, state.uint_type, ret, condition, true_value, false_value});
+	state.builder.AddFunction({OpSelect, state.uint_type, ret, condition, true_value, false_value});
 	return ret;
 }
 
@@ -212,12 +210,12 @@ bool IsStorageBufferMemoryKind(IR::ResourceKind kind) {
 
 void EmitStorageBufferOffsets(EmitterState& state) {
 	for (uint32_t i = 0; i < state.program.bindings.buffer_offset_count; i++) {
-		const auto word = EmitShaderDataDwordLoad(
-		    state, state.program.bindings.buffer_offset_dword + i / 4u);
-		const auto shift = ConstantU32(state, (i % 4u) * 8u + 2u);
+		const auto word =
+		    EmitShaderDataDwordLoad(state, state.program.bindings.buffer_offset_dword + i / 4u);
+		const auto shift                = ConstantU32(state, (i % 4u) * 8u + 2u);
 		state.storage_buffer_offsets[i] = EmitBinaryU32(
-		    state, OpBitwiseAnd,
-		    EmitBinaryU32(state, OpShiftRightLogical, word, shift), ConstantU32(state, 0x3fu));
+		    state, OpBitwiseAnd, EmitBinaryU32(state, OpShiftRightLogical, word, shift),
+		    ConstantU32(state, 0x3fu));
 	}
 }
 
@@ -329,8 +327,7 @@ uint32_t EmitRelativeAddress(EmitterState& state, const IR::Instruction& inst, u
 
 uint32_t EmitFlatVirtualAddress(EmitterState& state, const IR::Instruction& inst,
                                 uint32_t first_src, uint32_t src_count) {
-	if (inst.memory.resource >= state.resources.addresses.size() ||
-	    src_count < 2) {
+	if (inst.memory.resource >= state.resources.addresses.size() || src_count < 2) {
 		ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::AddressMemory,
 		                             inst.memory.resource, "flat address snapshot is missing");
 	}
@@ -429,20 +426,20 @@ uint32_t EmitStorageBufferObjectPointer(EmitterState& state, const IR::MemoryInf
 		    ResourceForDescriptor(state, IR::DescriptorBindingKind::AddressMemory, mem.resource);
 		const auto pointer = state.builder.AllocateId();
 		state.builder.AddFunction({OpAccessChain, state.ptr_storage_buffer, pointer,
-		                            state.address_memory_variable,
-		                            ConstantU32(state, binding.array_index)});
+		                           state.address_memory_variable,
+		                           ConstantU32(state, binding.array_index)});
 		return pointer;
 	}
 	const auto binding = StorageBufferBindingForMemory(state, mem, use_pc);
 	const auto pointer = state.builder.AllocateId();
 	state.builder.AddFunction({OpAccessChain, state.ptr_storage_buffer, pointer,
-	                            state.storage_buffer_variable,
-	                            ConstantU32(state, binding.array_index)});
+	                           state.storage_buffer_variable,
+	                           ConstantU32(state, binding.array_index)});
 	return pointer;
 }
 
 uint32_t EmitStorageBufferElementInBounds(EmitterState& state, const IR::MemoryInfo& mem,
-                                           uint32_t index, uint32_t use_pc) {
+                                          uint32_t index, uint32_t use_pc) {
 	index                = EmitStorageBufferIndex(state, mem, index, use_pc);
 	const auto object    = EmitStorageBufferObjectPointer(state, mem, use_pc);
 	const auto length    = state.builder.AllocateId();
@@ -453,7 +450,7 @@ uint32_t EmitStorageBufferElementInBounds(EmitterState& state, const IR::MemoryI
 }
 
 uint32_t EmitStorageBufferElementPointer(EmitterState& state, const IR::MemoryInfo& mem,
-                                          uint32_t index, uint32_t use_pc) {
+                                         uint32_t index, uint32_t use_pc) {
 	index = EmitStorageBufferIndex(state, mem, index, use_pc);
 	if (IsFlatMemoryKind(mem.kind)) {
 		if (state.address_memory_variable == 0) {
@@ -589,9 +586,9 @@ uint32_t EmitMemoryLoadSubDwordValueU32(EmitterState& state, const IR::Instructi
 			const auto left       = state.builder.AllocateId();
 			const auto sign_shift = 32u - data_bits;
 			state.builder.AddFunction({OpShiftLeftLogical, state.uint_type, left, masked,
-			                            ConstantU32(state, sign_shift)});
+			                           ConstantU32(state, sign_shift)});
 			state.builder.AddFunction({OpShiftRightArithmetic, state.uint_type, value, left,
-			                            ConstantU32(state, sign_shift)});
+			                           ConstantU32(state, sign_shift)});
 		}
 		return value;
 	};
@@ -659,16 +656,15 @@ void EmitAtomicUpdateU32(EmitterState& state, uint32_t pointer, IR::ResourceKind
 	state.builder.AddFunction({OpBranch, preheader});
 	state.builder.AddFunction({OpLabel, preheader});
 	state.builder.AddFunction({OpAtomicLoad, state.uint_type, initial, pointer,
-	                            ConstantU32(state, scope),
-	                            ConstantU32(state, MemorySemanticsNone)});
+	                           ConstantU32(state, scope), ConstantU32(state, MemorySemanticsNone)});
 	state.builder.AddFunction({OpBranch, header});
 	state.builder.AddFunction({OpLabel, header});
 	state.builder.AddFunction(
 	    {OpPhi, state.uint_type, observed, initial, preheader, exchanged, continue_label});
 	const auto desired = desired_value(observed);
 	state.builder.AddFunction({OpAtomicCompareExchange, state.uint_type, exchanged, pointer,
-	                            ConstantU32(state, scope), ConstantU32(state, MemorySemanticsNone),
-	                            ConstantU32(state, MemorySemanticsNone), desired, observed});
+	                           ConstantU32(state, scope), ConstantU32(state, MemorySemanticsNone),
+	                           ConstantU32(state, MemorySemanticsNone), desired, observed});
 	const auto success = state.builder.AllocateId();
 	state.builder.AddFunction({OpIEqual, state.bool_type, success, exchanged, observed});
 	state.builder.AddFunction({OpLoopMerge, merge, continue_label, LoopControlNone});
@@ -793,8 +789,7 @@ uint32_t EmitTBufferBitcastU32ToI32(EmitterState& state, uint32_t value) {
 uint32_t EmitTBufferCompareU32Constant(EmitterState& state, uint32_t opcode, uint32_t value,
                                        uint32_t constant) {
 	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {opcode, state.bool_type, ret, value, ConstantU32(state, constant)});
+	state.builder.AddFunction({opcode, state.bool_type, ret, value, ConstantU32(state, constant)});
 	return ret;
 }
 
@@ -821,7 +816,7 @@ uint32_t EmitExtractFormatFieldU32(EmitterState& state, uint32_t raw_word, uint3
 		const auto signed_word = EmitTBufferBitcastU32ToI32(state, raw_word);
 		const auto extracted   = state.builder.AllocateId();
 		state.builder.AddFunction({OpBitFieldSExtract, state.int_type, extracted, signed_word,
-		                            ConstantU32(state, offset), ConstantU32(state, bits)});
+		                           ConstantU32(state, offset), ConstantU32(state, bits)});
 		const auto ret = state.builder.AllocateId();
 		state.builder.AddFunction({OpBitcast, state.uint_type, ret, extracted});
 		return ret;
@@ -829,7 +824,7 @@ uint32_t EmitExtractFormatFieldU32(EmitterState& state, uint32_t raw_word, uint3
 
 	const auto extracted = state.builder.AllocateId();
 	state.builder.AddFunction({OpBitFieldUExtract, state.uint_type, extracted, raw_word,
-	                            ConstantU32(state, offset), ConstantU32(state, bits)});
+	                           ConstantU32(state, offset), ConstantU32(state, bits)});
 	return extracted;
 }
 
@@ -940,7 +935,7 @@ uint32_t NormalizeFormatComponent(EmitterState& state, const Format::BufferForma
 			state.builder.AddFunction(
 			    {OpFDiv, state.float_type, normalized, value, ConstantF32Value(state, max_value)});
 			state.builder.AddFunction({OpExtInst, state.float_type, clamped, state.glsl_std450,
-			                            GlslFMax, normalized, ConstantF32Value(state, -1.0f)});
+			                           GlslFMax, normalized, ConstantF32Value(state, -1.0f)});
 			return EmitTBufferBitcastF32ToU32(state, clamped);
 		}
 		case Format::ComponentType::Float:
@@ -961,18 +956,8 @@ uint32_t UnpackTBufferFormat(EmitterState& state, const IR::Instruction& inst,
 	return NormalizeFormatComponent(state, info, inst.memory.component_index, raw);
 }
 
-bool EmitTypedTBufferLoad(EmitterState& state, const IR::Instruction& inst,
-                          const Format::BufferFormatInfo& info) {
-	if (!Format::CanUseTypedBufferLoad(info.format)) {
-		return false;
-	}
-	const auto value = EmitMemoryLoadDwordValueU32(state, inst, IR::ResourceKind::Buffer, 0,
-	                                               AddressSourceCount(inst, 0));
-	EmitStoreU32(state, inst.dst, value);
-	return true;
-}
-
-bool EmitFormattedBufferLoad(EmitterState& state, const IR::Instruction& inst) {
+bool EmitFormattedBufferLoadValueU32(EmitterState& state, const IR::Instruction& inst,
+                                     uint32_t& value) {
 	if (!IsFormattedBufferComponent(inst)) {
 		return false;
 	}
@@ -984,18 +969,29 @@ bool EmitFormattedBufferLoad(EmitterState& state, const IR::Instruction& inst) {
 
 	const auto info = Format::GetFormatInfo(format);
 	if (inst.memory.component_index >= info.component_count) {
-		EmitStoreU32(state, inst.dst, ConstantU32(state, 0));
+		value = ConstantU32(state, 0);
 		return true;
 	}
 
-	if (EmitTypedTBufferLoad(state, inst, info)) {
+	if (Format::CanUseTypedBufferLoad(info.format)) {
+		value = EmitMemoryLoadDwordValueU32(state, inst, IR::ResourceKind::Buffer, 0,
+		                                    AddressSourceCount(inst, 0));
 		return true;
 	}
 
-	EmitStoreU32(state, inst.dst, UnpackTBufferFormat(state, inst, info));
+	value = UnpackTBufferFormat(state, inst, info);
 	return true;
 }
 
+uint32_t EmitBufferLoadDwordValueU32(EmitterState& state, const IR::Instruction& inst) {
+	uint32_t value = 0;
+	if (EmitFormattedBufferLoadValueU32(state, inst, value)) {
+		return value;
+	}
+	return EmitMemoryLoadDwordValueU32(state, inst, IR::ResourceKind::Buffer, 0,
+	                                   AddressSourceCount(inst, 0));
+}
+
 uint32_t FormattedBufferDwordStoreComponentCount(Prospero::BufferFormat format,
                                                  uint32_t               opcode_components) {
 	switch (format) {
@@ -1117,8 +1113,8 @@ uint32_t EmitAtomicPointer(EmitterState& state, const IR::Instruction& inst) {
 			    StorageImageDescriptorPointer(state, inst.memory.resource, true, inst.pc, view);
 			const auto pointer = state.builder.AllocateId();
 			state.builder.AddFunction({OpImageTexelPointer, state.ptr_image_uint, pointer,
-			                            image_pointer, EmitImageCoordU32(state, inst, view),
-			                            ConstantU32(state, 0)});
+			                           image_pointer, EmitImageCoordU32(state, inst, view),
+			                           ConstantU32(state, 0)});
 			return pointer;
 		}
 		default: return 0;
@@ -1142,8 +1138,8 @@ void EmitAtomicU32(EmitterState& state, const IR::Instruction& inst, uint32_t op
 			    EmitStorageBufferElementPointer(state, inst.memory, index, inst.pc);
 			const auto result = state.builder.AllocateId();
 			state.builder.AddFunction({opcode, state.uint_type, result, pointer,
-			                            ConstantU32(state, ScopeDevice),
-			                            ConstantU32(state, MemorySemanticsNone), value});
+			                           ConstantU32(state, ScopeDevice),
+			                           ConstantU32(state, MemorySemanticsNone), value});
 			EmitDeviceAtomicMemoryBarrier(state);
 			return result;
 		});
@@ -1158,8 +1154,8 @@ void EmitAtomicU32(EmitterState& state, const IR::Instruction& inst, uint32_t op
 			const auto pointer = EmitGdsElementPointer(state, index);
 			const auto result  = state.builder.AllocateId();
 			state.builder.AddFunction({opcode, state.uint_type, result, pointer,
-			                            ConstantU32(state, ScopeDevice),
-			                            ConstantU32(state, MemorySemanticsNone), value});
+			                           ConstantU32(state, ScopeDevice),
+			                           ConstantU32(state, MemorySemanticsNone), value});
 			EmitDeviceAtomicMemoryBarrier(state);
 			return result;
 		});
@@ -1177,7 +1173,7 @@ void EmitAtomicU32(EmitterState& state, const IR::Instruction& inst, uint32_t op
 	const auto old   = state.builder.AllocateId();
 	const auto scope = inst.memory.kind == IR::ResourceKind::Lds ? ScopeWorkgroup : ScopeDevice;
 	state.builder.AddFunction({opcode, state.uint_type, old, pointer, ConstantU32(state, scope),
-	                            ConstantU32(state, MemorySemanticsNone), value});
+	                           ConstantU32(state, MemorySemanticsNone), value});
 	if (inst.memory.kind == IR::ResourceKind::StorageImageUint ||
 	    inst.memory.kind == IR::ResourceKind::Gds) {
 		EmitDeviceAtomicMemoryBarrier(state);
@@ -1199,8 +1195,8 @@ void EmitSLoadDword(EmitterState& state, const IR::Instruction& inst) {
 	    {OpShiftRightLogical, state.uint_type, index, address, ConstantU32(state, 2)});
 	const auto object = state.builder.AllocateId();
 	state.builder.AddFunction({OpAccessChain, state.ptr_storage_buffer, object,
-	                            state.address_memory_variable,
-	                            ConstantU32(state, binding.array_index)});
+	                           state.address_memory_variable,
+	                           ConstantU32(state, binding.array_index)});
 	const auto length    = state.builder.AllocateId();
 	const auto in_bounds = state.builder.AllocateId();
 	state.builder.AddFunction({OpArrayLength, state.uint_type, length, object, 0});
@@ -1227,8 +1223,8 @@ void EmitLoadSrtDword(EmitterState& state, const IR::Instruction& inst) {
 	const auto pointer = state.builder.AllocateId();
 	const auto value   = state.builder.AllocateId();
 	state.builder.AddFunction({OpAccessChain, state.ptr_storage_buffer_uint, pointer,
-	                            state.flattened_srt_variable, ConstantU32(state, 0),
-	                            ConstantU32(state, inst.src[0].imm)});
+	                           state.flattened_srt_variable, ConstantU32(state, 0),
+	                           ConstantU32(state, inst.src[0].imm)});
 	state.builder.AddFunction({OpLoad, state.uint_type, value, pointer});
 	EmitStoreU32(state, inst.dst, value);
 }
@@ -1262,11 +1258,27 @@ void EmitBufferLoadSshort(EmitterState& state, const IR::Instruction& inst) {
 }
 
 void EmitBufferLoadDword(EmitterState& state, const IR::Instruction& inst) {
+	EmitGuardedByExec(
+	    state, [&]() { EmitStoreU32(state, inst.dst, EmitBufferLoadDwordValueU32(state, inst)); });
+}
+
+void EmitBufferLoadDwordGroup(EmitterState& state, const IR::Instruction* instructions,
+                              uint32_t count) {
+	if (instructions == nullptr || count == 0u) {
+		return;
+	}
+
 	EmitGuardedByExec(state, [&]() {
-		if (EmitFormattedBufferLoad(state, inst)) {
-			return;
+		// RDNA VMEM captures every VADDR component before making overlapping VDATA writes
+		// visible. Keep the split IR components instruction-atomic by deferring all stores.
+		std::vector<uint32_t> values;
+		values.reserve(count);
+		for (uint32_t i = 0; i < count; i++) {
+			values.push_back(EmitBufferLoadDwordValueU32(state, instructions[i]));
+		}
+		for (uint32_t i = 0; i < count; i++) {
+			EmitStoreU32(state, instructions[i].dst, values[i]);
 		}
-		EmitMemoryLoadU32(state, inst, IR::ResourceKind::Buffer, 0, AddressSourceCount(inst, 0));
 	});
 }
 
@@ -1385,7 +1397,7 @@ DsCounterAddress EmitAppendConsumeAddress(EmitterState& state, const IR::Instruc
 	state.builder.AddFunction(
 	    {OpShiftRightLogical, state.uint_type, index, address, ConstantU32(state, 2)});
 	state.builder.AddFunction({OpULessThan, state.bool_type, in_bounds,
-	                            ConstantU32(state, inst.memory.offset + 3u), size});
+	                           ConstantU32(state, inst.memory.offset + 3u), size});
 	return {index, size, in_bounds};
 }
 
@@ -1400,7 +1412,7 @@ uint32_t EmitGdsElementInBounds(EmitterState& state, uint32_t index) {
 uint32_t EmitGdsElementPointer(EmitterState& state, uint32_t index) {
 	const auto pointer = state.builder.AllocateId();
 	state.builder.AddFunction({OpAccessChain, state.ptr_storage_buffer_uint, pointer,
-	                            state.gds_variable, ConstantU32(state, 0), index});
+	                           state.gds_variable, ConstantU32(state, 0), index});
 	return pointer;
 }
 
@@ -1431,7 +1443,7 @@ ExecMaskInfo EmitExecMaskInfo(EmitterState& state) {
 	if (state.per_invocation_masks) {
 		const auto ballot = state.builder.AllocateId();
 		state.builder.AddFunction({OpGroupNonUniformBallot, state.vec4_uint_type, ballot,
-		                            ConstantU32(state, ScopeSubgroup), EmitExecActiveBool(state)});
+		                           ConstantU32(state, ScopeSubgroup), EmitExecActiveBool(state)});
 		exec_lo = state.builder.AllocateId();
 		state.builder.AddFunction({OpCompositeExtract, state.uint_type, exec_lo, ballot, 0});
 		if (state.wave_size == 64u) {
@@ -1482,28 +1494,27 @@ void EmitDsAppendConsume(EmitterState& state, const IR::Instruction& inst, uint3
 	const auto do_atomic  = state.builder.AllocateId();
 	state.builder.AddFunction({OpIEqual, state.bool_type, first_lane, subid, exec.first_lane});
 	const auto first_active = EmitLogicalAndBool(state, first_lane, exec.any_active);
-	state.builder.AddFunction(
-	    {OpLogicalAnd, state.bool_type, do_atomic, first_active, in_bounds});
+	state.builder.AddFunction({OpLogicalAnd, state.bool_type, do_atomic, first_active, in_bounds});
 
 	const auto atomic_value = EmitValueOrZeroIfCondition(state, do_atomic, [&]() {
 		const auto pointer = gds ? EmitGdsElementPointer(state, address.index)
 		                         : EmitLdsElementPointer(state, address.index);
 		const auto result  = state.builder.AllocateId();
 		state.builder.AddFunction({atomic_opcode, state.uint_type, result, pointer,
-		                            ConstantU32(state, gds ? ScopeDevice : ScopeWorkgroup),
-		                            ConstantU32(state, MemorySemanticsNone), exec.active_count});
+		                           ConstantU32(state, gds ? ScopeDevice : ScopeWorkgroup),
+		                           ConstantU32(state, MemorySemanticsNone), exec.active_count});
 		if (gds) {
 			EmitDeviceAtomicMemoryBarrier(state);
 		} else {
 			const auto semantics = MemorySemanticsAcquireRelease | MemorySemanticsWorkgroupMemory;
 			state.builder.AddFunction({OpMemoryBarrier, ConstantU32(state, ScopeWorkgroup),
-			                            ConstantU32(state, semantics)});
+			                           ConstantU32(state, semantics)});
 		}
 		return result;
 	});
 	const auto broadcast    = state.builder.AllocateId();
 	state.builder.AddFunction({OpGroupNonUniformShuffle, state.uint_type, broadcast,
-	                            ConstantU32(state, ScopeSubgroup), atomic_value, exec.first_lane});
+	                           ConstantU32(state, ScopeSubgroup), atomic_value, exec.first_lane});
 	const auto value = EmitSelectU32Value(state, exec.any_active, broadcast, ConstantU32(state, 0));
 	EmitStoreU32(state, inst.dst, value);
 }
@@ -1523,8 +1534,8 @@ void EmitDsFloatMinMaxF32(EmitterState& state, const IR::Instruction& inst, bool
 			const auto value_u32 = state.builder.AllocateId();
 			state.builder.AddFunction({OpBitcast, state.float_type, old_f32, old_u32});
 			state.builder.AddFunction({max_value ? OpFOrdGreaterThan : OpFOrdLessThan,
-			                            state.bool_type, store_src, max_value ? old_f32 : cmp_f32,
-			                            max_value ? cmp_f32 : old_f32});
+			                           state.bool_type, store_src, max_value ? old_f32 : cmp_f32,
+			                           max_value ? cmp_f32 : old_f32});
 			state.builder.AddFunction(
 			    {OpSelect, state.float_type, value_f32, store_src, data_f32, old_f32});
 			state.builder.AddFunction({OpBitcast, state.uint_type, value_u32, value_f32});
@@ -1575,14 +1586,13 @@ uint32_t EmitDsSwizzleTargetLane(EmitterState& state, uint32_t subid, uint32_t c
 	const auto xored  = state.builder.AllocateId();
 	const auto base   = state.builder.AllocateId();
 	const auto target = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, 31)});
+	state.builder.AddFunction({OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, 31)});
 	state.builder.AddFunction(
 	    {OpBitwiseAnd, state.uint_type, masked, lane, ConstantU32(state, control & 0x1fu)});
 	state.builder.AddFunction(
 	    {OpBitwiseOr, state.uint_type, ored, masked, ConstantU32(state, (control >> 5u) & 0x1fu)});
-	state.builder.AddFunction({OpBitwiseXor, state.uint_type, xored, ored,
-	                            ConstantU32(state, (control >> 10u) & 0x1fu)});
+	state.builder.AddFunction(
+	    {OpBitwiseXor, state.uint_type, xored, ored, ConstantU32(state, (control >> 10u) & 0x1fu)});
 	state.builder.AddFunction(
 	    {OpBitwiseAnd, state.uint_type, base, subid, ConstantU32(state, 0xffffffe0u)});
 	state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, base, xored});
@@ -1596,7 +1606,7 @@ void EmitDsSwizzleB32(EmitterState& state, const IR::Instruction& inst) {
 	const auto target  = EmitDsSwizzleTargetLane(state, subid, control);
 	const auto value   = state.builder.AllocateId();
 	state.builder.AddFunction({OpGroupNonUniformShuffle, state.uint_type, value,
-	                            ConstantU32(state, ScopeSubgroup), source, target});
+	                           ConstantU32(state, ScopeSubgroup), source, target});
 	const auto exec_active     = EmitLaneIndexActiveBool(state, target);
 	const auto subgroup_active = EmitSubgroupLaneActiveBool(state, target);
 	const auto source_active   = state.builder.AllocateId();

src/graphics/shader/recompiler/ir/ResourceMaterialization.cpp

@@ -12,7 +12,7 @@ namespace {
 
 constexpr uint64_t AddressMask = 0x0000ffffffffffffull;
 
-Decoder::ImageDimension DescriptorDimension(const DescriptorValue&       descriptor,
+Decoder::ImageDimension DescriptorDimension(const DescriptorValue&  descriptor,
                                             Decoder::ImageDimension requested) {
 	const bool is_array = requested == Decoder::ImageDimension::Dim1DArray ||
 	                      requested == Decoder::ImageDimension::Dim2DArray;
@@ -51,8 +51,7 @@ bool ValidImageDescriptor(const DescriptorValue& descriptor) {
 		const auto base_level = (descriptor.dwords[3] >> 12u) & 0xfu;
 		const auto fragments  = (descriptor.dwords[3] >> 16u) & 0xfu;
 		const auto max_mip    = (descriptor.dwords[5] >> 4u) & 0xfu;
-		return base_level == 0 && fragments >= 1 && fragments <= 3 &&
-		       max_mip == fragments;
+		return base_level == 0 && fragments >= 1 && fragments <= 3 && max_mip == fragments;
 	}
 	return true;
 }
@@ -61,6 +60,11 @@ uint32_t DescriptorImageSwizzle(const DescriptorValue& descriptor) {
 	return descriptor.dwords[3] & 0xfffu;
 }
 
+bool DescriptorIsCube(const DescriptorValue& descriptor) {
+	return static_cast<Prospero::ImageType>((descriptor.dwords[3] >> 28u) & 0xfu) ==
+	       Prospero::ImageType::kCube;
+}
+
 bool DecodeBufferDescriptor(const DescriptorValue& descriptor, ShaderBufferResource& result) {
 	if (descriptor.dword_count != std::size(result.fields)) {
 		return false;
@@ -171,12 +175,13 @@ bool ValidateResourceSpecialization(const Program& program, const ResourceSnapsh
 		const auto& image      = program.info.images[i];
 		const auto& descriptor = snapshot.images[i];
 		if (NullImageDescriptor(descriptor)) {
-			bool canonical_kind = image.kind == ResourceKind::Image ||
-			                      image.kind == ResourceKind::StorageImage;
+			bool canonical_kind =
+			    image.kind == ResourceKind::Image || image.kind == ResourceKind::StorageImage;
 			if (image.atomic) {
 				canonical_kind = image.kind == ResourceKind::StorageImageUint;
 			}
-			if (image.dimension != Decoder::ImageDimension::Dim2D || !canonical_kind) {
+			if (image.dimension != Decoder::ImageDimension::Dim2D || image.cube ||
+			    !canonical_kind) {
 				if (error != nullptr) {
 					*error = fmt::format(
 					    "image descriptor {} no longer matches canonical null specialization", i);
@@ -186,7 +191,8 @@ bool ValidateResourceSpecialization(const Program& program, const ResourceSnapsh
 			continue;
 		}
 		const auto dimension = DescriptorDimension(descriptor, image.dimension);
-		if (dimension == Decoder::ImageDimension::Unknown || dimension != image.dimension) {
+		if (dimension == Decoder::ImageDimension::Unknown || dimension != image.dimension ||
+		    DescriptorIsCube(descriptor) != image.cube) {
 			if (error != nullptr) {
 				*error =
 				    fmt::format("image descriptor {} no longer matches specialized dimension", i);
@@ -361,6 +367,7 @@ bool SpecializeResources(Program& program, const ResourceSnapshot& snapshot, std
 		auto&       image      = next.images[i];
 		if (NullImageDescriptor(descriptor)) {
 			image.dimension = Decoder::ImageDimension::Dim2D;
+			image.cube      = false;
 			switch (image.kind) {
 				case ResourceKind::ImageUint: image.kind = ResourceKind::Image; break;
 				case ResourceKind::StorageImageUint:
@@ -386,6 +393,7 @@ bool SpecializeResources(Program& program, const ResourceSnapshot& snapshot, std
 			return false;
 		}
 		image.dimension = descriptor_dimension;
+		image.cube      = DescriptorIsCube(descriptor);
 		if (image.kind == ResourceKind::StorageImage ||
 		    image.kind == ResourceKind::StorageImageUint) {
 			image.storage_swizzle = DescriptorImageSwizzle(descriptor);
@@ -402,6 +410,7 @@ bool SpecializeResources(Program& program, const ResourceSnapshot& snapshot, std
 		std::reference_wrapper<Instruction> inst;
 		ResourceKind                        kind;
 		Decoder::ImageDimension             dimension;
+		bool                                cube;
 	};
 	std::vector<ImagePatch> patches;
 	for (auto& block: program.blocks) {
@@ -420,13 +429,14 @@ bool SpecializeResources(Program& program, const ResourceSnapshot& snapshot, std
 				return false;
 			}
 			const auto& image = next.images[inst.memory.resource];
-			patches.push_back({std::ref(inst), image.kind, image.dimension});
+			patches.push_back({std::ref(inst), image.kind, image.dimension, image.cube});
 		}
 	}
 	program.info = std::move(next);
 	for (const auto& patch: patches) {
 		patch.inst.get().memory.kind            = patch.kind;
 		patch.inst.get().memory.image_dimension = patch.dimension;
+		patch.inst.get().memory.image_cube      = patch.cube;
 	}
 	return true;
 }

src/graphics/shader/recompiler/ir/ScalarProvenance.cpp

@@ -784,10 +784,10 @@ private:
 			if (incoming.empty()) {
 				return ScalarProvenance::Undefined;
 			}
-			if (incoming.size() == 1) {
-				return incoming[0];
-			}
 			if (*phi == ScalarProvenance::Undefined) {
+				if (incoming.size() == 1) {
+					return incoming[0];
+				}
 				*phi = AddValue({ScalarValueOp::Phi, block.start_pc});
 			}
 			m_graph.values[*phi].phi_args = std::move(incoming);

src/graphics/shader/recompiler/ir/ShaderIR.h

@@ -432,6 +432,7 @@ struct MemoryInfo {
 	bool     typed           = false;
 	bool     formatted       = false;
 	bool     image_has_mip   = false;
+	bool     image_cube      = false;
 	bool     glc             = false;
 	bool     slc             = false;
 	bool     idxen           = false;
@@ -607,6 +608,7 @@ struct ImageResource {
 	bool                    written         = false;
 	bool                    atomic          = false;
 	bool                    depth_compare   = false;
+	bool                    cube            = false;
 
 	bool operator==(const ImageResource& other) const = default;
 };

src/kernel/fileSystem.cpp

@@ -238,8 +238,9 @@ static std::filesystem::path ResolvePathIgnoringCase(const std::filesystem::path
 	}
 
 	// Preserve unmatched components for the caller's ENOENT path.
-	std::filesystem::path resolved = path.has_root_path() ? path.root_path() : std::filesystem::path(".");
-	bool                  matched  = true;
+	std::filesystem::path resolved =
+	    path.has_root_path() ? path.root_path() : std::filesystem::path(".");
+	bool matched = true;
 
 	for (const auto& component: path.relative_path()) {
 		if (component.empty()) {
@@ -568,11 +569,11 @@ int64_t KYTY_SYSV_ABI KernelRead(int d, void* buf, size_t nbytes) {
 
 	file->mutex.Lock();
 
-	bool     is_invalid = file->f.IsInvalid();
-	const auto pos       = file->f.Tell();
-	const auto file_size = file->f.Size();
-	const auto remaining = pos < file_size ? file_size - pos : 0;
-	Memory::PrepareHostWrite(reinterpret_cast<uint64_t>(buf),
+	bool       is_invalid = file->f.IsInvalid();
+	const auto pos        = file->f.Tell();
+	const auto file_size  = file->f.Size();
+	const auto remaining  = pos < file_size ? file_size - pos : 0;
+	Memory::InvalidateMemory(reinterpret_cast<uint64_t>(buf),
 	                         std::min<uint64_t>(nbytes, remaining));
 	uint32_t bytes_read = 0;
 	file->f.Read(buf, static_cast<uint32_t>(nbytes), &bytes_read);
@@ -692,13 +693,12 @@ int64_t KYTY_SYSV_ABI KernelPread(int d, void* buf, size_t nbytes, int64_t offse
 
 	file->mutex.Lock();
 
-	bool     is_invalid = file->f.IsInvalid();
-	auto     pos        = file->f.Tell();
-	const auto file_size = file->f.Size();
-	const auto remaining = static_cast<uint64_t>(offset) < file_size
-	                           ? file_size - static_cast<uint64_t>(offset)
-	                           : 0;
-	Memory::PrepareHostWrite(reinterpret_cast<uint64_t>(buf),
+	bool       is_invalid = file->f.IsInvalid();
+	auto       pos        = file->f.Tell();
+	const auto file_size  = file->f.Size();
+	const auto remaining =
+	    static_cast<uint64_t>(offset) < file_size ? file_size - static_cast<uint64_t>(offset) : 0;
+	Memory::InvalidateMemory(reinterpret_cast<uint64_t>(buf),
 	                         std::min<uint64_t>(nbytes, remaining));
 	uint32_t bytes_read = 0;
 	file->f.Seek(offset);

src/kernel/macosGuestAddressSpace.cpp

@@ -0,0 +1,8 @@
+#if defined(__APPLE__) && defined(__x86_64__)
+
+// Make the process own the guest ranges before any runtime initialization.
+asm(".zerofill SYSTEM_MANAGED,SYSTEM_MANAGED,__kyty_system_managed,0x7fffbc000");
+asm(".zerofill SYSTEM_RESERVED,SYSTEM_RESERVED,__kyty_system_reserved,0x7c0004000");
+asm(".zerofill USER_AREA,USER_AREA,__kyty_user_area,0x8c00000000");
+
+#endif

src/kernel/memory.h

@@ -99,13 +99,14 @@ struct KernelMemoryPoolBlockStats {
 static_assert(sizeof(KernelMemoryPoolBlockStats) == 16,
               "KernelMemoryPoolBlockStats struct size is incorrect");
 
-void RegisterCallbacks(callback_func_t alloc_func, callback_func_t free_func);
-void SetFlexibleMemorySize(uint64_t size);
-bool TryWriteBacking(uint64_t vaddr, const void* data, uint64_t size);
-bool TryReadBacking(uint64_t vaddr, void* data, uint64_t size);
-void WriteBacking(uint64_t vaddr, const void* data, uint64_t size) noexcept;
-void PrepareHostWrite(uint64_t vaddr, uint64_t size);
-void InstallGpuResources(Graphics::GpuResourceManager* resources) noexcept;
+void                   RegisterCallbacks(callback_func_t alloc_func, callback_func_t free_func);
+void                   SetFlexibleMemorySize(uint64_t size);
+bool                   TryWriteBacking(uint64_t vaddr, const void* data, uint64_t size);
+bool                   TryReadBacking(uint64_t vaddr, void* data, uint64_t size);
+[[nodiscard]] uint64_t ClampRangeSize(uint64_t vaddr, uint64_t size);
+void                   WriteBacking(uint64_t vaddr, const void* data, uint64_t size) noexcept;
+void                   InvalidateMemory(uint64_t vaddr, uint64_t size);
+void                   InstallGpuResources(Graphics::GpuResourceManager* resources) noexcept;
 [[nodiscard]] bool HandleGpuFault(Graphics::PageFaultAccess access, uint64_t fault_vaddr) noexcept;
 
 int KYTY_SYSV_ABI KernelMapNamedFlexibleMemory(void** addr_in_out, size_t len, int prot, int flags,
@@ -144,7 +145,7 @@ int KYTY_SYSV_ABI KernelIsStack(void* addr, void** start, void** end);
 int KYTY_SYSV_ABI KernelReserveVirtualRange(void** addr, size_t len, int flags, size_t alignment);
 bool              KernelHandleReservedRangeAccessViolation(uint64_t vaddr);
 int KYTY_SYSV_ABI KernelAvailableFlexibleMemorySize(size_t* size);
-int KYTY_SYSV_ABI KernelConfiguredFlexibleMemorySize(uint64_t* size);
+int KYTY_SYSV_ABI KernelConfiguredFlexibleMemorySize(size_t* size);
 int KYTY_SYSV_ABI KernelMprotect(const void* addr, size_t len, int prot);
 int KYTY_SYSV_ABI KernelMtypeprotect(const void* addr, size_t len, int type, int prot);
 int KYTY_SYSV_ABI KernelBatchMap(KernelBatchMapEntry* entries, int num_entries,
@@ -162,17 +163,30 @@ int KYTY_SYSV_ABI KernelMemoryPoolBatch(const KernelMemoryPoolBatchEntry* entrie
 int KYTY_SYSV_ABI KernelMemoryPoolGetBlockStats(KernelMemoryPoolBlockStats* output,
                                                 size_t                      output_size);
 
-void RegisterProgramMemory(uint64_t vaddr, uint64_t size, Common::VirtualMemory::Mode mode,
-                           const char* name);
-void UpdateProgramMemoryProtection(uint64_t vaddr, uint64_t size, Common::VirtualMemory::Mode mode);
-void UnregisterProgramMemory(uint64_t vaddr, uint64_t size);
+uint64_t AllocateProgramMemory(uint64_t search_addr, uint64_t size,
+                               Common::VirtualMemory::Mode mode, const char* name);
+void SetProgramMemoryProtection(uint64_t vaddr, uint64_t size, Common::VirtualMemory::Mode mode);
+uint64_t AllocateRuntimeMemory(uint64_t search_addr, uint64_t size,
+                               Common::VirtualMemory::Mode mode, const char* name,
+                               bool fixed = false);
+uint64_t AllocateGuestStackMemory(uint64_t search_addr, uint64_t size,
+                                  Common::VirtualMemory::Mode mode, const char* name);
+bool     ProtectGuestMemory(uint64_t vaddr, uint64_t size, Common::VirtualMemory::Mode mode,
+                            Common::VirtualMemory::Mode* old_mode = nullptr);
+// Transient PageManager watch state; does not change the guest mapping's semantic protection.
+bool ProtectGuestHostMemory(uint64_t vaddr, uint64_t size, Common::VirtualMemory::Mode mode);
+bool FreeGuestMemory(uint64_t vaddr, uint64_t size);
 
 #if defined(KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS)
-void TestFailNextPhysicalMemoryUnmap();
-void TestFailPhysicalMemoryUnmapAfter(uint32_t successful_unmaps);
-void TestFailHostReservationAfter(uint32_t successful_pages);
-void TestFailNextFixedReserveRangeRegistration();
-bool TestPlaceholderRangeIsFree(uint64_t vaddr, uint64_t size);
+void     TestFailNextPhysicalMemoryUnmap();
+void     TestFailPhysicalMemoryUnmapAfter(uint32_t successful_unmaps);
+void     TestFailGuestBackingStoreUnmapAfter(uint32_t successful_unmaps);
+void     TestFailNextFixedReserveRangeRegistration();
+bool     TestPlaceholderRangeIsFree(uint64_t vaddr, uint64_t size);
+bool     TestGuestAddressRangeIsOwned(uint64_t vaddr, uint64_t size);
+bool     TestGuestBackingOutsideAddressSpace();
+uint64_t TestGuestBackingSize();
+bool     TestGuestFreeRangeBounds();
 #endif
 
 } // namespace Libs::LibKernel::Memory

src/kernel/pthread.cpp

@@ -75,16 +75,16 @@ LIB_NAME("libkernel", "libkernel");
 #undef PTHREAD_STACK_MIN
 #endif
 
-constexpr int      KEYS_MAX                  = 256;
-constexpr int      DESTRUCTOR_ITERATIONS     = 4;
-constexpr size_t   PTHREAD_STACK_DEFAULT     = 0x100000;
-constexpr size_t   GUEST_PTHREAD_STACK_MIN   = 0x4000;
-constexpr size_t   PTHREAD_STACK_PAGE        = 0x4000;
-constexpr size_t   PTHREAD_STACK_GRANULARITY = 0x10000;
-constexpr size_t   PTHREAD_STACK_INITIAL     = 0x200000;
-constexpr size_t   PTHREAD_STACK_EXTRA       = 0x100000;
-constexpr uint64_t PTHREAD_STACK_TOP         = 0x7efff8000ull;
-constexpr uint32_t SIGNAL_APC_POLL_MICROS    = 10000;
+constexpr int      KEYS_MAX                = 256;
+constexpr int      DESTRUCTOR_ITERATIONS   = 4;
+constexpr size_t   PTHREAD_STACK_DEFAULT   = 0x100000;
+constexpr size_t   GUEST_PTHREAD_STACK_MIN = 0x4000;
+constexpr size_t   PTHREAD_STACK_PAGE      = 0x4000;
+constexpr size_t   PTHREAD_STACK_INITIAL   = 0x200000;
+constexpr size_t   PTHREAD_STACK_EXTRA     = 0x100000;
+constexpr uint64_t PTHREAD_STACK_TOP       = 0x7efff8000ull;
+constexpr uint64_t PTHREAD_STACK_BOTTOM    = 0x0000040000ull;
+constexpr uint32_t SIGNAL_APC_POLL_MICROS  = 10000;
 
 static constexpr KernelClockid KERNEL_CLOCK_REALTIME          = 0;
 static constexpr KernelClockid KERNEL_CLOCK_VIRTUAL           = 1;
@@ -697,16 +697,17 @@ static std::atomic<int32_t> g_pthread_thread_id      = 0;
 
 static Common::Mutex g_guest_stack_mutex;
 static uint64_t      g_guest_stack_last = 0;
+struct CachedGuestStack {
+	uint64_t address;
+	size_t   map_size;
+	size_t   guard_size;
+};
+static std::vector<CachedGuestStack> g_guest_stack_cache;
 
 static size_t RoundStackSize(size_t size) {
 	return ((size + PTHREAD_STACK_PAGE - 1) / PTHREAD_STACK_PAGE) * PTHREAD_STACK_PAGE;
 }
 
-static size_t RoundStackMappingSize(size_t size) {
-	return ((size + PTHREAD_STACK_GRANULARITY - 1) / PTHREAD_STACK_GRANULARITY) *
-	       PTHREAD_STACK_GRANULARITY;
-}
-
 static int CreateGuestStack(PthreadAttr attr) {
 	if (attr == nullptr) {
 		return KERNEL_ERROR_EINVAL;
@@ -722,34 +723,41 @@ static int CreateGuestStack(PthreadAttr attr) {
 
 	const auto stack_size = RoundStackSize(attr->stack_size);
 	const auto guard_size = RoundStackSize(attr->guard_size);
-	const auto map_size   = RoundStackMappingSize(stack_size + guard_size);
+	const auto map_size   = stack_size + guard_size;
 
 	uint64_t stack_addr = 0;
+	bool     cached     = false;
 	{
 		Common::LockGuard lock(g_guest_stack_mutex);
 
-		if (g_guest_stack_last == 0) {
-			g_guest_stack_last = (PTHREAD_STACK_TOP - PTHREAD_STACK_INITIAL - PTHREAD_STACK_PAGE) &
-			                     ~(static_cast<uint64_t>(PTHREAD_STACK_GRANULARITY) - 1);
+		auto cached_stack =
+		    std::find_if(g_guest_stack_cache.begin(), g_guest_stack_cache.end(),
+		                 [map_size, guard_size](const auto& stack) {
+			                 return stack.map_size == map_size && stack.guard_size == guard_size;
+		                 });
+		if (cached_stack != g_guest_stack_cache.end()) {
+			stack_addr = cached_stack->address;
+			g_guest_stack_cache.erase(cached_stack);
+			cached = true;
+		} else {
+			if (g_guest_stack_last == 0) {
+				g_guest_stack_last = PTHREAD_STACK_TOP - PTHREAD_STACK_INITIAL - PTHREAD_STACK_PAGE;
+			}
+			if (map_size > g_guest_stack_last - PTHREAD_STACK_BOTTOM) {
+				return KERNEL_ERROR_EAGAIN;
+			}
+			stack_addr = g_guest_stack_last - map_size;
+			g_guest_stack_last -= map_size;
 		}
-
-		stack_addr = g_guest_stack_last - map_size;
-		g_guest_stack_last -= map_size;
 	}
 
-	void* mapped_addr = reinterpret_cast<void*>(stack_addr);
-
-	constexpr int GUEST_PROT_READ_WRITE = 0x03;
-	constexpr int GUEST_MAP_PRIVATE     = 0x02;
-	constexpr int GUEST_MAP_FIXED       = 0x10;
-	constexpr int GUEST_MAP_STACK       = 0x400;
-	constexpr int GUEST_MAP_ANON        = 0x1000;
-
-	int result = Memory::KernelMapNamedFlexibleMemory(
-	    &mapped_addr, map_size, GUEST_PROT_READ_WRITE,
-	    GUEST_MAP_PRIVATE | GUEST_MAP_FIXED | GUEST_MAP_STACK | GUEST_MAP_ANON, "stack");
-	if (result != OK) {
-		return KERNEL_ERROR_EAGAIN;
+	int result = OK;
+	if (!cached) {
+		stack_addr = Memory::AllocateGuestStackMemory(
+		    stack_addr, map_size, Common::VirtualMemory::Mode::ReadWrite, "stack");
+		if (stack_addr == 0) {
+			return KERNEL_ERROR_EAGAIN;
+		}
 	}
 
 	if (guard_size != 0) {
@@ -761,7 +769,7 @@ static int CreateGuestStack(PthreadAttr attr) {
 	}
 
 	attr->stack_addr     = reinterpret_cast<void*>(stack_addr + guard_size);
-	attr->stack_size     = map_size - guard_size;
+	attr->stack_size     = stack_size;
 	attr->stack_user     = false;
 	attr->stack_map_addr = stack_addr;
 	attr->stack_map_size = map_size;
@@ -777,13 +785,90 @@ static void FreeGuestStack(PthreadAttr attr) {
 		return;
 	}
 
-	Memory::KernelMunmap(attr->stack_map_addr, attr->stack_map_size);
+	const auto guard_size = attr->stack_map_size - attr->stack_size;
+	{
+		Common::LockGuard lock(g_guest_stack_mutex);
+		g_guest_stack_cache.push_back({attr->stack_map_addr, attr->stack_map_size, guard_size});
+	}
 
 	attr->stack_addr     = nullptr;
 	attr->stack_map_addr = 0;
 	attr->stack_map_size = 0;
 }
 
+#if defined(KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS)
+bool TestGuestStackOwnerLifecycle(uint64_t* first_address, uint64_t* second_address,
+                                  uint64_t* map_size) {
+	if (first_address == nullptr || second_address == nullptr || map_size == nullptr) {
+		return false;
+	}
+
+	size_t flexible_before = 0;
+	if (Memory::KernelAvailableFlexibleMemorySize(&flexible_before) != OK) {
+		return false;
+	}
+
+	PthreadAttr attr = nullptr;
+	if (PthreadAttrInit(&attr) != OK) {
+		return false;
+	}
+	if (CreateGuestStack(attr) != OK) {
+		PthreadAttrDestroy(&attr);
+		return false;
+	}
+	*first_address = attr->stack_map_addr;
+	*map_size      = attr->stack_map_size;
+	const bool first_owned =
+	    Memory::TestGuestAddressRangeIsOwned(*first_address, static_cast<uint64_t>(*map_size));
+	uint64_t   backing_value = 0;
+	const bool first_private =
+	    !Memory::TryReadBacking(*first_address, &backing_value, sizeof(backing_value));
+	size_t     flexible_during_first = 0;
+	const bool first_capacity_unchanged =
+	    Memory::KernelAvailableFlexibleMemorySize(&flexible_during_first) == OK &&
+	    flexible_during_first == flexible_before;
+	FreeGuestStack(attr);
+
+	if (CreateGuestStack(attr) != OK) {
+		PthreadAttrDestroy(&attr);
+		return false;
+	}
+	*second_address = attr->stack_map_addr;
+	const bool second_owned =
+	    Memory::TestGuestAddressRangeIsOwned(*second_address, static_cast<uint64_t>(*map_size));
+	const bool second_private =
+	    !Memory::TryReadBacking(*second_address, &backing_value, sizeof(backing_value));
+	size_t     flexible_during_second = 0;
+	const bool second_capacity_unchanged =
+	    Memory::KernelAvailableFlexibleMemorySize(&flexible_during_second) == OK &&
+	    flexible_during_second == flexible_before;
+	FreeGuestStack(attr);
+
+	CachedGuestStack cached {};
+	bool             found = false;
+	{
+		Common::LockGuard lock(g_guest_stack_mutex);
+		const auto        entry = std::find_if(
+		    g_guest_stack_cache.begin(), g_guest_stack_cache.end(),
+		    [second_address](const auto& stack) { return stack.address == *second_address; });
+		if (entry != g_guest_stack_cache.end()) {
+			cached = *entry;
+			g_guest_stack_cache.erase(entry);
+			found = true;
+		}
+	}
+
+	const bool unmapped = found && Memory::KernelMunmap(cached.address, cached.map_size) == OK;
+	size_t     flexible_after = 0;
+	const bool final_capacity_unchanged =
+	    Memory::KernelAvailableFlexibleMemorySize(&flexible_after) == OK &&
+	    flexible_after == flexible_before;
+	return PthreadAttrDestroy(&attr) == OK && first_owned && first_private &&
+	       first_capacity_unchanged && second_owned && second_private &&
+	       second_capacity_unchanged && unmapped && final_capacity_unchanged;
+}
+#endif
+
 static KYTY_SYSV_ABI void* RunOnGuestStack(void* arg, pthread_entry_func_t func, void* stack_top) {
 #if defined(__x86_64__) || defined(_M_X64)
 	void*      ret       = nullptr;

src/kernel/pthread.h

@@ -112,11 +112,15 @@ void                  PthreadQueuePendingSignal(Pthread thread, int signum);
 bool                  PthreadHasPendingSignal(Pthread thread, int signum);
 bool                  PthreadTakePendingSignal(Pthread thread, int signum);
 bool PthreadGetGuestStack(Pthread thread, uint64_t* stack_addr, uint64_t* stack_size);
+#if defined(KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS)
+bool TestGuestStackOwnerLifecycle(uint64_t* first_address, uint64_t* second_address,
+                                  uint64_t* map_size);
+#endif
 #if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS
 bool PthreadKillHost(Pthread thread, int host_signal);
 #endif
-int  PthreadGetPriorityForKernel(Pthread thread);
-int  PthreadGetCurrentPriorityForKernel();
+int PthreadGetPriorityForKernel(Pthread thread);
+int PthreadGetCurrentPriorityForKernel();
 
 int KYTY_SYSV_ABI          KernelUsleep(KernelUseconds microseconds);
 unsigned int KYTY_SYSV_ABI KernelSleep(unsigned int seconds);

src/libs/libKernel.cpp

@@ -40,7 +40,10 @@
 #define NOMINMAX
 #endif
 #include <windows.h>
-#elif !defined(__APPLE__)
+#elif defined(__APPLE__)
+#include <csignal>
+#include <sys/ucontext.h>
+#else
 #include <csignal>
 #include <ucontext.h>
 #endif
@@ -837,7 +840,7 @@ static void ApplySignalUcontext(CONTEXT* dst_ctx, const SignalUcontext& src_ctx)
 }
 #endif
 
-#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS && !defined(__APPLE__) && defined(__x86_64__)
+#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS && defined(__x86_64__)
 
 static SignalUcontext CreateSignalUcontextFromHost(const ucontext_t* host_ctx) {
 	SignalUcontext ctx = {};
@@ -845,6 +848,34 @@ static SignalUcontext CreateSignalUcontextFromHost(const ucontext_t* host_ctx) {
 		return ctx;
 	}
 
+#if defined(__APPLE__)
+	const auto& ss = host_ctx->uc_mcontext->__ss;
+
+	ctx.uc_mcontext.mc_rdi    = ss.__rdi;
+	ctx.uc_mcontext.mc_rsi    = ss.__rsi;
+	ctx.uc_mcontext.mc_rdx    = ss.__rdx;
+	ctx.uc_mcontext.mc_rcx    = ss.__rcx;
+	ctx.uc_mcontext.mc_r8     = ss.__r8;
+	ctx.uc_mcontext.mc_r9     = ss.__r9;
+	ctx.uc_mcontext.mc_rax    = ss.__rax;
+	ctx.uc_mcontext.mc_rbx    = ss.__rbx;
+	ctx.uc_mcontext.mc_rbp    = ss.__rbp;
+	ctx.uc_mcontext.mc_r10    = ss.__r10;
+	ctx.uc_mcontext.mc_r11    = ss.__r11;
+	ctx.uc_mcontext.mc_r12    = ss.__r12;
+	ctx.uc_mcontext.mc_r13    = ss.__r13;
+	ctx.uc_mcontext.mc_r14    = ss.__r14;
+	ctx.uc_mcontext.mc_r15    = ss.__r15;
+	ctx.uc_mcontext.mc_rip    = ss.__rip;
+	ctx.uc_mcontext.mc_rsp    = ss.__rsp;
+	ctx.uc_mcontext.mc_rflags = ss.__rflags;
+	ctx.uc_mcontext.mc_cs     = ss.__cs & 0xffffu;
+	ctx.uc_mcontext.mc_gs     = static_cast<uint16_t>(ss.__gs & 0xffffu);
+	ctx.uc_mcontext.mc_fs     = static_cast<uint16_t>(ss.__fs & 0xffffu);
+	ctx.uc_mcontext.mc_len    = sizeof(SignalMcontext);
+
+	return ctx;
+#else
 	const auto* gregs = host_ctx->uc_mcontext.gregs;
 
 	ctx.uc_mcontext.mc_rdi    = static_cast<uint64_t>(gregs[REG_RDI]);
@@ -874,6 +905,7 @@ static SignalUcontext CreateSignalUcontextFromHost(const ucontext_t* host_ctx) {
 	ctx.uc_mcontext.mc_len = sizeof(SignalMcontext);
 
 	return ctx;
+#endif
 }
 
 static void ApplySignalUcontextToHost(ucontext_t* dst_ctx, const SignalUcontext& src_ctx) {
@@ -881,6 +913,29 @@ static void ApplySignalUcontextToHost(ucontext_t* dst_ctx, const SignalUcontext&
 		return;
 	}
 
+#if defined(__APPLE__)
+	auto& ss = dst_ctx->uc_mcontext->__ss;
+
+	ss.__rdi    = src_ctx.uc_mcontext.mc_rdi;
+	ss.__rsi    = src_ctx.uc_mcontext.mc_rsi;
+	ss.__rdx    = src_ctx.uc_mcontext.mc_rdx;
+	ss.__rcx    = src_ctx.uc_mcontext.mc_rcx;
+	ss.__r8     = src_ctx.uc_mcontext.mc_r8;
+	ss.__r9     = src_ctx.uc_mcontext.mc_r9;
+	ss.__rax    = src_ctx.uc_mcontext.mc_rax;
+	ss.__rbx    = src_ctx.uc_mcontext.mc_rbx;
+	ss.__rbp    = src_ctx.uc_mcontext.mc_rbp;
+	ss.__r10    = src_ctx.uc_mcontext.mc_r10;
+	ss.__r11    = src_ctx.uc_mcontext.mc_r11;
+	ss.__r12    = src_ctx.uc_mcontext.mc_r12;
+	ss.__r13    = src_ctx.uc_mcontext.mc_r13;
+	ss.__r14    = src_ctx.uc_mcontext.mc_r14;
+	ss.__r15    = src_ctx.uc_mcontext.mc_r15;
+	ss.__rip    = src_ctx.uc_mcontext.mc_rip;
+	ss.__rsp    = src_ctx.uc_mcontext.mc_rsp;
+	ss.__rflags = src_ctx.uc_mcontext.mc_rflags;
+	// Segment selectors are left untouched; XNU validates them on sigreturn.
+#else
 	auto* gregs = dst_ctx->uc_mcontext.gregs;
 
 	gregs[REG_RDI] = static_cast<greg_t>(src_ctx.uc_mcontext.mc_rdi);
@@ -903,10 +958,17 @@ static void ApplySignalUcontextToHost(ucontext_t* dst_ctx, const SignalUcontext&
 	gregs[REG_EFL] = static_cast<greg_t>(src_ctx.uc_mcontext.mc_rflags);
 
 	// The kernel validates packed segment selectors on sigreturn.
+#endif
 }
 
 static int SignalDispatchHostSignal() {
+#if defined(__APPLE__)
+	// macOS has no realtime signals; SIGUSR1 is otherwise unused on the host side (the
+	// guest's SIGUSR1 is an emulated signal number, not a host registration).
+	static const int host_signal = SIGUSR1;
+#else
 	static const int host_signal = SIGRTMIN + 3;
+#endif
 	return host_signal;
 }
 
@@ -1174,7 +1236,7 @@ static int KYTY_SYSV_ABI KernelRaiseException(Pthread thread, int signum) {
 		}
 		CloseHandle(target_thread);
 		return OK;
-#elif !defined(__APPLE__) && defined(__x86_64__)
+#elif defined(__x86_64__)
 		// Deliver on the target thread.
 		if (thread == PthreadSelfOrNull()) {
 			SignalDispatchScope scope;

src/libs/libNet.cpp

@@ -34,7 +34,7 @@ namespace LibNet {
 
 LIB_VERSION("Net", 1, "Net", 1, 1);
 
-static thread_local int g_net_errno = 0;
+static thread_local int   g_net_errno = 0;
 static constexpr uint32_t g_in6addr_any[4] {};
 
 namespace Net = Network::Net;
@@ -654,6 +654,8 @@ namespace LibHttp2 {
 LIB_VERSION("Http2", 1, "Http2", 1, 1);
 
 constexpr int HTTP2_ERROR_INVALID_ID   = -2122641152; /* 0x817B1100 */
+constexpr int HTTP2_ERROR_BEFORE_SEND  = -2122641307; /* 0x817B1065 */
+constexpr int HTTP2_ERROR_TIMEOUT      = -2122641304; /* 0x817B1068 */
 constexpr int HTTP2_ERROR_NULL_POINTER = -2122640859; /* 0x817B1225 */
 
 struct Http2Options {
@@ -694,14 +696,14 @@ struct Http2Request {
 	std::string                                      url;
 	uint64_t                                         content_length = 0;
 	std::vector<std::pair<std::string, std::string>> headers;
-	bool                                             sent        = false;
-	int                                              status_code = 204;
-	std::string  response_headers                                = "HTTP/2 204 No Content\r\n\r\n";
-	std::string  response_body;
-	size_t       read_offset  = 0;
-	int          async_result = 0;
-	int          async_event  = 0;
-	Http2Options options;
+	int                                              send_result = HTTP2_ERROR_BEFORE_SEND;
+	int                                              status_code = 0;
+	std::string                                      response_headers;
+	std::string                                      response_body;
+	size_t                                           read_offset  = 0;
+	int                                              async_result = HTTP2_ERROR_BEFORE_SEND;
+	int                                              async_event  = 0;
+	Http2Options                                     options;
 };
 
 struct Http2AsyncResult {
@@ -1114,9 +1116,9 @@ static int KYTY_SYSV_ABI Http2SendRequest(int req_id, const void* post_data, siz
 		return HTTP2_ERROR_INVALID_ID;
 	}
 
-	request->second.sent = true;
+	request->second.send_result = HTTP2_ERROR_TIMEOUT;
 
-	return 0;
+	return request->second.send_result;
 }
 
 static int KYTY_SYSV_ABI Http2SendRequestAsync(int req_id, const void* post_data, size_t size,
@@ -1136,8 +1138,8 @@ static int KYTY_SYSV_ABI Http2SendRequestAsync(int req_id, const void* post_data
 		return HTTP2_ERROR_INVALID_ID;
 	}
 
-	request->second.sent         = true;
-	request->second.async_result = 0;
+	request->second.send_result  = HTTP2_ERROR_TIMEOUT;
+	request->second.async_result = request->second.send_result;
 	request->second.async_event  = 0;
 
 	return 0;
@@ -1159,11 +1161,10 @@ static int KYTY_SYSV_ABI Http2WaitAsync(int req_id, Http2AsyncResult* result, ui
 		return HTTP2_ERROR_INVALID_ID;
 	}
 
-	request->second.sent = true;
-	*result              = {};
-	result->event_type   = request->second.async_event;
-	result->req_id       = req_id;
-	result->result       = request->second.async_result;
+	*result            = {};
+	result->event_type = request->second.async_event;
+	result->req_id     = req_id;
+	result->result     = request->second.async_result;
 
 	return 0;
 }
@@ -1178,13 +1179,19 @@ static int KYTY_SYSV_ABI Http2GetStatusCode(int req_id, int* status_code) {
 		return HTTP2_ERROR_NULL_POINTER;
 	}
 
+	*status_code = 0;
+
 	auto request = g_http2_requests.find(req_id);
 	if (request == g_http2_requests.end()) {
 		return HTTP2_ERROR_INVALID_ID;
 	}
 
-	*status_code = request->second.status_code;
+	const int send_result = request->second.send_result;
+	if (send_result != 0) {
+		return send_result;
+	}
 
+	*status_code = request->second.status_code;
 	return 0;
 }
 
@@ -1200,14 +1207,21 @@ static int KYTY_SYSV_ABI Http2GetResponseContentLength(int req_id, int* result,
 		return HTTP2_ERROR_NULL_POINTER;
 	}
 
+	*result         = 0;
+	*content_length = 0;
+
 	auto request = g_http2_requests.find(req_id);
 	if (request == g_http2_requests.end()) {
 		return HTTP2_ERROR_INVALID_ID;
 	}
 
-	*result         = 0; // SCE_HTTP2_CONTENTLEN_EXIST
-	*content_length = request->second.response_body.size();
+	const int send_result = request->second.send_result;
+	if (send_result != 0) {
+		*result = -1;
+		return send_result;
+	}
 
+	*content_length = request->second.response_body.size();
 	return 0;
 }
 
@@ -1223,14 +1237,21 @@ static int KYTY_SYSV_ABI Http2GetAllResponseHeaders(int req_id, char** header,
 		return HTTP2_ERROR_NULL_POINTER;
 	}
 
+	*header      = nullptr;
+	*header_size = 0;
+
 	auto request = g_http2_requests.find(req_id);
 	if (request == g_http2_requests.end()) {
 		return HTTP2_ERROR_INVALID_ID;
 	}
 
+	const int send_result = request->second.send_result;
+	if (send_result != 0) {
+		return send_result;
+	}
+
 	*header      = const_cast<char*>(request->second.response_headers.c_str());
 	*header_size = request->second.response_headers.size();
-
 	return 0;
 }
 
@@ -1250,6 +1271,11 @@ static int KYTY_SYSV_ABI Http2ReadData(int req_id, void* data, size_t size) {
 		return HTTP2_ERROR_INVALID_ID;
 	}
 
+	const int send_result = request->second.send_result;
+	if (send_result != 0) {
+		return send_result;
+	}
+
 	const auto& body = request->second.response_body;
 	const auto  remaining =
 	    request->second.read_offset < body.size() ? body.size() - request->second.read_offset : 0;
@@ -1280,6 +1306,13 @@ static int KYTY_SYSV_ABI Http2ReadDataAsync(int req_id, void* data, size_t size,
 		return HTTP2_ERROR_INVALID_ID;
 	}
 
+	const int send_result = request->second.send_result;
+	if (send_result != 0) {
+		request->second.async_result = send_result;
+		request->second.async_event  = 1;
+		return 0;
+	}
+
 	const auto& body = request->second.response_body;
 	const auto  remaining =
 	    request->second.read_offset < body.size() ? body.size() - request->second.read_offset : 0;
@@ -1398,6 +1431,7 @@ LIB_DEFINE(InitNet_1_NpManager) {
 	LIB_FUNC("O80NrhUOPGY", NpManager::NpCheckPremium);
 	LIB_FUNC("eQH7nWPcAgc", NpManager::NpGetState);
 	LIB_FUNC("e-ZuhGEoeC4", NpManager::NpGetNpReachabilityState);
+	LIB_FUNC("Oad3rvY-NJQ", NpManager::NpHasSignedUp);
 }
 
 } // namespace LibNpManager

src/libs/libTextToSpeech2.cpp

@@ -0,0 +1,31 @@
+#include "common/abi.h"
+#include "libs/errno.h"
+#include "libs/libs.h"
+#include "loader/symbolDatabase.h"
+
+namespace Libs {
+
+LIB_VERSION("TextToSpeech2", 1, "TextToSpeech2", 1, 1);
+
+namespace TextToSpeech2 {
+
+static int KYTY_SYSV_ABI TextToSpeech2GetSpeechStatus() {
+	PRINT_NAME();
+
+	return OK;
+}
+
+static int KYTY_SYSV_ABI TextToSpeech2Cancel() {
+	PRINT_NAME();
+
+	return OK;
+}
+
+} // namespace TextToSpeech2
+
+LIB_DEFINE(InitTextToSpeech2_1) {
+	LIB_FUNC("08JSg9p6bgQ", TextToSpeech2::TextToSpeech2GetSpeechStatus);
+	LIB_FUNC("2jiIxUmcsGo", TextToSpeech2::TextToSpeech2Cancel);
+}
+
+} // namespace Libs

src/libs/libs.cpp

@@ -66,6 +66,7 @@ LIB_DEFINE(InitSaveData_1);
 LIB_DEFINE(InitShare_1);
 LIB_DEFINE(InitSysmodule_1);
 LIB_DEFINE(InitSystemService_1);
+LIB_DEFINE(InitTextToSpeech2_1);
 LIB_DEFINE(InitUserService_1);
 LIB_DEFINE(InitVideoOut_1);
 
@@ -100,6 +101,7 @@ void InitAll(Loader::SymbolDatabase* s) {
 	LIB_LOAD(InitShare_1);
 	LIB_LOAD(InitSysmodule_1);
 	LIB_LOAD(InitSystemService_1);
+	LIB_LOAD(InitTextToSpeech2_1);
 	LIB_LOAD(LibUlt::InitUlt_1);
 	LIB_LOAD(InitUserService_1);
 	LIB_LOAD(VideoDec2::InitVideoDec2_1);

src/libs/network.cpp

@@ -19,7 +19,7 @@
 
 // POSIX uses plain int file descriptors for sockets; provide the Winsock spellings
 // the shared (non-guarded) code paths reference.
-using SOCKET                   = int;
+using SOCKET                           = int;
 static constexpr SOCKET INVALID_SOCKET = -1;
 #endif
 
@@ -820,10 +820,10 @@ struct NetEtherAddr {
 };
 
 #if defined(_WIN32)
-using NativeSocket = SOCKET;
+using NativeSocket                                  = SOCKET;
 static constexpr NativeSocket INVALID_NATIVE_SOCKET = INVALID_SOCKET;
 #else
-using NativeSocket = int;
+using NativeSocket                                  = int;
 static constexpr NativeSocket INVALID_NATIVE_SOCKET = -1;
 #endif
 
@@ -1738,7 +1738,8 @@ int KYTY_SYSV_ABI Accept(int s, void* addr, uint32_t* addrlen) {
 #if defined(_WIN32)
 	sockaddr_storage host_addr {};
 	int              host_addrlen = sizeof(host_addr);
-	NativeSocket accepted = ::accept(socket, reinterpret_cast<sockaddr*>(&host_addr), &host_addrlen);
+	NativeSocket     accepted =
+	    ::accept(socket, reinterpret_cast<sockaddr*>(&host_addr), &host_addrlen);
 	if (accepted == INVALID_NATIVE_SOCKET) {
 		return SetPosixSocketError();
 	}
@@ -3753,8 +3754,6 @@ int KYTY_SYSV_ABI NpGetState(int user_id, uint32_t* state) {
 int KYTY_SYSV_ABI NpGetNpReachabilityState(int user_id, uint32_t* state) {
 	PRINT_NAME();
 
-	constexpr int np_error_invalid_argument = -2141913085; /* 0x80550003 */
-
 	if (state == nullptr) {
 		return np_error_invalid_argument;
 	}
@@ -3767,6 +3766,20 @@ int KYTY_SYSV_ABI NpGetNpReachabilityState(int user_id, uint32_t* state) {
 	return OK;
 }
 
+int KYTY_SYSV_ABI NpHasSignedUp(int user_id, bool* has_signed_up) {
+	PRINT_NAME();
+
+	if (has_signed_up == nullptr) {
+		return np_error_invalid_argument;
+	}
+
+	LOGF("\t user_id = %d\n", user_id);
+
+	*has_signed_up = false;
+
+	return OK;
+}
+
 } // namespace NpManager
 
 } // namespace Libs::Network

src/libs/network.h

@@ -184,6 +184,7 @@ int KYTY_SYSV_ABI  NpCheckPremium(int req_id, const NpCheckPremiumParameter* par
                                   NpCheckPremiumResult* result);
 int KYTY_SYSV_ABI  NpGetState(int user_id, uint32_t* state);
 int KYTY_SYSV_ABI  NpGetNpReachabilityState(int user_id, uint32_t* state);
+int KYTY_SYSV_ABI  NpHasSignedUp(int user_id, bool* has_signed_up);
 
 } // namespace NpManager
 

src/loader/gamePatch.cpp

@@ -2,6 +2,7 @@
 
 #include "common/stringUtils.h"
 #include "common/virtualMemory.h"
+#include "kernel/memory.h"
 #include "loader/elf.h"
 #include "loader/runtimeLinker.h"
 #include "loader/systemContent.h"
@@ -153,18 +154,6 @@ bool ValidateTarget(const Plan& plan, const Program* program, std::string* error
 	return true;
 }
 
-Common::VirtualMemory::Mode ReadableMode(Elf64_Word flags) {
-	const bool executable = (flags & PF_X) != 0;
-	const bool writable   = (flags & PF_W) != 0;
-	if (executable && writable) {
-		return Common::VirtualMemory::Mode::ExecuteReadWrite;
-	}
-	if (executable) {
-		return Common::VirtualMemory::Mode::ExecuteRead;
-	}
-	return writable ? Common::VirtualMemory::Mode::ReadWrite : Common::VirtualMemory::Mode::Read;
-}
-
 bool ResolveWrite(const Program& program, Write* write, std::string* error) {
 	const auto* ehdr = program.elf->GetEhdr();
 	const auto* phdr = program.elf->GetPhdr();
@@ -178,16 +167,9 @@ bool ResolveWrite(const Program& program, Write* write, std::string* error) {
 			continue;
 		}
 
-		const auto                  segment_address = program.base_vaddr + segment.p_vaddr;
-		const bool                  add_read        = (segment.p_flags & PF_R) == 0;
-		Common::VirtualMemory::Mode old_mode {};
-		if (add_read && !Common::VirtualMemory::Protect(segment_address, segment.p_memsz,
-		                                                ReadableMode(segment.p_flags), &old_mode)) {
-			return Fail(error, "could not read a loaded executable segment");
-		}
-
-		const auto* begin = reinterpret_cast<const uint8_t*>(segment_address);
-		const auto* end   = begin + segment.p_filesz;
+		const auto  segment_address = program.base_vaddr + segment.p_vaddr;
+		const auto* begin           = reinterpret_cast<const uint8_t*>(segment_address);
+		const auto* end             = begin + segment.p_filesz;
 		for (auto* current = begin; current < end;) {
 			const auto* found =
 			    std::search(current, end, write->expected.begin(), write->expected.end());
@@ -202,15 +184,9 @@ bool ResolveWrite(const Program& program, Write* write, std::string* error) {
 			match_count++;
 			current = found + 1;
 		}
-
-		if (add_read &&
-		    !Common::VirtualMemory::Protect(segment_address, segment.p_memsz, old_mode)) {
-			return Fail(error, "could not restore executable segment protection");
-		}
 	}
 
-	::printf("Game patch: found %zu entries for '%s'\n", match_count,
-	         write->patch_name.c_str());
+	::printf("Game patch: found %zu entries for '%s'\n", match_count, write->patch_name.c_str());
 	if (match == 0) {
 		return Fail(error, "original bytes not found for '" + write->patch_name + "'");
 	}
@@ -229,16 +205,9 @@ bool PrepareWrites(Plan* plan, const Program& program, std::string* error) {
 
 bool ApplyWrites(Plan* plan, std::string* error) {
 	for (auto& write: plan->writes) {
-		Common::VirtualMemory::Mode old_mode {};
-		const auto                  size = write.replacement.size();
-		if (!Common::VirtualMemory::Protect(
-		        write.address, size, Common::VirtualMemory::Mode::ExecuteReadWrite, &old_mode)) {
-			return Fail(error, "could not make patch memory writable");
-		}
-
+		const auto size = write.replacement.size();
 		std::memcpy(reinterpret_cast<void*>(write.address), write.replacement.data(), size);
-		if (!Common::VirtualMemory::Protect(write.address, size, old_mode) ||
-		    !Common::VirtualMemory::FlushInstructionCache(write.address, size)) {
+		if (!Common::VirtualMemory::FlushInstructionCache(write.address, size)) {
 			return Fail(error, "could not finalize patch");
 		}
 	}

src/loader/runtimeLinker.cpp

@@ -62,14 +62,16 @@ static void FreeTlsBlock(ThreadLocalStorage::Block* block) {
 	if (block->free_func != nullptr) {
 		block->free_func(block->ptr);
 	} else if (block->vm_alloc) {
-		Common::VirtualMemory::Free(reinterpret_cast<uint64_t>(block->ptr));
+		EXIT_IF(!Libs::LibKernel::Memory::FreeGuestMemory(reinterpret_cast<uint64_t>(block->ptr),
+		                                                  block->alloc_size));
 	} else {
 		delete[] block->ptr;
 	}
 
-	block->ptr       = nullptr;
-	block->free_func = nullptr;
-	block->vm_alloc  = false;
+	block->ptr        = nullptr;
+	block->free_func  = nullptr;
+	block->vm_alloc   = false;
+	block->alloc_size = 0;
 }
 
 static uint64_t AlignUp(uint64_t value, uint64_t alignment) {
@@ -131,17 +133,25 @@ static std::vector<StubbedImportRecord> g_stubbed_imports;
 static std::atomic_uint32_t             g_unresolved_stub_call_log_count {0};
 static std::vector<uint64_t>            g_unresolved_stub_thunk_pages;
 static uint64_t                         g_unresolved_stub_thunk_offset = 0;
+static constexpr uint64_t               UNRESOLVED_STUB_PAGE_SIZE      = 4096;
 
 static KYTY_SYSV_ABI uint64_t ResolveImportStubWithId(uint64_t record_id);
 
+static bool PatchGuestMemory64(uint64_t vaddr, uint64_t value) {
+	auto* ptr     = reinterpret_cast<uint64_t*>(vaddr);
+	bool  changed = (*ptr != value);
+	std::memcpy(ptr, &value, sizeof(value));
+	return changed;
+}
+
 static uint64_t AllocateUnresolvedImportThunk(uint64_t record_id) {
-	constexpr uint64_t page_size  = 4096;
 	constexpr uint64_t thunk_size = 162;
 
 	if (g_unresolved_stub_thunk_pages.empty() ||
-	    g_unresolved_stub_thunk_offset + thunk_size > page_size) {
-		auto page = Common::VirtualMemory::Alloc(0, page_size,
-		                                         Common::VirtualMemory::Mode::ExecuteReadWrite);
+	    g_unresolved_stub_thunk_offset + thunk_size > UNRESOLVED_STUB_PAGE_SIZE) {
+		auto page = Libs::LibKernel::Memory::AllocateRuntimeMemory(
+		    0, UNRESOLVED_STUB_PAGE_SIZE, Common::VirtualMemory::Mode::ExecuteReadWrite,
+		    "unresolved_import_thunk");
 		EXIT_NOT_IMPLEMENTED(page == 0);
 		g_unresolved_stub_thunk_pages.push_back(page);
 		g_unresolved_stub_thunk_offset = 0;
@@ -298,7 +308,7 @@ static KYTY_SYSV_ABI uint64_t ResolveImportStubWithId(uint64_t record_id) {
 			     resolved.name.c_str(), resolved.vaddr);
 
 			if (record.patch_vaddr != 0) {
-				*reinterpret_cast<uint64_t*>(record.patch_vaddr) = resolved.vaddr;
+				PatchGuestMemory64(record.patch_vaddr, resolved.vaddr);
 			}
 
 			return resolved.vaddr;
@@ -360,7 +370,7 @@ static KYTY_SYSV_ABI void RunEntry(uint64_t addr, EntryParams* params, atexit_fu
 		register uintptr_t    guest_rbp_reg asm("r15") = guest_rbp;
 #endif
 
-#if defined(__APPLE__) || KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
+#if defined(__APPLE__)
 		asm volatile(
 		    "pushq %%r12\n\t"
 		    "pushq %%r13\n\t"
@@ -374,16 +384,44 @@ static KYTY_SYSV_ABI void RunEntry(uint64_t addr, EntryParams* params, atexit_fu
 		    "popq %%r13\n\t"
 		    "popq %%r12\n\t"
 		    :
-#if defined(__APPLE__)
 		    : [func] "r"(func_reg), "D"(params),
 		      "S"(atexit_func), [guest_rsp] "r"(guest_rsp_reg), [guest_rbp] "r"(guest_rbp_reg)
-#else
-		    : [func] "r"(func), "D"(params),
-		      "S"(atexit_func), [guest_rsp] "r"(guest_rsp), [guest_rbp] "r"(guest_rbp)
-#endif
 		    : "cc", "memory", "rax", "rcx", "rdx", "r8", "r9", "r10", "r11", "xmm0", "xmm1", "xmm2",
 		      "xmm3", "xmm4", "xmm5", "xmm6", "xmm7", "xmm8", "xmm9", "xmm10", "xmm11", "xmm12",
 		      "xmm13", "xmm14", "xmm15");
+#elif KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
+		// Windows stack probes use the TEB stack limits during the guest stack switch.
+		// bounds, which describe the host stack and are invalid while RSP is in guest memory.
+		register entry_func_t func_reg asm("rbx")     = func;
+		register uintptr_t    guest_rsp_reg asm("r8") = guest_rsp;
+		register uintptr_t    guest_rbp_reg asm("r9") = guest_rbp;
+		asm volatile("pushq %%r12\n\t"
+		             "pushq %%r13\n\t"
+		             "pushq %%r14\n\t"
+		             "pushq %%r15\n\t"
+		             "movq %%gs:0x08, %%r14\n\t"
+		             "movq %%gs:0x10, %%r15\n\t"
+		             "xorq %%rcx, %%rcx\n\t"
+		             "movq %%rcx, %%gs:0x08\n\t"
+		             "movq %%rcx, %%gs:0x10\n\t"
+		             "movq %%rsp, %%r12\n\t"
+		             "movq %%rbp, %%r13\n\t"
+		             "movq %[guest_rsp], %%rsp\n\t"
+		             "movq %[guest_rbp], %%rbp\n\t"
+		             "callq *%[func]\n\t"
+		             "movq %%r13, %%rbp\n\t"
+		             "movq %%r12, %%rsp\n\t"
+		             "movq %%r14, %%gs:0x08\n\t"
+		             "movq %%r15, %%gs:0x10\n\t"
+		             "popq %%r15\n\t"
+		             "popq %%r14\n\t"
+		             "popq %%r13\n\t"
+		             "popq %%r12\n\t"
+		             : [guest_rsp] "+r"(guest_rsp_reg), [guest_rbp] "+r"(guest_rbp_reg)
+		             : [func] "r"(func_reg), "D"(params), "S"(atexit_func)
+		             : "cc", "memory", "rax", "rcx", "rdx", "r10", "r11", "xmm0", "xmm1", "xmm2",
+		               "xmm3", "xmm4", "xmm5", "xmm6", "xmm7", "xmm8", "xmm9", "xmm10", "xmm11",
+		               "xmm12", "xmm13", "xmm14", "xmm15");
 #else
 		// Clobbers prevent inputs from being allocated to r12/r13.
 		asm volatile("movq %%rsp, %%r12\n\t"
@@ -449,6 +487,110 @@ static KYTY_SYSV_ABI void RunEntry(uint64_t addr, EntryParams* params, atexit_fu
 #endif
 }
 
+#if defined(KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS)
+struct MainEntryStackTestState {
+	bool      called = false;
+	uintptr_t rsp    = 0;
+#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
+	uintptr_t teb_stack_base  = UINTPTR_MAX;
+	uintptr_t teb_stack_limit = UINTPTR_MAX;
+#endif
+};
+
+static KYTY_SYSV_ABI void TestMainEntryStackCallback(EntryParams* params,
+                                                     atexit_func_t /*atexit_func*/) {
+	auto* state = reinterpret_cast<MainEntryStackTestState*>(const_cast<char*>(params->argv[0]));
+	asm volatile("movq %%rsp, %0" : "=r"(state->rsp) : : "memory");
+#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
+	asm volatile("movq %%gs:0x08, %0\n\t"
+	             "movq %%gs:0x10, %1\n\t"
+	             : "=r"(state->teb_stack_base), "=r"(state->teb_stack_limit)
+	             :
+	             : "memory");
+#endif
+	state->called = true;
+}
+
+bool TestMainEntryUsesGuestStack() {
+	constexpr uint64_t stack_size = 0x10000;
+	const auto         stack_base = Libs::LibKernel::Memory::AllocateRuntimeMemory(
+	    0, stack_size, Common::VirtualMemory::Mode::ReadWrite, "main_entry_stack_test");
+	if (stack_base == 0) {
+		return false;
+	}
+
+	MainEntryStackTestState state {};
+	EntryParams             params {};
+	params.argv[0] = reinterpret_cast<const char*>(&state);
+
+#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
+	uintptr_t original_teb_stack_base  = 0;
+	uintptr_t original_teb_stack_limit = 0;
+	asm volatile("movq %%gs:0x08, %0\n\t"
+	             "movq %%gs:0x10, %1\n\t"
+	             : "=r"(original_teb_stack_base), "=r"(original_teb_stack_limit)
+	             :
+	             : "memory");
+#endif
+
+	RunEntry(reinterpret_cast<uint64_t>(TestMainEntryStackCallback), &params, nullptr,
+	         reinterpret_cast<void*>(stack_base + stack_size));
+
+#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
+	uintptr_t restored_teb_stack_base  = 0;
+	uintptr_t restored_teb_stack_limit = 0;
+	asm volatile("movq %%gs:0x08, %0\n\t"
+	             "movq %%gs:0x10, %1\n\t"
+	             : "=r"(restored_teb_stack_base), "=r"(restored_teb_stack_limit)
+	             :
+	             : "memory");
+	const bool teb_ok = state.teb_stack_base == 0 && state.teb_stack_limit == 0 &&
+	                    restored_teb_stack_base == original_teb_stack_base &&
+	                    restored_teb_stack_limit == original_teb_stack_limit;
+#else
+	constexpr bool teb_ok = true;
+#endif
+
+	const bool rsp_ok = state.rsp >= stack_base && state.rsp < stack_base + stack_size;
+	const bool freed  = Libs::LibKernel::Memory::FreeGuestMemory(stack_base, stack_size);
+	return state.called && rsp_ok && teb_ok && freed;
+}
+
+bool TestModuleRelocationUsesWritableHostMapping() {
+	constexpr uint64_t page_size = 0x4000;
+	constexpr uint64_t value     = 0x4b59545950415443;
+	const auto         base      = Libs::LibKernel::Memory::AllocateProgramMemory(
+	    0, page_size, Common::VirtualMemory::Mode::ReadWrite, "host_only_patch_test");
+	if (base == 0) {
+		return false;
+	}
+	Libs::LibKernel::Memory::SetProgramMemoryProtection(base, page_size,
+	                                                    Common::VirtualMemory::Mode::Read);
+
+	Libs::LibKernel::Memory::VirtualQueryInfo before {};
+	Libs::LibKernel::Memory::VirtualQueryInfo after {};
+	const bool                                before_ok =
+	    Libs::LibKernel::Memory::KernelVirtualQuery(reinterpret_cast<const void*>(base), 0, &before,
+	                                                sizeof(before)) == 0;
+	const bool changed  = PatchGuestMemory64(base, value);
+	const bool after_ok = Libs::LibKernel::Memory::KernelVirtualQuery(
+	                          reinterpret_cast<const void*>(base), 0, &after, sizeof(after)) == 0;
+	const bool value_ok = *reinterpret_cast<const uint64_t*>(base) == value;
+#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
+	MEMORY_BASIC_INFORMATION mbi {};
+	const bool               host_mode_ok =
+	    VirtualQuery(reinterpret_cast<const void*>(base), &mbi, sizeof(mbi)) != 0 &&
+	    mbi.Protect == PAGE_READWRITE;
+#else
+	constexpr bool host_mode_ok = true;
+#endif
+	const bool freed = Libs::LibKernel::Memory::FreeGuestMemory(base, page_size);
+
+	return before_ok && after_ok && changed && value_ok && host_mode_ok && freed &&
+	       before.protection == after.protection;
+}
+#endif
+
 static uint64_t GetAlignedSize(const Elf64_Phdr* p) {
 	return (p->p_align != 0 ? (p->p_memsz + (p->p_align - 1)) & ~(p->p_align - 1) : p->p_memsz);
 }
@@ -1038,7 +1180,7 @@ static void RelocateRecord(uint32_t index, Elf64_Rela* r, Program* program, bool
 	// KYTY_PROFILER_BLOCK("patch");
 
 	if (ri.resolved) {
-		patched = Common::VirtualMemory::PatchReplace(ri.vaddr, ri.value);
+		patched = PatchGuestMemory64(ri.vaddr, ri.value);
 	} else {
 		uint64_t value = 0;
 		bool     weak  = (ri.bind == BindType::Weak || !program->fail_if_global_not_resolved);
@@ -1056,7 +1198,7 @@ static void RelocateRecord(uint32_t index, Elf64_Rela* r, Program* program, bool
 		}
 
 		if (value != 0) {
-			patched = Common::VirtualMemory::PatchReplace(ri.vaddr, value);
+			patched = PatchGuestMemory64(ri.vaddr, value);
 		} else {
 			auto dbg_str = fmt::format("[{:016x}] <- {:016x}, {}, {}, {}, {}", ri.vaddr, ri.value,
 			                           ri.name.c_str(), Common::EnumName(ri.type).c_str(),
@@ -1079,7 +1221,7 @@ static void RelocateRecord(uint32_t index, Elf64_Rela* r, Program* program, bool
 			}
 
 			if (value != 0) {
-				patched = Common::VirtualMemory::PatchReplace(ri.vaddr, value);
+				patched = PatchGuestMemory64(ri.vaddr, value);
 			}
 		}
 	}
@@ -1459,6 +1601,7 @@ void RuntimeLinker::Execute(const std::filesystem::path& game_patch) {
 
 	PreloadAdjacentPrograms();
 	RelocateAll();
+
 	if (!game_patch.empty()) {
 		GamePatch::Apply(game_patch, m_programs.empty() ? nullptr : m_programs.front());
 	}
@@ -1489,6 +1632,21 @@ void RuntimeLinker::Clear() {
 		DeleteProgram(p);
 	}
 	m_programs.clear();
+	for (const auto page: g_unresolved_stub_thunk_pages) {
+		EXIT_IF(!Libs::LibKernel::Memory::FreeGuestMemory(page, UNRESOLVED_STUB_PAGE_SIZE));
+	}
+	g_unresolved_stub_thunk_pages.clear();
+	g_unresolved_stub_thunk_offset = 0;
+	g_stubbed_imports.clear();
+	g_unresolved_stub_call_log_count.store(0);
+	if (g_invalid_memory != 0) {
+		EXIT_IF(!Libs::LibKernel::Memory::FreeGuestMemory(g_invalid_memory, 4096));
+		g_invalid_memory = 0;
+	}
+	g_tls_main_program        = nullptr;
+	g_tls_cached_main_program = nullptr;
+	g_tls_cached_main_tcb     = nullptr;
+	g_desired_base_addr       = SYSTEM_RESERVED + CODE_BASE_OFFSET;
 	m_symbols.reset();
 	m_relocated = false;
 }
@@ -1926,10 +2084,11 @@ uint8_t* RuntimeLinker::TlsGetAddr(Program* program) {
 		const auto tcb_offset =
 		    program->tls.tcb_offset != 0 ? program->tls.tcb_offset : program->tls.image_size;
 		const auto alloc_size = AlignUp(tcb_offset, TCB_ALIGN) + TCB_SIZE;
-		tls.ptr               = reinterpret_cast<uint8_t*>(
-		    Common::VirtualMemory::Alloc(0, alloc_size, Common::VirtualMemory::Mode::ReadWrite));
-		tls.free_func = nullptr;
-		tls.vm_alloc  = true;
+		tls.ptr        = reinterpret_cast<uint8_t*>(Libs::LibKernel::Memory::AllocateRuntimeMemory(
+		    0, alloc_size, Common::VirtualMemory::Mode::ReadWrite, "thread_local_storage"));
+		tls.free_func  = nullptr;
+		tls.vm_alloc   = true;
+		tls.alloc_size = alloc_size;
 
 		EXIT_IF(tls.ptr == nullptr);
 
@@ -2006,8 +2165,9 @@ void RuntimeLinker::LoadProgramToMemory(Program* program) {
 	EXIT_IF(tls_handler_size > UINT64_MAX - program->base_size_aligned);
 	program->mapped_size = program->base_size_aligned + tls_handler_size;
 
-	program->base_vaddr = Common::VirtualMemory::Alloc(
-	    g_desired_base_addr, program->mapped_size, Common::VirtualMemory::Mode::ExecuteReadWrite);
+	program->base_vaddr = Libs::LibKernel::Memory::AllocateProgramMemory(
+	    g_desired_base_addr, program->mapped_size, Common::VirtualMemory::Mode::ExecuteReadWrite,
+	    Common::PathToString(program->file_name.filename()).c_str());
 
 	if (!is_shared) {
 		program->tls.handler_vaddr = program->base_vaddr + program->base_size_aligned;
@@ -2017,10 +2177,6 @@ void RuntimeLinker::LoadProgramToMemory(Program* program) {
 
 	EXIT_IF(program->base_vaddr == 0);
 	EXIT_IF(program->base_size_aligned < program->base_size);
-	Libs::LibKernel::Memory::RegisterProgramMemory(
-	    program->base_vaddr, program->mapped_size, Common::VirtualMemory::Mode::ExecuteReadWrite,
-	    Common::PathToString(program->file_name.filename()).c_str());
-
 	LOGF("base_vaddr             = 0x%016" PRIx64 "\n"
 	     "base_size              = 0x%016" PRIx64 "\n"
 	     "base_size_aligned      = 0x%016" PRIx64 "\n"
@@ -2060,11 +2216,8 @@ void RuntimeLinker::LoadProgramToMemory(Program* program) {
 			}
 
 			if (!skip_protect) {
-				if (!Common::VirtualMemory::Protect(segment_addr, segment_memory_size, mode)) {
-					EXIT("failed to protect ELF segment %u\n", static_cast<unsigned>(i));
-				}
-				Libs::LibKernel::Memory::UpdateProgramMemoryProtection(segment_addr,
-				                                                       segment_memory_size, mode);
+				Libs::LibKernel::Memory::SetProgramMemoryProtection(segment_addr,
+				                                                    segment_memory_size, mode);
 
 				if (Common::VirtualMemory::IsExecute(mode)) {
 					Common::VirtualMemory::FlushInstructionCache(segment_addr, segment_memory_size);
@@ -2105,15 +2258,29 @@ void RuntimeLinker::LoadProgramToMemory(Program* program) {
 
 void RuntimeLinker::DeleteProgram(Program* p) {
 	auto program = std::unique_ptr<Program>(p);
+	if (g_tls_main_program == program.get()) {
+		g_tls_main_program = nullptr;
+	}
+	if (g_tls_cached_main_program == program.get()) {
+		g_tls_cached_main_program = nullptr;
+		g_tls_cached_main_tcb     = nullptr;
+	}
+	for (auto& record: g_stubbed_imports) {
+		if (record.patch_vaddr >= program->base_vaddr &&
+		    record.patch_vaddr < program->base_vaddr + program->mapped_size) {
+			record.patch_vaddr = 0;
+		}
+	}
 
 	if (program->base_vaddr != 0 || program->mapped_size != 0) {
 		EXIT_IF(program->base_vaddr == 0 || program->mapped_size == 0);
-		Libs::LibKernel::Memory::UnregisterProgramMemory(program->base_vaddr, program->mapped_size);
-		EXIT_IF(!Common::VirtualMemory::Free(program->base_vaddr));
+		EXIT_IF(
+		    !Libs::LibKernel::Memory::FreeGuestMemory(program->base_vaddr, program->mapped_size));
 	}
 
 	if (program->custom_call_plt_vaddr != 0 || program->custom_call_plt_num != 0) {
-		Common::VirtualMemory::Free(program->custom_call_plt_vaddr);
+		const auto size = Jit::CallPlt::GetSize(program->custom_call_plt_num);
+		EXIT_IF(!Libs::LibKernel::Memory::FreeGuestMemory(program->custom_call_plt_vaddr, size));
 	}
 }
 
@@ -2237,13 +2404,13 @@ static void InstallRelocateHandler(Program* program) {
 	void**   pltgot       = reinterpret_cast<void**>(pltgot_vaddr);
 
 	Common::VirtualMemory::Mode old_mode {};
-	Common::VirtualMemory::Protect(pltgot_vaddr, pltgot_size, Common::VirtualMemory::Mode::Write,
-	                               &old_mode);
+	EXIT_IF(!Libs::LibKernel::Memory::ProtectGuestMemory(
+	    pltgot_vaddr, pltgot_size, Common::VirtualMemory::Mode::Write, &old_mode));
 
 	pltgot[1] = program;
 	pltgot[2] = reinterpret_cast<void*>(RelocateHandler);
 
-	Common::VirtualMemory::Protect(pltgot_vaddr, pltgot_size, old_mode);
+	EXIT_IF(!Libs::LibKernel::Memory::ProtectGuestMemory(pltgot_vaddr, pltgot_size, old_mode));
 
 	if (Common::VirtualMemory::IsExecute(old_mode)) {
 		Common::VirtualMemory::FlushInstructionCache(pltgot_vaddr, pltgot_size);
@@ -2253,15 +2420,15 @@ static void InstallRelocateHandler(Program* program) {
 	if (program->custom_call_plt_vaddr == 0) {
 		program->custom_call_plt_num =
 		    program->dynamic_info->jmprela_table_size / sizeof(Elf64_Rela);
-		auto size = Jit::CallPlt::GetSize(program->custom_call_plt_num);
-		program->custom_call_plt_vaddr =
-		    Common::VirtualMemory::Alloc(SYSTEM_RESERVED, size, Common::VirtualMemory::Mode::Write);
+		auto size                      = Jit::CallPlt::GetSize(program->custom_call_plt_num);
+		program->custom_call_plt_vaddr = Libs::LibKernel::Memory::AllocateRuntimeMemory(
+		    SYSTEM_RESERVED, size, Common::VirtualMemory::Mode::Write, "custom_call_plt");
 		EXIT_NOT_IMPLEMENTED(program->custom_call_plt_vaddr == 0);
 		auto* code = new (reinterpret_cast<void*>(program->custom_call_plt_vaddr))
 		    Jit::CallPlt(program->custom_call_plt_num);
 		code->SetPltGot(pltgot_vaddr);
-		Common::VirtualMemory::Protect(program->custom_call_plt_vaddr, size,
-		                               Common::VirtualMemory::Mode::Execute);
+		EXIT_IF(!Libs::LibKernel::Memory::ProtectGuestMemory(program->custom_call_plt_vaddr, size,
+		                                                     Common::VirtualMemory::Mode::Execute));
 		Common::VirtualMemory::FlushInstructionCache(program->custom_call_plt_vaddr, size);
 	}
 }
@@ -2272,8 +2439,8 @@ void RuntimeLinker::Relocate(Program* program) {
 	EXIT_IF(program == nullptr);
 
 	if (g_invalid_memory == 0) {
-		g_invalid_memory = Common::VirtualMemory::Alloc(INVALID_MEMORY, 4096,
-		                                                Common::VirtualMemory::Mode::NoAccess);
+		g_invalid_memory = Libs::LibKernel::Memory::AllocateRuntimeMemory(
+		    INVALID_MEMORY, 4096, Common::VirtualMemory::Mode::NoAccess, "invalid_memory", true);
 		EXIT_NOT_IMPLEMENTED(g_invalid_memory == 0);
 	}
 
@@ -2447,12 +2614,9 @@ void RuntimeLinker::SetupTlsHandler(Program* program) {
 		stub->SetOutputReg(reg);
 	}
 
-	if (!Common::VirtualMemory::Protect(program->tls.handler_vaddr, Jit::SafeCall::GetSize(),
-	                                    Common::VirtualMemory::Mode::Execute)) {
-		EXIT("failed to protect program TLS handler\n");
-	}
-	Libs::LibKernel::Memory::UpdateProgramMemoryProtection(
-	    program->tls.handler_vaddr, Jit::SafeCall::GetSize(), Common::VirtualMemory::Mode::Execute);
+	EXIT_IF(!Libs::LibKernel::Memory::ProtectGuestMemory(program->tls.handler_vaddr,
+	                                                     Jit::SafeCall::GetSize(),
+	                                                     Common::VirtualMemory::Mode::Execute));
 	Common::VirtualMemory::FlushInstructionCache(program->tls.handler_vaddr,
 	                                             Jit::SafeCall::GetSize());
 }

src/loader/runtimeLinker.h

@@ -48,9 +48,10 @@ struct LibraryId {
 
 struct ThreadLocalStorage {
 	struct Block {
-		uint8_t*                     ptr       = nullptr;
-		application_heap_free_func_t free_func = nullptr;
-		bool                         vm_alloc  = false;
+		uint8_t*                     ptr        = nullptr;
+		application_heap_free_func_t free_func  = nullptr;
+		bool                         vm_alloc   = false;
+		uint64_t                     alloc_size = 0;
 	};
 
 	~ThreadLocalStorage();
@@ -204,6 +205,11 @@ private:
 	application_heap_posix_memalign_func_t m_application_heap_posix_memalign = nullptr;
 };
 
+#if defined(KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS)
+bool TestMainEntryUsesGuestStack();
+bool TestModuleRelocationUsesWritableHostMapping();
+#endif
+
 } // namespace Loader
 
 #endif /* EMULATOR_INCLUDE_EMULATOR_LOADER_RUNTIMELINKER_H_ */

src/loader/x64InstructionEmulator.cpp

@@ -2,6 +2,8 @@
 
 #include "common/common.h"
 
+#include <cstring>
+
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
 #include <windows.h> // IWYU pragma: keep
 #elif !defined(__APPLE__)
@@ -56,6 +58,362 @@ static uint64_t InsertBitField(uint64_t dst, uint64_t src, uint32_t length, uint
 	return (dst & ~shifted) | src_shifted;
 }
 
+struct XmmWords {
+	uint32_t w[4];
+};
+
+static uint32_t Rol32(uint32_t value, unsigned int shift) {
+	shift &= 31u;
+	return (value << shift) | (value >> (32u - shift));
+}
+
+static uint32_t Rotr32(uint32_t value, unsigned int shift) {
+	shift &= 31u;
+	return (value >> shift) | (value << (32u - shift));
+}
+
+static void Sha1Msg1(XmmWords& dest, const XmmWords& src2) {
+	const uint32_t w0 = dest.w[3];
+	const uint32_t w1 = dest.w[2];
+	const uint32_t w2 = dest.w[1];
+	const uint32_t w3 = dest.w[0];
+	const uint32_t w4 = src2.w[3];
+	const uint32_t w5 = src2.w[2];
+	dest.w[3] = w2 ^ w0;
+	dest.w[2] = w3 ^ w1;
+	dest.w[1] = w4 ^ w2;
+	dest.w[0] = w5 ^ w3;
+}
+
+static void Sha1Msg2(XmmWords& dest, const XmmWords& src2) {
+	const uint32_t w13 = src2.w[2];
+	const uint32_t w14 = src2.w[1];
+	const uint32_t w15 = src2.w[0];
+	const uint32_t w16 = Rol32(dest.w[3] ^ w13, 1u);
+	const uint32_t w17 = Rol32(dest.w[2] ^ w14, 1u);
+	const uint32_t w18 = Rol32(dest.w[1] ^ w15, 1u);
+	const uint32_t w19 = Rol32(dest.w[0] ^ w16, 1u);
+	dest.w[3] = w16;
+	dest.w[2] = w17;
+	dest.w[1] = w18;
+	dest.w[0] = w19;
+}
+
+static void Sha1Nexte(XmmWords& dest, const XmmWords& src2) {
+	const uint32_t tmp  = Rol32(dest.w[3], 30u);
+	dest.w[3]           = src2.w[3] + tmp;
+	dest.w[2]           = src2.w[2];
+	dest.w[1]           = src2.w[1];
+	dest.w[0]           = src2.w[0];
+}
+
+static uint32_t Sha1RoundFunc(uint8_t group, uint32_t b, uint32_t c, uint32_t d) {
+	switch (group & 3u) {
+		case 0: return (b & c) ^ ((~b) & d);
+		case 1: return b ^ c ^ d;
+		case 2: return (b & c) ^ (b & d) ^ (c & d);
+		default: return b ^ c ^ d;
+	}
+}
+
+static uint32_t Sha1RoundConstant(uint8_t group) {
+	switch (group & 3u) {
+		case 0: return 0x5a827999u;
+		case 1: return 0x6ed9eba1u;
+		case 2: return 0x8f1bbcdcu;
+		default: return 0xca62c1d6u;
+	}
+}
+
+static void Sha1Rnds4(XmmWords& dest, const XmmWords& src2, uint8_t imm8) {
+	const uint8_t  group = imm8 & 3u;
+	const uint32_t k     = Sha1RoundConstant(group);
+	const uint32_t w[4]  = {src2.w[3], src2.w[2], src2.w[1], src2.w[0]};
+
+	uint32_t a = dest.w[3];
+	uint32_t b = dest.w[2];
+	uint32_t c = dest.w[1];
+	uint32_t d = dest.w[0];
+	uint32_t e = 0;
+
+	for (unsigned int round = 0; round < 4u; round++) {
+		uint32_t term = Sha1RoundFunc(group, b, c, d) + Rol32(a, 5u) + w[round] + k;
+		if (round > 0u) {
+			term += e;
+		}
+		const uint32_t a1 = term;
+		e                 = d;
+		d                 = c;
+		c                 = Rol32(b, 30u);
+		b                 = a;
+		a                 = a1;
+	}
+
+	dest.w[3] = a;
+	dest.w[2] = b;
+	dest.w[1] = c;
+	dest.w[0] = d;
+}
+
+static uint32_t Sha256Sigma0(uint32_t x) {
+	return Rotr32(x, 7u) ^ Rotr32(x, 18u) ^ (x >> 3u);
+}
+
+static uint32_t Sha256Sigma1(uint32_t x) {
+	return Rotr32(x, 17u) ^ Rotr32(x, 19u) ^ (x >> 10u);
+}
+
+static uint32_t Sha256Sum0(uint32_t x) {
+	return Rotr32(x, 2u) ^ Rotr32(x, 13u) ^ Rotr32(x, 22u);
+}
+
+static uint32_t Sha256Sum1(uint32_t x) {
+	return Rotr32(x, 6u) ^ Rotr32(x, 11u) ^ Rotr32(x, 25u);
+}
+
+static uint32_t Sha256Ch(uint32_t e, uint32_t f, uint32_t g) {
+	return (e & f) ^ ((~e) & g);
+}
+
+static uint32_t Sha256Maj(uint32_t a, uint32_t b, uint32_t c) {
+	return (a & b) ^ (a & c) ^ (b & c);
+}
+
+static void Sha256Msg1(XmmWords& dest, const XmmWords& src2) {
+	const uint32_t w4 = src2.w[0];
+	const uint32_t w3 = dest.w[3];
+	const uint32_t w2 = dest.w[2];
+	const uint32_t w1 = dest.w[1];
+	const uint32_t w0 = dest.w[0];
+	dest.w[3]           = w3 + Sha256Sigma0(w4);
+	dest.w[2]           = w2 + Sha256Sigma0(w3);
+	dest.w[1]           = w1 + Sha256Sigma0(w2);
+	dest.w[0]           = w0 + Sha256Sigma0(w1);
+}
+
+static void Sha256Msg2(XmmWords& dest, const XmmWords& src2) {
+	const uint32_t w14 = src2.w[2];
+	const uint32_t w15 = src2.w[3];
+	const uint32_t w16 = dest.w[0] + Sha256Sigma1(w14);
+	const uint32_t w17 = dest.w[1] + Sha256Sigma1(w15);
+	const uint32_t w18 = dest.w[2] + Sha256Sigma1(w16);
+	const uint32_t w19 = dest.w[3] + Sha256Sigma1(w17);
+	dest.w[3]          = w19;
+	dest.w[2]          = w18;
+	dest.w[1]          = w17;
+	dest.w[0]          = w16;
+}
+
+static void Sha256Rnds2(XmmWords& dest, const XmmWords& src2, const XmmWords& xmm0) {
+	uint32_t a = src2.w[3];
+	uint32_t b = src2.w[2];
+	uint32_t c = dest.w[3];
+	uint32_t d = dest.w[2];
+	uint32_t e = src2.w[1];
+	uint32_t f = src2.w[0];
+	uint32_t g = dest.w[1];
+	uint32_t h = dest.w[0];
+
+	for (unsigned int round = 0; round < 2u; round++) {
+		const uint32_t wk = xmm0.w[round];
+		const uint32_t t1 = Sha256Ch(e, f, g) + Sha256Sum1(e) + wk + h;
+		const uint32_t t2 = Sha256Maj(a, b, c) + Sha256Sum0(a);
+		const uint32_t a1 = t1 + t2;
+		const uint32_t e1 = t1 + d;
+		const uint32_t b1 = a;
+		const uint32_t c1 = b;
+		const uint32_t d1 = c;
+		const uint32_t f1 = e;
+		const uint32_t g1 = f;
+		const uint32_t h1 = g;
+		a                 = a1;
+		b                 = b1;
+		c                 = c1;
+		d                 = d1;
+		e                 = e1;
+		f                 = f1;
+		g                 = g1;
+		h                 = h1;
+	}
+
+	dest.w[3] = a;
+	dest.w[2] = b;
+	dest.w[1] = e;
+	dest.w[0] = f;
+}
+
+struct ShaNiInsn {
+	uint8_t escape;
+	uint8_t opcode;
+	uint8_t imm8;
+	uint8_t rex;
+	size_t  modrm_offset;
+	size_t  length;
+};
+
+static bool DecodeShaNiInsn(const uint8_t* rip, ShaNiInsn& insn) {
+	size_t  offset = 0;
+	uint8_t rex    = 0;
+	if ((rip[0] & 0xf0u) == 0x40u) {
+		rex    = rip[0];
+		offset = 1;
+	}
+
+	if (rip[offset] != 0x0f) {
+		return false;
+	}
+
+	if (rip[offset + 1] == 0x38) {
+		const uint8_t op = rip[offset + 2];
+		if (op != 0xc8 && op != 0xc9 && op != 0xca && op != 0xcb && op != 0xcc && op != 0xcd) {
+			return false;
+		}
+		insn.escape       = 0x38;
+		insn.opcode       = op;
+		insn.imm8         = 0;
+		insn.rex          = rex;
+		insn.modrm_offset = offset + 3;
+	} else if (rip[offset + 1] == 0x3a && rip[offset + 2] == 0xcc) {
+		insn.escape       = 0x3a;
+		insn.opcode       = 0xcc;
+		insn.rex          = rex;
+		insn.modrm_offset = offset + 3;
+	} else {
+		return false;
+	}
+
+	const uint8_t modrm = rip[insn.modrm_offset];
+	const uint8_t mod   = modrm >> 6u;
+	const uint8_t rm    = modrm & 0x07u;
+	size_t        end   = insn.modrm_offset + 1;
+
+	if (mod != 3u) {
+		uint8_t sib_base = 0xffu;
+		if (rm == 4u) {
+			sib_base = rip[end] & 0x07u;
+			end++;
+		}
+
+		if (mod == 0u && (rm == 5u || (rm == 4u && sib_base == 5u))) {
+			end += 4;
+		} else if (mod == 1u) {
+			end++;
+		} else if (mod == 2u) {
+			end += 4;
+		}
+	}
+
+	if (insn.escape == 0x3a) {
+		insn.imm8 = rip[end];
+		end++;
+	}
+
+	insn.length = end;
+	return true;
+}
+
+static bool ShaNiModrmIsRegister(uint8_t modrm) { return (modrm & 0xc0u) == 0xc0u; }
+
+static uint8_t ShaNiRegIndex(uint8_t modrm, uint8_t rex, bool reg_field) {
+	if (reg_field) {
+		return ((modrm >> 3u) & 0x07u) | ((rex & 0x04u) << 1u);
+	}
+	return (modrm & 0x07u) | ((rex & 0x01u) << 3u);
+}
+
+static bool ResolveShaNiMemoryAddress(const uint8_t* rip, const ShaNiInsn& insn,
+                                      const uint64_t (&gpr)[16], const void*& address) {
+	const uint8_t modrm = rip[insn.modrm_offset];
+	const uint8_t mod   = modrm >> 6u;
+	const uint8_t rm    = modrm & 0x07u;
+	if (mod == 3u) {
+		return false;
+	}
+
+	size_t   offset = insn.modrm_offset + 1;
+	uint64_t result = 0;
+
+	if (rm == 4u) {
+		const uint8_t sib        = rip[offset++];
+		const uint8_t scale      = sib >> 6u;
+		const uint8_t index_low  = (sib >> 3u) & 0x07u;
+		const uint8_t base_low   = sib & 0x07u;
+		const bool    has_index  = index_low != 4u || (insn.rex & 0x02u) != 0;
+		const bool    has_base   = mod != 0u || base_low != 5u;
+
+		if (has_base) {
+			const uint8_t base = base_low | ((insn.rex & 0x01u) << 3u);
+			result += gpr[base];
+		}
+		if (has_index) {
+			const uint8_t index = index_low | ((insn.rex & 0x02u) << 2u);
+			result += gpr[index] << scale;
+		}
+
+		if (!has_base) {
+			int32_t displacement = 0;
+			std::memcpy(&displacement, rip + offset, sizeof(displacement));
+			result += static_cast<uint64_t>(static_cast<int64_t>(displacement));
+			offset += sizeof(displacement);
+		}
+	} else if (mod == 0u && rm == 5u) {
+		int32_t displacement = 0;
+		std::memcpy(&displacement, rip + offset, sizeof(displacement));
+		result = reinterpret_cast<uint64_t>(rip + insn.length) +
+		         static_cast<uint64_t>(static_cast<int64_t>(displacement));
+		offset += sizeof(displacement);
+	} else {
+		const uint8_t base = rm | ((insn.rex & 0x01u) << 3u);
+		result             = gpr[base];
+	}
+
+	if (mod == 1u) {
+		const auto displacement = static_cast<int8_t>(rip[offset]);
+		result += static_cast<uint64_t>(static_cast<int64_t>(displacement));
+	} else if (mod == 2u) {
+		int32_t displacement = 0;
+		std::memcpy(&displacement, rip + offset, sizeof(displacement));
+		result += static_cast<uint64_t>(static_cast<int64_t>(displacement));
+	}
+
+	address = reinterpret_cast<const void*>(result);
+	return true;
+}
+
+static bool ExecuteShaNiInsn(const ShaNiInsn& insn, const XmmWords& src2, const XmmWords& xmm0,
+                             XmmWords& dest) {
+	if (insn.escape == 0x3a && insn.opcode == 0xcc) {
+		Sha1Rnds4(dest, src2, insn.imm8);
+		return true;
+	}
+
+	switch (insn.opcode) {
+		case 0xc8: Sha1Nexte(dest, src2); return true;
+		case 0xc9: Sha1Msg1(dest, src2); return true;
+		case 0xca: Sha1Msg2(dest, src2); return true;
+		case 0xcb: Sha256Rnds2(dest, src2, xmm0); return true;
+		case 0xcc: Sha256Msg1(dest, src2); return true;
+		case 0xcd: Sha256Msg2(dest, src2); return true;
+		default: return false;
+	}
+}
+
+#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
+
+static void LoadXmmWordsWin(const M128A* xmm, XmmWords& out) {
+	out.w[0] = static_cast<uint32_t>(xmm->Low);
+	out.w[1] = static_cast<uint32_t>(xmm->Low >> 32u);
+	out.w[2] = static_cast<uint32_t>(xmm->High);
+	out.w[3] = static_cast<uint32_t>(xmm->High >> 32u);
+}
+
+static void StoreXmmWordsWin(M128A* xmm, const XmmWords& in) {
+	xmm->Low  = static_cast<uint64_t>(in.w[0]) | (static_cast<uint64_t>(in.w[1]) << 32u);
+	xmm->High = static_cast<uint64_t>(in.w[2]) | (static_cast<uint64_t>(in.w[3]) << 32u);
+}
+
+#endif
+
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
 
 static M128A* GetContextXmm(PCONTEXT context, uint8_t index) {
@@ -66,6 +424,76 @@ static M128A* GetContextXmm(PCONTEXT context, uint8_t index) {
 	return &context->Xmm0 + index;
 }
 
+static void LoadContextGprsWin(PCONTEXT context, uint64_t (&gpr)[16]) {
+	gpr[0]  = context->Rax;
+	gpr[1]  = context->Rcx;
+	gpr[2]  = context->Rdx;
+	gpr[3]  = context->Rbx;
+	gpr[4]  = context->Rsp;
+	gpr[5]  = context->Rbp;
+	gpr[6]  = context->Rsi;
+	gpr[7]  = context->Rdi;
+	gpr[8]  = context->R8;
+	gpr[9]  = context->R9;
+	gpr[10] = context->R10;
+	gpr[11] = context->R11;
+	gpr[12] = context->R12;
+	gpr[13] = context->R13;
+	gpr[14] = context->R14;
+	gpr[15] = context->R15;
+}
+
+static bool TryEmulateShaNi(PCONTEXT context) {
+	if (context == nullptr) {
+		return false;
+	}
+
+	const auto* rip = reinterpret_cast<const uint8_t*>(context->Rip);
+	ShaNiInsn   insn {};
+	if (!DecodeShaNiInsn(rip, insn)) {
+		return false;
+	}
+
+	const uint8_t modrm_byte = rip[insn.modrm_offset];
+	const uint8_t dest_index = ShaNiRegIndex(modrm_byte, insn.rex, true);
+	auto*         dest_xmm   = GetContextXmm(context, dest_index);
+	auto*         xmm0       = GetContextXmm(context, 0);
+	if (dest_xmm == nullptr || xmm0 == nullptr) {
+		return false;
+	}
+
+	XmmWords dest {};
+	XmmWords src2 {};
+	XmmWords xmm0_words {};
+	LoadXmmWordsWin(dest_xmm, dest);
+	LoadXmmWordsWin(xmm0, xmm0_words);
+
+	if (ShaNiModrmIsRegister(modrm_byte)) {
+		const uint8_t src_index = ShaNiRegIndex(modrm_byte, insn.rex, false);
+		auto*         src_xmm   = GetContextXmm(context, src_index);
+		if (src_xmm == nullptr) {
+			return false;
+		}
+		LoadXmmWordsWin(src_xmm, src2);
+	} else {
+		uint64_t    gpr[16] {};
+		const void* source = nullptr;
+		LoadContextGprsWin(context, gpr);
+		if (!ResolveShaNiMemoryAddress(rip, insn, gpr, source)) {
+			return false;
+		}
+		std::memcpy(&src2, source, sizeof(src2));
+	}
+
+	if (!ExecuteShaNiInsn(insn, src2, xmm0_words, dest)) {
+		return false;
+	}
+
+	StoreXmmWordsWin(dest_xmm, dest);
+	context->Rip += insn.length;
+	return true;
+}
+
 static bool TryEmulateSse4a(PCONTEXT context) {
 	if (context == nullptr) {
 		return false;
@@ -160,6 +588,91 @@ static uint32_t* GetContextXmm(ucontext_t* context, uint8_t index) {
 	return static_cast<uint32_t*>(fpregs->_xmm[index].element);
 }
 
+static void LoadContextGprsLin(ucontext_t* context, uint64_t (&gpr)[16]) {
+	gpr[0]  = static_cast<uint64_t>(context->uc_mcontext.gregs[REG_RAX]);
+	gpr[1]  = static_cast<uint64_t>(context->uc_mcontext.gregs[REG_RCX]);
+	gpr[2]  = static_cast<uint64_t>(context->uc_mcontext.gregs[REG_RDX]);
+	gpr[3]  = static_cast<uint64_t>(context->uc_mcontext.gregs[REG_RBX]);
+	gpr[4]  = static_cast<uint64_t>(context->uc_mcontext.gregs[REG_RSP]);
+	gpr[5]  = static_cast<uint64_t>(context->uc_mcontext.gregs[REG_RBP]);
+	gpr[6]  = static_cast<uint64_t>(context->uc_mcontext.gregs[REG_RSI]);
+	gpr[7]  = static_cast<uint64_t>(context->uc_mcontext.gregs[REG_RDI]);
+	gpr[8]  = static_cast<uint64_t>(context->uc_mcontext.gregs[REG_R8]);
+	gpr[9]  = static_cast<uint64_t>(context->uc_mcontext.gregs[REG_R9]);
+	gpr[10] = static_cast<uint64_t>(context->uc_mcontext.gregs[REG_R10]);
+	gpr[11] = static_cast<uint64_t>(context->uc_mcontext.gregs[REG_R11]);
+	gpr[12] = static_cast<uint64_t>(context->uc_mcontext.gregs[REG_R12]);
+	gpr[13] = static_cast<uint64_t>(context->uc_mcontext.gregs[REG_R13]);
+	gpr[14] = static_cast<uint64_t>(context->uc_mcontext.gregs[REG_R14]);
+	gpr[15] = static_cast<uint64_t>(context->uc_mcontext.gregs[REG_R15]);
+}
+
+static void LoadXmmWordsLin(const uint32_t* xmm, XmmWords& out) {
+	out.w[0] = xmm[0];
+	out.w[1] = xmm[1];
+	out.w[2] = xmm[2];
+	out.w[3] = xmm[3];
+}
+
+static void StoreXmmWordsLin(uint32_t* xmm, const XmmWords& in) {
+	xmm[0] = in.w[0];
+	xmm[1] = in.w[1];
+	xmm[2] = in.w[2];
+	xmm[3] = in.w[3];
+}
+
+static bool TryEmulateShaNi(ucontext_t* context) {
+	if (context == nullptr) {
+		return false;
+	}
+
+	auto&       rip_reg = context->uc_mcontext.gregs[REG_RIP];
+	const auto* rip     = reinterpret_cast<const uint8_t*>(rip_reg);
+	ShaNiInsn   insn {};
+	if (!DecodeShaNiInsn(rip, insn)) {
+		return false;
+	}
+
+	const uint8_t modrm_byte = rip[insn.modrm_offset];
+	const uint8_t dest_index = ShaNiRegIndex(modrm_byte, insn.rex, true);
+	auto*         dest_xmm   = GetContextXmm(context, dest_index);
+	auto*         xmm0       = GetContextXmm(context, 0);
+	if (dest_xmm == nullptr || xmm0 == nullptr) {
+		return false;
+	}
+
+	XmmWords dest {};
+	XmmWords src2 {};
+	XmmWords xmm0_words {};
+	LoadXmmWordsLin(dest_xmm, dest);
+	LoadXmmWordsLin(xmm0, xmm0_words);
+
+	if (ShaNiModrmIsRegister(modrm_byte)) {
+		const uint8_t src_index = ShaNiRegIndex(modrm_byte, insn.rex, false);
+		auto*         src_xmm   = GetContextXmm(context, src_index);
+		if (src_xmm == nullptr) {
+			return false;
+		}
+		LoadXmmWordsLin(src_xmm, src2);
+	} else {
+		uint64_t    gpr[16] {};
+		const void* source = nullptr;
+		LoadContextGprsLin(context, gpr);
+		if (!ResolveShaNiMemoryAddress(rip, insn, gpr, source)) {
+			return false;
+		}
+		std::memcpy(&src2, source, sizeof(src2));
+	}
+
+	if (!ExecuteShaNiInsn(insn, src2, xmm0_words, dest)) {
+		return false;
+	}
+
+	StoreXmmWordsLin(dest_xmm, dest);
+	rip_reg += static_cast<greg_t>(insn.length);
+	return true;
+}
+
 static uint64_t GetXmmLow(const uint32_t* xmm) {
 	return static_cast<uint64_t>(xmm[0]) | (static_cast<uint64_t>(xmm[1]) << 32u);
 }
@@ -258,10 +771,12 @@ static bool TryEmulateMonitorxMwaitx(ucontext_t* context) {
 bool TryEmulate(void* native_context) {
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
 	auto* context = static_cast<PCONTEXT>(native_context);
-	return TryEmulateMonitorxMwaitx(context) || TryEmulateSse4a(context);
+	return TryEmulateMonitorxMwaitx(context) || TryEmulateSse4a(context) ||
+	       TryEmulateShaNi(context);
 #elif !defined(__APPLE__)
 	auto* context = static_cast<ucontext_t*>(native_context);
-	return TryEmulateMonitorxMwaitx(context) || TryEmulateSse4a(context);
+	return TryEmulateMonitorxMwaitx(context) || TryEmulateSse4a(context) ||
+	       TryEmulateShaNi(context);
 #else
 	(void)native_context;
 	return false;

tests/MemoryTrackerTests.cpp

@@ -1,6 +1,7 @@
 #include "graphics/host_gpu/memoryTracker.h"
 #include "graphics/host_gpu/rangeSet.h"
 #include "common/assert.h"
+#include "common/virtualMemory.h"
 
 #include <atomic>
 #include <cstdint>
@@ -209,6 +210,20 @@ class SharedPage final {
   HANDLE mapping_ = nullptr;
 };
 #endif
+
+bool ProtectAddressSpace(uint64_t vaddr, uint64_t size,
+                         Common::VirtualMemory::Mode mode) {
+  uint32_t protection = PAGE_NOACCESS;
+  if (mode == Common::VirtualMemory::Mode::Read) {
+    protection = PAGE_READONLY;
+  } else if (mode == Common::VirtualMemory::Mode::ReadWrite) {
+    protection = PAGE_READWRITE;
+  }
+  DWORD old_protection = 0;
+  return VirtualProtect(reinterpret_cast<void *>(vaddr), size, protection,
+                        &old_protection) != 0;
+}
+
 #if 1
 
 bool DummyFault(void *, PageFaultAccess, uint64_t, uint64_t, PageFaultPhase) noexcept {
@@ -353,7 +368,8 @@ struct DownloadTrackerHarness {
     return completed;
   }
 
-  DownloadTrackerHarness() : page_manager(Fault, this), tracker(page_manager) {}
+  DownloadTrackerHarness()
+      : page_manager(Fault, this), tracker(page_manager) {}
 
   PageFaultAccess pending_access = PageFaultAccess::Unknown;
   uint64_t download_address = 0;
@@ -366,11 +382,6 @@ struct DownloadTrackerHarness {
 
 std::atomic<PageManager *> g_native_page_manager{nullptr};
 std::atomic_bool g_native_fault_entered{false};
-std::atomic_bool g_unmap_contended{false};
-
-void UnmapContended() noexcept {
-  g_unmap_contended.store(true, std::memory_order_release);
-}
 
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
 LONG CALLBACK NativeTrackerFaultHandler(EXCEPTION_POINTERS *exception) {
@@ -552,7 +563,7 @@ void TestGpuDownloadFaultOwnership() {
             !harness.tracker.IsRegionGpuModified(address, page_size) &&
             harness.tracker.IsRegionCpuModified(address, page_size) && IsWritable(memory),
         "GPU write fault did not download before granting CPU ownership");
-  harness.tracker.UnmapMemory(address, page_size);
+  harness.tracker.UntrackMemory(address, page_size);
 }
 
 void TestVirtualGpuWriteDiscard() {
@@ -578,7 +589,7 @@ void TestVirtualGpuWriteDiscard() {
   Check(!harness.tracker.IsRegionGpuModified(address, page_size) &&
             harness.tracker.IsRegionCpuModified(address, page_size) && IsWritable(memory),
         "virtual GPU discard did not transfer the page to CPU ownership");
-  harness.tracker.UnmapMemory(address, page_size);
+  harness.tracker.UntrackMemory(address, page_size);
   Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
 }
 
@@ -669,6 +680,8 @@ void TestRangeSet() {
   ranges.Add(0x1000, 0x80);
   ranges.Add(0x1080, 0x80);
   ranges.Add(0x1200, 0x40);
+  Check(ranges.Contains(0x1010, 0xe0) && !ranges.Contains(0x1010, 0x200),
+        "range set containment did not require full coverage");
   auto intersections = ranges.Intersections(0x1070, 0x1b0);
   Check(intersections.size() == 2 && intersections[0].address == 0x1070 &&
             intersections[0].size == 0x90 && intersections[1].address == 0x1200 &&
@@ -682,6 +695,55 @@ void TestRangeSet() {
         "range set subtraction did not preserve both exact tails");
 }
 
+void TestQueriesDoNotRequireMappedOwnership() {
+  constexpr uint64_t address = 0x0000000203000000ull;
+  TrackerHarness harness;
+  const auto page_size = harness.page_manager.GetPageSize();
+  Check(harness.tracker.IsRegionCpuModified(address, page_size) &&
+            !harness.tracker.IsRegionGpuModified(address, page_size),
+        "unowned tracker range did not expose its initial CPU-dirty state");
+}
+
+void TestRangeInvalidation() {
+  constexpr uintptr_t base = 0x0000000201000000ull;
+  TrackerHarness harness;
+  auto &tracker = harness.tracker;
+  auto &page_manager = harness.page_manager;
+  constexpr uint64_t size = Libs::Graphics::TRACKER_REGION_SIZE * 2;
+  auto *memory = static_cast<uint8_t *>(
+      VirtualAlloc(reinterpret_cast<void *>(base), size, MEM_RESERVE | MEM_COMMIT,
+                   PAGE_READWRITE));
+  Check(memory == reinterpret_cast<void *>(base),
+        "range invalidation allocation failed");
+  const auto address = reinterpret_cast<uint64_t>(memory);
+  page_manager.OnGpuMap(address, size);
+
+  tracker.ForEachUploadRange(
+      address, size, true, [](uint64_t, uint64_t) noexcept {},
+      []() noexcept {});
+  Check(tracker.IsRegionGpuModified(address, size) && !IsWritable(memory),
+        "range invalidation setup did not establish GPU ownership");
+
+  uint32_t flushes = 0;
+  tracker.InvalidateRegion(address + 16, size - 32, [&] {
+    flushes++;
+    tracker.ForEachDownloadRange<true>(
+        address + 16, size - 32, [](uint64_t, uint64_t) noexcept {});
+  });
+  Check(flushes == 1 && !tracker.IsRegionGpuModified(address, size) &&
+            tracker.IsRegionCpuModified(address, size) && IsWritable(memory) &&
+            IsWritable(memory + size - 1),
+        "range invalidation did not batch ownership transfer across regions");
+
+  tracker.InvalidateRegion(address + 16, size - 32, [&] { flushes++; });
+  Check(flushes == 1,
+        "clean range invalidation unnecessarily requested a GPU flush");
+  tracker.UntrackMemory(address, size);
+  page_manager.OnGpuUnmap(address, size);
+  Check(VirtualFree(memory, 0, MEM_RELEASE) != 0,
+        "range invalidation VirtualFree failed");
+}
+
 void TestCpuDirtyUploadAndFault() {
   constexpr uintptr_t base = 0x0000000200010000ull;
   TrackerHarness harness;
@@ -725,7 +787,7 @@ void TestCpuDirtyUploadAndFault() {
   Check(IsWritable(memory),
         "explicit CPU dirty transition did not release the rearmed watch");
 
-  tracker.UnmapMemory(address, page_size * 2);
+  tracker.UntrackMemory(address, page_size * 2);
   Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
 }
 
@@ -753,7 +815,7 @@ void TestFaultDuringUploadRemainsDirty() {
       });
   Check(tracker.IsRegionCpuModified(address, page_size) && IsWritable(memory),
         "upload completion erased a racing CPU dirty transition");
-  tracker.UnmapMemory(address, page_size);
+  tracker.UntrackMemory(address, page_size);
   Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
 }
 
@@ -798,7 +860,7 @@ void TestNativeStoreDuringRangeEnumeration() {
             IsWritable(memory),
         "native store during range enumeration was lost");
 
-  tracker.UnmapMemory(address, page_size);
+  tracker.UntrackMemory(address, page_size);
   Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
 }
 
@@ -882,7 +944,7 @@ void TestFaultDuringDownloadSynchronization() {
             page_manager.IsTracked(address + page_size * 2),
         "uncontended dirty page did not retain its clean write watch");
 
-  tracker.UnmapMemory(address, page_size * 3);
+  tracker.UntrackMemory(address, page_size * 3);
 }
 
 void TestFaultAndExplicitDirtyRace() {
@@ -922,7 +984,7 @@ void TestFaultAndExplicitDirtyRace() {
               IsWritable(memory),
           "fault/explicit-dirty race lost dirty state or write access");
   }
-  tracker.UnmapMemory(address, page_size);
+  tracker.UntrackMemory(address, page_size);
   Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
 }
 
@@ -974,7 +1036,7 @@ void TestSharedTrackersAndConcurrentPageFaults() {
   }
 
   harness.first.UntrackMemory(address, page_size * 2);
-  harness.second.UnmapMemory(address, page_size * 2);
+  harness.second.UntrackMemory(address, page_size * 2);
   Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
 }
 
@@ -1007,7 +1069,7 @@ void TestGpuDirtyBits() {
         "explicit GPU dirty transition did not trap CPU access");
   tracker.UnmarkRegionAsGpuModified(address, page_size);
   tracker.MarkRegionAsCpuModified(address, page_size);
-  tracker.UnmapMemory(address, page_size * 2);
+  tracker.UntrackMemory(address, page_size * 2);
   Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
 }
 
@@ -1042,7 +1104,7 @@ void TestCrossRegionUpload() {
         "cross-region written upload did not mark GPU dirty state");
   tracker.UnmarkRegionAsGpuModified(boundary - page_size, page_size * 2);
   tracker.MarkRegionAsCpuModified(boundary - page_size, page_size * 2);
-  tracker.UnmapMemory(address, region_size * 2);
+  tracker.UntrackMemory(address, region_size * 2);
   Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
 }
 
@@ -1052,9 +1114,6 @@ void TestCrossRegionUpload() {
   auto &tracker = harness.tracker;
   auto &page_manager = harness.page_manager;
   const auto page_size = page_manager.GetPageSize();
-  if (std::strcmp(name, "unmapped") == 0) {
-    (void)tracker.IsRegionCpuModified(base, page_size);
-  }
   const auto allocation_size =
       std::strcmp(name, "missing-download-bytes") == 0 ? page_size * 2
                                                        : page_size;
@@ -1107,20 +1166,6 @@ void TestCrossRegionUpload() {
           }
     });
     fault.join();
-  } else if (std::strcmp(name, "gpu-dirty-unmap-race") == 0) {
-    g_unmap_contended.store(false, std::memory_order_release);
-    MemoryTracker::SetUnmapContentionHook(UnmapContended);
-    std::thread unmap;
-    tracker.ForEachUploadRange(
-        address, page_size, true, [](uint64_t, uint64_t) noexcept {},
-        [&]() noexcept {
-          unmap = std::thread(
-              [&] { tracker.UnmapMemory(address, page_size); });
-          while (!g_unmap_contended.load(std::memory_order_acquire)) {
-            std::this_thread::yield();
-          }
-        });
-    unmap.join();
   } else if (std::strcmp(name, "missing-download-bytes") == 0) {
     tracker.ForEachUploadRange(
         address, allocation_size, true, [](uint64_t, uint64_t) noexcept {},
@@ -1151,8 +1196,7 @@ void TestFatalPaths() {
 #endif
   for (const char *name : {"gpu-dirty-fault", "gpu-dirty-read", "virtual-gpu-read",
                            "gpu-dirty-explicit-cpu",
-                           "unmapped", "reentrant-upload",
-                           "writable-upload-race", "gpu-dirty-unmap-race",
+                           "reentrant-upload", "writable-upload-race",
                            "missing-download-bytes"}) {
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
     std::string command = std::string("\"") + path + "\" --death " + name;
@@ -1194,6 +1238,14 @@ void TestFatalPaths() {
 
 } // namespace
 
+namespace Libs::LibKernel::Memory {
+
+bool ProtectGuestHostMemory(uint64_t vaddr, uint64_t size, Common::VirtualMemory::Mode mode) {
+  return ProtectAddressSpace(vaddr, size, mode);
+}
+
+} // namespace Libs::LibKernel::Memory
+
 int main(int argc, char **argv) {
 #if 1
   if (argc == 3 && std::strcmp(argv[1], "--death") == 0) {
@@ -1207,6 +1259,8 @@ int main(int argc, char **argv) {
   TestSameSlabTrackerArbitration();
   TestSharedMetadataAndImagePageFault();
   TestRangeSet();
+  TestQueriesDoNotRequireMappedOwnership();
+  TestRangeInvalidation();
   TestGpuDirtyBits();
   TestCrossRegionUpload();
   TestFaultDuringUploadRemainsDirty();

tests/PageManagerTests.cpp

@@ -1,4 +1,5 @@
 #include "graphics/host_gpu/pageManager.h"
+#include "common/virtualMemory.h"
 
 #include <atomic>
 #include <cstdint>
@@ -28,7 +29,6 @@
 
 namespace {
 
-using Libs::Graphics::GpuAccess;
 using Libs::Graphics::PageFaultAccess;
 using Libs::Graphics::PageManager;
 
@@ -122,6 +122,23 @@ uint32_t Protection(const void *address) {
   return info.Protect;
 }
 #endif
+
+std::atomic_uint64_t g_protection_calls{0};
+
+bool ProtectAddressSpace(uint64_t vaddr, uint64_t size,
+                         Common::VirtualMemory::Mode mode) {
+  uint32_t protection = PAGE_NOACCESS;
+  if (mode == Common::VirtualMemory::Mode::Read) {
+    protection = PAGE_READONLY;
+  } else if (mode == Common::VirtualMemory::Mode::ReadWrite) {
+    protection = PAGE_READWRITE;
+  }
+  DWORD old_protection = 0;
+  g_protection_calls.fetch_add(1, std::memory_order_relaxed);
+  return VirtualProtect(reinterpret_cast<void *>(vaddr), size, protection,
+                        &old_protection) != 0;
+}
+
 #if 1
 
 struct FaultContext {
@@ -254,6 +271,7 @@ uint8_t *Allocate(uint64_t size, uint32_t protection = PAGE_READWRITE) {
 }
 
 void TestWatchFaultAndUnwatch() {
+  g_protection_calls.store(0, std::memory_order_relaxed);
   FaultContext context;
   PageManager manager(InvalidateFault, &context);
   context.manager = &manager;
@@ -266,6 +284,8 @@ void TestWatchFaultAndUnwatch() {
   Check(manager.IsTracked(reinterpret_cast<uint64_t>(memory)) &&
             !IsWritable(memory),
         "watch did not protect the page");
+  Check(g_protection_calls.load(std::memory_order_relaxed) != 0,
+        "watch protection bypassed the address-space owner callback");
   Check(manager.HandleFault(PageFaultAccess::Write,
                             reinterpret_cast<uint64_t>(memory + 32)),
         "tracked write fault was not handled");
@@ -301,36 +321,6 @@ void TestSharedWatcherFault() {
   Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
 }
 
-void TestMappedHostWriteRange() {
-  FaultContext context;
-  PageManager manager(InvalidateFault, &context);
-  context.manager = &manager;
-  const auto page_size = manager.GetPageSize();
-  auto *memory = Allocate(page_size * 3);
-  const auto address = reinterpret_cast<uint64_t>(memory);
-
-  manager.OnGpuMap(address, page_size);
-  manager.OnGpuMap(address + page_size * 2, page_size);
-  manager.UpdatePageWatchers(true, address, page_size);
-  manager.UpdatePageWatchers(true, address + page_size * 2, page_size);
-  Check(manager.HasAnyMapping(address + 16, page_size * 3 - 32),
-        "host-write range did not find partial GPU mappings");
-  Check(!manager.IsMapped(address, page_size * 3),
-        "partial GPU mappings were reported as a full mapping");
-  Check(manager.HandleWriteRange(address + 16, page_size * 3 - 32),
-        "mapped host-write range was not handled");
-  Check(context.calls.load(std::memory_order_relaxed) == 2,
-        "host-write range did not invalidate each mapped watched page");
-  Check(IsWritable(memory) && IsWritable(memory + page_size * 2),
-        "host-write range did not restore writable protection");
-
-  manager.OnGpuUnmap(address, page_size);
-  manager.OnGpuUnmap(address + page_size * 2, page_size);
-  Check(!manager.HasAnyMapping(address, page_size * 3),
-        "host-write range retained stale GPU mappings");
-  Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
-}
-
 void TestReadWriteWatcherFault() {
   FaultContext context;
   PageManager manager(InvalidateFault, &context);
@@ -373,14 +363,6 @@ void TestPermittedMappedLateFaultsResume() {
         "second delayed mapped write was not accepted");
   Check(manager.HandleFault(PageFaultAccess::Read, address),
         "delayed mapped read was not accepted on readable backing");
-  DWORD old_protection = 0;
-  Check(VirtualProtect(memory, page_size, PAGE_READONLY, &old_protection) != 0 &&
-            old_protection == PAGE_READWRITE,
-        "failed to prepare intentional read-only protection");
-  Check(!manager.HandleFault(PageFaultAccess::Write, address),
-        "intentional read-only mapping accepted a write fault");
-  Check(VirtualProtect(memory, page_size, PAGE_READWRITE, &old_protection) != 0,
-        "failed to restore writable protection");
   manager.OnGpuUnmap(address, page_size);
   Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
 }
@@ -526,31 +508,6 @@ void TestNativeAccessViolation() {
   Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
 }
 
-void TestInvalidLateWriteTokenIsConsumed() {
-  FaultContext context;
-  PageManager manager(InvalidateFault, &context);
-  context.manager = &manager;
-  const auto page_size = manager.GetPageSize();
-  auto *memory = Allocate(page_size);
-  const auto address = reinterpret_cast<uint64_t>(memory);
-  manager.OnGpuMap(address, page_size);
-  manager.UpdatePageWatchers(true, address, page_size);
-  Check(manager.HandleFault(PageFaultAccess::Write, address),
-        "initial write fault was not handled");
-  DWORD old_protection = 0;
-  Check(VirtualProtect(memory, page_size, PAGE_READONLY, &old_protection) !=
-                0 &&
-            old_protection == PAGE_READWRITE,
-        "failed to create invalid late-write protection state");
-  Check(!manager.HandleFault(PageFaultAccess::Write, address) &&
-            !manager.HandleFault(PageFaultAccess::Write, address),
-        "invalid late-write token was accepted or retained");
-  Check(VirtualProtect(memory, page_size, PAGE_READWRITE, &old_protection) != 0,
-        "failed to restore test protection");
-  manager.OnGpuUnmap(address, page_size);
-  Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
-}
-
 void TestCrossRegionRange() {
   FaultContext context;
   PageManager manager(InvalidateFault, &context);
@@ -577,6 +534,75 @@ void TestCrossRegionRange() {
   Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
 }
 
+void TestBatchedWatcherRanges() {
+  FaultContext context;
+  PageManager manager(InvalidateFault, &context);
+  context.manager = &manager;
+  const auto page_size = manager.GetPageSize();
+  constexpr uint64_t region_size = 4ull * 1024ull * 1024ull;
+  constexpr uint64_t allocation_size = region_size * 3;
+  auto *memory = Allocate(allocation_size);
+  const auto address = reinterpret_cast<uint64_t>(memory);
+
+  manager.OnGpuMap(address, allocation_size);
+
+  manager.UpdatePageWatchers(true, address + page_size, page_size);
+  manager.UpdatePageWatchers(true, address + page_size * 3, page_size);
+  manager.UpdatePageWatchers(true, address, page_size * 5);
+  for (uint64_t page = 0; page < 5; page++) {
+    Check(Protection(memory + page * page_size) == PAGE_READONLY,
+          "fragmented watch did not coalesce to read-only");
+  }
+  manager.UpdatePageWatchers(false, address, page_size * 5);
+  Check(IsWritable(memory) &&
+            Protection(memory + page_size) == PAGE_READONLY &&
+            IsWritable(memory + page_size * 2) &&
+            Protection(memory + page_size * 3) == PAGE_READONLY &&
+            IsWritable(memory + page_size * 4),
+        "fragmented unwatch lost overlapping watcher counts");
+  manager.UpdatePageWatchers(false, address + page_size, page_size);
+  manager.UpdatePageWatchers(false, address + page_size * 3, page_size);
+
+  manager.UpdatePageWatchers(true, address, allocation_size);
+  Check(!IsWritable(memory) &&
+            !IsWritable(memory + region_size) &&
+            !IsWritable(memory + region_size * 2) &&
+            !IsWritable(memory + allocation_size - page_size),
+        "large cross-region watch did not protect the full range");
+  manager.UpdatePageWatchers(false, address, allocation_size);
+  Check(IsWritable(memory) &&
+            IsWritable(memory + region_size) &&
+            IsWritable(memory + region_size * 2) &&
+            IsWritable(memory + allocation_size - page_size),
+        "large cross-region unwatch did not restore the full range");
+
+  manager.UpdatePageWatchers(true, address, page_size * 5);
+  manager.UpdatePageWatchers(true, address + page_size, page_size * 3,
+                             Libs::Graphics::PageWatchMode::ReadWrite);
+  Check(Protection(memory) == PAGE_READONLY &&
+            Protection(memory + page_size) == PAGE_NOACCESS &&
+            Protection(memory + page_size * 2) == PAGE_NOACCESS &&
+            Protection(memory + page_size * 3) == PAGE_NOACCESS &&
+            Protection(memory + page_size * 4) == PAGE_READONLY,
+        "mixed watcher modes installed incorrect protections");
+  manager.UpdatePageWatchers(false, address, page_size * 5);
+  Check(IsWritable(memory) &&
+            Protection(memory + page_size) == PAGE_NOACCESS &&
+            Protection(memory + page_size * 2) == PAGE_NOACCESS &&
+            Protection(memory + page_size * 3) == PAGE_NOACCESS &&
+            IsWritable(memory + page_size * 4),
+        "write unwatch incorrectly released read/write watchers");
+  manager.UpdatePageWatchers(false, address + page_size, page_size * 3,
+                             Libs::Graphics::PageWatchMode::ReadWrite);
+  Check(IsWritable(memory + page_size) &&
+            IsWritable(memory + page_size * 2) &&
+            IsWritable(memory + page_size * 3),
+        "read/write unwatch did not restore writable protection");
+
+  manager.OnGpuUnmap(address, allocation_size);
+  Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
+}
+
 [[noreturn]] void RunDeathCase(const char *name) {
   FaultContext context;
   auto manager = std::make_unique<PageManager>(InvalidateFault, &context);
@@ -588,9 +614,7 @@ void TestCrossRegionRange() {
     manager->UpdatePageWatchers(false, 0x1000, page_size);
   } else {
     const bool two_pages = std::strcmp(name, "cross-reentrant") == 0;
-    auto *memory = Allocate(
-        two_pages ? page_size * 2 : page_size,
-        std::strcmp(name, "protection") == 0 ? PAGE_READONLY : PAGE_READWRITE);
+    auto *memory = Allocate(two_pages ? page_size * 2 : page_size);
     const auto address = reinterpret_cast<uint64_t>(memory);
     manager->OnGpuMap(address, two_pages ? page_size * 2 : page_size);
     manager->UpdatePageWatchers(true, address, page_size);
@@ -620,9 +644,7 @@ void TestCrossRegionRange() {
       }
       (void)manager->HandleFault(PageFaultAccess::Read, address);
       first.join();
-    } else if (std::strcmp(name, "watched-unmap") == 0) {
-      manager->OnGpuUnmap(address, page_size);
-    } else if (std::strcmp(name, "protection") != 0) {
+    } else {
       std::_Exit(0x7f);
     }
   }
@@ -673,7 +695,7 @@ void TestFatalPaths() {
   for (const char *name :
        {"invalid-range", "unknown-untrack", "destructor-watch", "non-write",
         "callback-false", "reentrant", "cross-reentrant",
-        "concurrent-non-write", "watched-unmap", "protection"}) {
+        "concurrent-non-write"}) {
     CheckDeathCase(name);
   }
 }
@@ -734,51 +756,18 @@ void TestExternalDirtyTransferDuringResolution() {
   Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
 }
 
-void TestMappingDoesNotRequireCpuWriteAccess() {
-  FaultContext context;
-  PageManager manager(InvalidateFault, &context);
-  context.manager = &manager;
-  const auto page_size = manager.GetPageSize();
-  auto *memory = Allocate(page_size);
-  DWORD old_protection = 0;
-  Check(VirtualProtect(memory, page_size, PAGE_NOACCESS, &old_protection) != 0 &&
-            old_protection == PAGE_READWRITE,
-        "failed to prepare CPU-inaccessible mapping");
-  const auto address = reinterpret_cast<uint64_t>(memory);
-  manager.OnGpuMap(address, page_size);
-  Check(manager.IsMapped(address, page_size),
-        "CPU-inaccessible committed range was not GPU mapped");
-  manager.OnGpuUnmap(address, page_size);
-  Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
-}
-
-void TestGpuAccessPermissions() {
-  FaultContext context;
-  PageManager manager(InvalidateFault, &context);
-  context.manager = &manager;
-  const auto page_size = manager.GetPageSize();
-  auto *memory = Allocate(page_size);
-  const auto address = reinterpret_cast<uint64_t>(memory);
-  manager.OnGpuMap(address, page_size, GpuAccess::Read);
-  Check(manager.HasGpuAccess(address, page_size, GpuAccess::Read) &&
-            !manager.HasGpuAccess(address, page_size, GpuAccess::Write),
-        "read-only GPU mapping granted write access");
-  manager.OnGpuMap(address, page_size, GpuAccess::Write);
-  Check(manager.HasGpuAccess(address, page_size, GpuAccess::ReadWrite),
-        "overlapping GPU mappings did not combine permissions");
-  manager.OnGpuUnmap(address, page_size, GpuAccess::Read);
-  Check(!manager.HasGpuAccess(address, page_size, GpuAccess::Read) &&
-            manager.HasGpuAccess(address, page_size, GpuAccess::Write),
-        "GPU read unmap removed the wrong permission");
-  manager.OnGpuUnmap(address, page_size, GpuAccess::Write);
-  Check(!manager.IsMapped(address, page_size),
-        "GPU permission mappings were not fully balanced");
-  Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
-}
 #endif
 
 } // namespace
 
+namespace Libs::LibKernel::Memory {
+
+bool ProtectGuestHostMemory(uint64_t vaddr, uint64_t size, Common::VirtualMemory::Mode mode) {
+  return ProtectAddressSpace(vaddr, size, mode);
+}
+
+} // namespace Libs::LibKernel::Memory
+
 int main(int argc, char **argv) {
 #if 1
   if (argc == 3 && std::strcmp(argv[1], "--death") == 0) {
@@ -786,19 +775,16 @@ int main(int argc, char **argv) {
   }
   TestWatchFaultAndUnwatch();
   TestSharedWatcherFault();
-  TestMappedHostWriteRange();
   TestReadWriteWatcherFault();
   TestPermittedMappedLateFaultsResume();
   TestPartialMappingUnmapPreservesTokens();
   TestNativeDelayedReadAfterModeDowngrade();
   TestDelayedFaultAfterExplicitUnwatch();
   TestNativeAccessViolation();
-  TestInvalidLateWriteTokenIsConsumed();
   TestCrossRegionRange();
+  TestBatchedWatcherRanges();
   TestConcurrentFault();
   TestExternalDirtyTransferDuringResolution();
-  TestMappingDoesNotRequireCpuWriteAccess();
-  TestGpuAccessPermissions();
   TestFatalPaths();
   std::puts("PageManagerTests: all cases passed");
   return 0;

tests/ResourceTrackingTests.cpp

@@ -1217,6 +1217,34 @@ void TestResourceSpecializationIsTypedAndTransactional() {
                 Decoder::ImageDimension::Dim2DArray,
         "array MIMG intent did not produce a 2D-array view");
 
+  Program cube_view;
+  cube_view.stage = ShaderType::Compute;
+  cube_view.blocks.resize(1);
+  cube_view.blocks[0].instructions = {
+      ImageUse(0x24, Opcode::ImageLoad, ResourceKind::Image,
+               Decoder::ImageDimension::Dim2DArray)};
+  Prepare(cube_view);
+  auto cube_snapshot = array_2d_snapshot;
+  cube_snapshot.images[0].dwords[3] =
+      Prospero::GpuEnumValue(Prospero::ImageType::kCube) << 28u;
+  Check(SpecializeResources(cube_view, cube_snapshot, &error) &&
+            ValidateResourceSpecialization(cube_view, cube_snapshot, &error) &&
+            cube_view.info.images[0].cube &&
+            cube_view.blocks[0].instructions[0].memory.image_cube,
+        "cube descriptor identity did not reach the specialized image and IR");
+  auto array_after_cube = cube_snapshot;
+  array_after_cube.images[0].dwords[3] =
+      Prospero::GpuEnumValue(Prospero::ImageType::kColor2DArray) << 28u;
+  Check(!ValidateResourceSpecialization(cube_view, array_after_cube, &error),
+        "2D-array descriptor reused a cube-coordinate specialization");
+  auto null_after_cube = cube_snapshot;
+  null_after_cube.images[0].dwords.fill(0);
+  Check(SpecializeResources(cube_view, null_after_cube, &error) &&
+            ValidateResourceSpecialization(cube_view, null_after_cube, &error) &&
+            !cube_view.info.images[0].cube &&
+            !cube_view.blocks[0].instructions[0].memory.image_cube,
+        "canonical null respecialization retained stale cube-coordinate state");
+
   Program program;
   program.stage = ShaderType::Compute;
   program.blocks.resize(1);

tests/ScalarProvenanceTests.cpp

@@ -186,6 +186,39 @@ void TestCfgPhi() {
 	      "acyclic control-flow descriptor phi was not classified dynamic");
 }
 
+void TestNestedLoopPhiConvergence() {
+	Program program;
+	program.blocks.resize(4);
+	program.blocks[0].predecessors = {1};
+	program.blocks[0].successors   = {1};
+	program.blocks[1].predecessors = {0, 3};
+	program.blocks[1].successors   = {0, 2};
+	program.blocks[2].predecessors = {1};
+	program.blocks[2].successors   = {3};
+	program.blocks[3].predecessors = {2};
+	program.blocks[3].successors   = {1};
+	Instruction increment;
+	increment.op                     = Opcode::IAddU32;
+	increment.dst                    = Sgpr(0);
+	increment.src[0]                 = Sgpr(0);
+	increment.src[1]                 = Imm(1);
+	increment.src_count              = 2;
+	program.blocks[0].instructions   = {increment};
+	program.blocks[2].instructions   = {BufferUse(4, 0)};
+
+	std::string error;
+	Check(BuildScalarProvenance(program, &error), error.c_str());
+	const auto* source =
+	    GetDescriptorSource(program, program.blocks[2].instructions[0].memory.resource_source);
+	Check(source != nullptr, "nested-loop descriptor source was not attached");
+	const auto  value_id = source->dwords[0];
+	const auto& phi      = Value(program, value_id);
+	Check(phi.op == ScalarValueOp::Phi && phi.phi_args.size() == 2 &&
+	          ((phi.phi_args[0] == value_id && phi.phi_args[1] != value_id) ||
+	           (phi.phi_args[1] == value_id && phi.phi_args[0] != value_id)),
+	      "nested loop did not retain its recursive scalar provenance phi");
+}
+
 void TestDiamondReadPathsAreDynamic() {
 	std::array<uint32_t, 1> left  = {0x11111111u};
 	std::array<uint32_t, 1> right = {0x22222222u};
@@ -1045,6 +1078,7 @@ int main() {
 	try {
 		TestPerUseDescriptorDefinitions();
 		TestCfgPhi();
+		TestNestedLoopPhiConvergence();
 		TestDiamondReadPathsAreDynamic();
 		TestEquivalentConstantPhiIsStatic();
 		TestWideMoveInvalidatesAndCopiesBothDwords();

tests/ShaderRecompilerComputeTests.cpp

@@ -1274,6 +1274,59 @@ public:
     std::printf("[host]    %-32s ok\n", "SchedulerTimeline");
   }
 
+  void CheckGpuMappedRangeLifecycle() {
+    EnsureRuntimeContext();
+    CommandScheduler scheduler(Renderer(), m_runtime_context);
+    HW::Context registers{};
+    HW::UserConfig user_config{};
+    HW::Shader shaders{};
+    scheduler.Begin(registers, user_config, shaders);
+    Gpu gpu(Renderer());
+    GpuResourceManager resources(m_runtime_context, scheduler);
+    resources.SetGpu(&gpu);
+
+    constexpr uint64_t base = 0x0000000200000000ull;
+    constexpr uint64_t page = 0x4000;
+    resources.MapMemory(base, page * 4);
+    resources.MapMemory(base + page * 2, page * 4);
+    Require("GpuMappedRangeLifecycle", "union",
+            resources.IsMapped(base, page * 6) &&
+                !resources.IsMapped(base, page * 7),
+            "overlapping maps did not form one interval union");
+
+    resources.UnmapMemory(base + page * 2, page * 2);
+    Require("GpuMappedRangeLifecycle", "subtract",
+            resources.IsMapped(base, page * 2) &&
+                resources.IsMapped(base + page * 4, page * 2) &&
+                !resources.IsMapped(base, page * 6),
+            "partial unmap did not punch the expected interval hole");
+
+    resources.UnmapMemory(base + page * 2, page * 2);
+    Require("GpuMappedRangeLifecycle", "idempotent unmap",
+            resources.IsMapped(base, page * 2) &&
+                resources.IsMapped(base + page * 4, page * 2),
+            "unmapping an absent interval changed neighboring mappings");
+
+    resources.UnmapMemory(base, page * 6);
+    Require("GpuMappedRangeLifecycle", "clear",
+            !resources.IsMapped(base, page * 6),
+            "full unmap did not clear the interval union");
+
+    constexpr uint64_t old_prt = base + page * 8;
+    constexpr uint64_t new_prt = base + page * 16;
+    resources.MapMemory(old_prt, page * 4);
+    resources.UnmapMemory(old_prt, page * 4);
+    resources.MapMemory(new_prt, page * 6);
+    Require("GpuMappedRangeLifecycle", "PRT replacement",
+            !resources.IsMapped(old_prt, page * 4) &&
+                resources.IsMapped(new_prt, page * 6),
+            "old-unmap/new-map did not replace full PRT coverage");
+
+    resources.SetGpu(nullptr);
+    scheduler.Finish();
+    std::printf("[host]    %-32s ok\n", "GpuMappedRangeLifecycle");
+  }
+
   void CheckStreamBufferRing() {
     EnsureRuntimeContext();
     CommandScheduler scheduler(Renderer(), m_runtime_context);
@@ -1526,7 +1579,7 @@ public:
             fault_memory == reinterpret_cast<void *>(fault_base),
             "fixed processor-fault allocation failed");
     auto &resources = context.GetGpuResources();
-    resources.MapMemory(fault_base, fault_size, GpuAccess::ReadWrite);
+    resources.MapMemory(fault_base, fault_size);
 
     constexpr uint64_t immediate_dst = fault_base + 0x1000;
     constexpr uint64_t immediate_memory_dst = fault_base + 0x2000;
@@ -1679,9 +1732,11 @@ public:
       Require("GpuCommandLane", "processor fault context",
               Gpu::CurrentCommandProcessor() == &processor,
               "processor resource test lost its command context");
-      resources.PrepareHostWrite(fault_base, sizeof(uint32_t));
+      Require("GpuCommandLane", "processor memory invalidation",
+              resources.InvalidateMemory(fault_base, sizeof(uint32_t)),
+              "processor memory invalidation did not find its mapped range");
     });
-    resources.UnmapMemory(fault_base, fault_size, GpuAccess::ReadWrite);
+    resources.UnmapMemory(fault_base, fault_size);
     Require("GpuCommandLane", "processor fault unmap",
             Libs::LibKernel::Memory::KernelMunmap(fault_base, fault_size) == 0,
             "processor-fault direct-memory mapping release failed");
@@ -1951,7 +2006,7 @@ public:
       GpuResourceManager resources(m_runtime_context, scheduler);
       resources.SetGpu(&gpu);
       auto &cache = resources.GetBufferCache();
-      resources.MapMemory(base, allocation_size, GpuAccess::ReadWrite);
+      resources.MapMemory(base, allocation_size);
 
       const auto MarkGpuWrite = [&](uint64_t address, uint64_t size) {
         auto allocation =
@@ -2374,7 +2429,7 @@ public:
                                  sizeof(reacquire_value));
 
       resources.SetGpu(nullptr);
-      resources.UnmapMemory(base, allocation_size, GpuAccess::ReadWrite);
+      resources.UnmapMemory(base, allocation_size);
       scheduler.Finish();
     }
     gpu.Shutdown();
@@ -2429,7 +2484,7 @@ public:
               narrow_download != nullptr && narrow_download_offset % 4 == 0 &&
                   wide_download != nullptr && wide_download_offset % 16 == 0,
               "wide/block image readback was not aligned to its texel block");
-      resources.MapMemory(base, allocation_size, GpuAccess::ReadWrite);
+      resources.MapMemory(base, allocation_size);
 
       ImageDesc sampled{};
       sampled.type = BindingType::Texture;
@@ -3195,6 +3250,10 @@ public:
       const auto fault_b_image = texture_cache.FindImage(fault_b_desc);
       texture_cache.MarkGpuWritten(fault_a_image);
       texture_cache.MarkGpuWritten(fault_b_image);
+      Require(name, "per-image watcher installation",
+              texture_cache.GetImage(fault_a_image).IsTracked() &&
+                  texture_cache.GetImage(fault_b_image).IsTracked(),
+              "same-page images did not install independent write watchers");
       constexpr uint64_t padding_fault_offset = 0x8080;
       uint32_t write_only_read_a = 0;
       uint32_t write_only_read_b = 0;
@@ -3212,13 +3271,24 @@ public:
               resources.HandleFault(PageFaultAccess::Write,
                                     base + padding_fault_offset) &&
                   texture_cache.GetImage(fault_a_image).IsGpuModified() &&
-                  texture_cache.GetImage(fault_b_image).IsGpuModified(),
+                  texture_cache.GetImage(fault_b_image).IsGpuModified() &&
+                  !texture_cache.GetImage(fault_a_image).IsTracked() &&
+                  !texture_cache.GetImage(fault_b_image).IsTracked() &&
+                  texture_cache.GetImage(fault_a_image).IsMaybeCpuDirty() &&
+                  texture_cache.GetImage(fault_b_image).IsMaybeCpuDirty(),
               "a byte-disjoint CPU write discarded authoritative images");
+      const auto retracked_a = texture_cache.FindImage(fault_a_desc);
+      const auto retracked_b = texture_cache.FindImage(fault_b_desc);
       Require(name, "same-page image re-track",
-              texture_cache.FindImage(fault_a_desc) == fault_a_image &&
-                  texture_cache.FindImage(fault_b_desc) == fault_b_image &&
+              retracked_a == fault_a_image && retracked_b == fault_b_image &&
+                  texture_cache.GetImage(fault_a_image).IsTracked() &&
+                  texture_cache.GetImage(fault_b_image).IsTracked() &&
+                  !texture_cache.GetImage(fault_a_image).IsCpuDirty() &&
+                  !texture_cache.GetImage(fault_b_image).IsCpuDirty() &&
                   texture_cache.SynchronizeImageToBuffer(base + 0x8000,
                                                          sizeof(fault_a)) &&
+                  texture_cache.GetImage(fault_a_image).IsTracked() &&
+                  texture_cache.GetImage(fault_b_image).IsTracked() &&
                   !texture_cache.GetImage(fault_a_image).IsGpuModified() &&
                   texture_cache.GetImage(fault_b_image).IsGpuModified(),
               "retiring one same-page image lost the surviving owner");
@@ -4755,6 +4825,52 @@ public:
               "successive near-capacity image transfers replaced the shared "
               "download buffer");
 
+      constexpr uint64_t tile_alias_offset = 0x2000000;
+      constexpr uint64_t tile_alias_size = 0x400000;
+      constexpr uint32_t tile_alias_extent = 1024;
+      std::memset(memory + tile_alias_offset, 0,
+                  static_cast<size_t>(tile_alias_size));
+      auto render_target_alias = MakeLinearDesc(
+          base + tile_alias_offset, tile_alias_size,
+          vk::Format::eR8G8B8A8Unorm,
+          Prospero::GpuEnumValue(Prospero::BufferFormat::k8_8_8_8UNorm),
+          Prospero::ImageType::kColor2D,
+          {tile_alias_extent, tile_alias_extent, 1}, 1, 4, 1);
+      render_target_alias.type = BindingType::Storage;
+      render_target_alias.info.tile_mode =
+          Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget);
+      render_target_alias.view_info.usage =
+          vk::ImageUsageFlagBits::eStorage;
+      const auto render_target_alias_image =
+          texture_cache.FindImage(render_target_alias);
+
+      auto standard_4kb_alias = render_target_alias;
+      standard_4kb_alias.type = BindingType::Texture;
+      standard_4kb_alias.info.tile_mode =
+          Prospero::GpuEnumValue(Prospero::TileMode::kStandard4KB);
+      standard_4kb_alias.view_info.usage =
+          vk::ImageUsageFlagBits::eSampled;
+      const auto standard_4kb_alias_image =
+          texture_cache.FindImage(standard_4kb_alias);
+      auto repeated_standard_4kb_alias = standard_4kb_alias;
+      const auto repeated_standard_4kb_alias_image =
+          texture_cache.FindImage(repeated_standard_4kb_alias);
+      Require(
+          name, "equal-size tile-mode alias",
+          render_target_alias_image && standard_4kb_alias_image &&
+              standard_4kb_alias_image != render_target_alias_image &&
+              repeated_standard_4kb_alias_image ==
+                  standard_4kb_alias_image &&
+              texture_cache.GetImage(render_target_alias_image)
+                      .info.tile_mode ==
+                  Prospero::GpuEnumValue(
+                      Prospero::TileMode::kRenderTarget) &&
+              texture_cache.GetImage(standard_4kb_alias_image)
+                      .info.tile_mode ==
+                  Prospero::GpuEnumValue(
+                      Prospero::TileMode::kStandard4KB),
+          "equal address/size lookup reused an incompatible tiled backing");
+
       for (auto &output : ms_observer_outputs) {
         DestroyBuffer(&output);
       }
@@ -4771,7 +4887,7 @@ public:
       m_device.destroyShaderModule(ms_depth_module, nullptr);
 
       resources.SetGpu(nullptr);
-      resources.UnmapMemory(base, allocation_size, GpuAccess::ReadWrite);
+      resources.UnmapMemory(base, allocation_size);
       scheduler.Finish();
     }
     gpu.Shutdown();
@@ -4816,7 +4932,7 @@ public:
     scheduler.Begin(registers, user_config, shaders);
     {
       GpuResourceManager resources(m_runtime_context, scheduler);
-      resources.MapMemory(base, allocation_size, GpuAccess::ReadWrite);
+      resources.MapMemory(base, allocation_size);
       const uint32_t pitch = TileGetTexturePitch(format, 1, 1, tile);
       TileSizeAlign total{};
       TileSizeOffset mip{};
@@ -4915,7 +5031,7 @@ public:
                   std::vector<u32>{0x40004200u, 0x44003c00u},
               "tiled BGRA16 Buffer mirror changed guest component order");
       DestroyBuffer(&mirror_readback);
-      resources.UnmapMemory(base, allocation_size, GpuAccess::ReadWrite);
+      resources.UnmapMemory(base, allocation_size);
       scheduler.Finish();
     }
     Require(name, "unmap",
@@ -4962,7 +5078,7 @@ public:
       auto &resources = context.GetGpuResources();
       auto &texture_cache = resources.GetTextureCache();
       auto &executor = context.GetRenderExecutor();
-      resources.MapMemory(base, allocation_size, GpuAccess::ReadWrite);
+      resources.MapMemory(base, allocation_size);
 
       constexpr auto stencil_format =
           Prospero::GpuEnumValue(Prospero::BufferFormat::k8UInt);
@@ -5948,10 +6064,8 @@ public:
       const auto stale_ordered_color =
           texture_cache.FindImage(ordered_color_desc);
       RenderExecutorTestAccess::BindRenderTarget(executor, stale_ordered_color);
-      resources.UnmapMemory(ordered_color_address, target_mip_size,
-                            GpuAccess::ReadWrite);
-      resources.MapMemory(ordered_color_address, target_mip_size,
-                          GpuAccess::ReadWrite);
+      resources.UnmapMemory(ordered_color_address, target_mip_size);
+      resources.MapMemory(ordered_color_address, target_mip_size);
 
       auto ordered_depth_desc = depth;
       ordered_depth_desc.info.stencil = {ordered_color_address,
@@ -6104,7 +6218,7 @@ public:
               texture_cache.GetImage(depth_id).usage.storage,
           "storage stencil binding did not acquire the associated depth owner");
       RenderExecutorTestAccess::ResetBindings(executor);
-      resources.UnmapMemory(base, allocation_size, GpuAccess::ReadWrite);
+      resources.UnmapMemory(base, allocation_size);
       scheduler.Finish();
     }
 
@@ -12066,6 +12180,84 @@ TestCase BufferLoadVariants() {
            O::BufferLoadDwordx4, O::VMovB32, O::BufferStoreDword, O::SEndpgm}};
 }
 
+TestCase BufferLoadDwordx4SnapshotsOverlappingAddress() {
+  using O = ShaderOpcode;
+
+  std::vector<u32> code;
+  AppendVMovU32(&code, 21, 0);
+  AppendVMovU32(&code, 22, 0);
+  code.push_back(EncodeMubuf0(0x0eu, 0, true, true));
+  code.push_back(EncodeMubuf1(21, 0, 21));
+  for (u32 i = 0; i < 4; i++) {
+    AppendStoreVgpr(&code, 21 + i, 4 + i);
+  }
+  AppendEnd(&code);
+
+  TestCase test;
+  test.name = "BufferLoadDwordx4SnapshotsOverlappingAddress";
+  test.code = std::move(code);
+  test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u,
+                  0,           0,           0,           0};
+  test.expected = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u,
+                   0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u};
+  test.opcodes = {O::VMovB32, O::BufferLoadDwordx4, O::BufferStoreDword,
+                  O::SEndpgm};
+  test.user_data = MakeStructuredStorageBufferData(16, 2);
+  test.has_user_data = true;
+  return test;
+}
+
+TestCase BufferLoadDwordx2SnapshotsOverlappingAddress() {
+  using O = ShaderOpcode;
+
+  std::vector<u32> code;
+  AppendVMovU32(&code, 21, 0);
+  AppendVMovU32(&code, 22, 0);
+  code.push_back(EncodeMubuf0(0x0du, 0, true, true));
+  code.push_back(EncodeMubuf1(21, 0, 21));
+  for (u32 i = 0; i < 2; i++) {
+    AppendStoreVgpr(&code, 21 + i, 2 + i);
+  }
+  AppendEnd(&code);
+
+  TestCase test;
+  test.name = "BufferLoadDwordx2SnapshotsOverlappingAddress";
+  test.code = std::move(code);
+  test.initial = {0x11111111u, 0x22222222u, 0, 0};
+  test.expected = {0x11111111u, 0x22222222u, 0x11111111u, 0x22222222u};
+  test.opcodes = {O::VMovB32, O::BufferLoadDwordx2, O::BufferStoreDword,
+                  O::SEndpgm};
+  test.user_data = MakeStructuredStorageBufferData(8, 2);
+  test.has_user_data = true;
+  return test;
+}
+
+TestCase BufferLoadDwordx3SnapshotsOverlappingAddress() {
+  using O = ShaderOpcode;
+
+  std::vector<u32> code;
+  AppendVMovU32(&code, 21, 0);
+  AppendVMovU32(&code, 22, 0);
+  code.push_back(EncodeMubuf0(0x0fu, 0, true, true));
+  code.push_back(EncodeMubuf1(21, 0, 21));
+  for (u32 i = 0; i < 3; i++) {
+    AppendStoreVgpr(&code, 21 + i, 3 + i);
+  }
+  AppendEnd(&code);
+
+  TestCase test;
+  test.name = "BufferLoadDwordx3SnapshotsOverlappingAddress";
+  test.code = std::move(code);
+  test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0, 0, 0};
+  test.expected = {0x11111111u, 0x22222222u, 0x33333333u,
+                   0x11111111u, 0x22222222u, 0x33333333u};
+  test.opcodes = {O::VMovB32, O::BufferLoadDwordx3, O::BufferStoreDword,
+                  O::SEndpgm};
+  test.user_data = MakeStructuredStorageBufferData(12, 2);
+  test.has_user_data = true;
+  return test;
+}
+
 TestCase BufferStoreVariants() {
   using O = ShaderOpcode;
 
@@ -12134,6 +12326,69 @@ TestCase BufferFormatVariants() {
   return load;
 }
 
+TestCase BufferLoadFormatXyzwSnapshotsOverlappingAddress() {
+  using O = ShaderOpcode;
+
+  std::vector<u32> code;
+  AppendVMovU32(&code, 21, 0);
+  AppendVMovU32(&code, 22, 0);
+  code.push_back(EncodeMubuf0(0x03u, 0, true, true));
+  code.push_back(EncodeMubuf1(21, 0, 21));
+  for (u32 i = 0; i < 4; i++) {
+    AppendStoreVgpr(&code, 21 + i, 4 + i);
+  }
+  AppendEnd(&code);
+
+  TestCase test;
+  test.name = "BufferLoadFormatXyzwSnapshotsOverlappingAddress";
+  test.code = std::move(code);
+  test.initial = {0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u,
+                  0,           0,           0,           0};
+  test.expected = {0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u,
+                   0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u};
+  test.opcodes = {O::VMovB32, O::BufferLoadFormatXyzw, O::BufferStoreDword,
+                  O::SEndpgm};
+  test.user_data = MakeStructuredStorageBufferData(
+      16, 2, false,
+      BufferFormat(Prospero::BufferFormat::k32_32_32_32Float));
+  test.has_user_data = true;
+  return test;
+}
+
+TestCase BufferLoadFormatXyzwInactiveExecPreservesOverlappingAddress() {
+  using O = ShaderOpcode;
+
+  std::vector<u32> code;
+  AppendVMovLiteral(&code, 21, 0x11111111u);
+  AppendVMovLiteral(&code, 22, 0x22222222u);
+  AppendVMovLiteral(&code, 23, 0x33333333u);
+  AppendVMovLiteral(&code, 24, 0x44444444u);
+  code.push_back(EncodeSop1(0x04, 126, InlineU32(0)));
+  code.push_back(EncodeMubuf0(0x03u, 0, true, true));
+  code.push_back(EncodeMubuf1(21, 0, 21));
+  code.push_back(EncodeSMovB32(126, InlineU32(1)));
+  code.push_back(EncodeSMovB32(127, InlineU32(0)));
+  for (u32 i = 0; i < 4; i++) {
+    AppendStoreVgpr(&code, 21 + i, 4 + i);
+  }
+  AppendEnd(&code);
+
+  TestCase test;
+  test.name = "BufferLoadFormatXyzwInactiveExecPreservesOverlappingAddress";
+  test.code = std::move(code);
+  test.initial = {0xaaaaaaaa, 0xbbbbbbbb, 0xcccccccc, 0xdddddddd,
+                  0,          0,          0,          0};
+  test.expected = {0xaaaaaaaau, 0xbbbbbbbbu, 0xccccccccu, 0xddddddddu,
+                   0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u};
+  test.opcodes = {O::VMovB32, O::SMovB64, O::BufferLoadFormatXyzw,
+                  O::SMovB32, O::BufferStoreDword, O::SEndpgm};
+  test.user_data = MakeStructuredStorageBufferData(
+      16, 2, false,
+      BufferFormat(Prospero::BufferFormat::k32_32_32_32Float));
+  test.has_user_data = true;
+  return test;
+}
+
 TestCase BufferFormatStoreVariants() {
   using O = ShaderOpcode;
 
@@ -12442,6 +12697,63 @@ TestCase TBufferLoadVariants() {
            O::BufferStoreDword, O::SEndpgm}};
 }
 
+TestCase TBufferLoadFormatXyzwSnapshotsOverlappingAddress() {
+  using O = ShaderOpcode;
+
+  std::vector<u32> code;
+  AppendVMovU32(&code, 21, 0);
+  AppendVMovU32(&code, 22, 0);
+  constexpr auto format =
+      BufferFormat(Prospero::BufferFormat::k32_32_32_32Float);
+  code.push_back(
+      EncodeMtbuf0(0x03u, format & 0xfu, (format >> 4u) & 0x7u, 0, true, true));
+  code.push_back(EncodeMtbuf1(0x03u, 21, 0, 21));
+  for (u32 i = 0; i < 4; i++) {
+    AppendStoreVgpr(&code, 21 + i, 4 + i);
+  }
+  AppendEnd(&code);
+
+  TestCase test;
+  test.name = "TBufferLoadFormatXyzwSnapshotsOverlappingAddress";
+  test.code = std::move(code);
+  test.initial = {0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u,
+                  0,           0,           0,           0};
+  test.expected = {0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u,
+                   0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u};
+  test.opcodes = {O::VMovB32, O::TBufferLoadFormatXyzw,
+                  O::BufferStoreDword, O::SEndpgm};
+  test.user_data = MakeStructuredStorageBufferData(16, 2);
+  test.has_user_data = true;
+  return test;
+}
+
+TestCase TBufferLoadFormatXyzwPackedSnapshotsOverlappingAddress() {
+  using O = ShaderOpcode;
+
+  std::vector<u32> code;
+  AppendVMovU32(&code, 21, 0);
+  AppendVMovU32(&code, 22, 0);
+  constexpr auto format = BufferFormat(Prospero::BufferFormat::k8_8_8_8UInt);
+  code.push_back(
+      EncodeMtbuf0(0x03u, format & 0xfu, (format >> 4u) & 0x7u, 0, true, true));
+  code.push_back(EncodeMtbuf1(0x03u, 21, 0, 21));
+  for (u32 i = 0; i < 4; i++) {
+    AppendStoreVgpr(&code, 21 + i, 4 + i);
+  }
+  AppendEnd(&code);
+
+  TestCase test;
+  test.name = "TBufferLoadFormatXyzwPackedSnapshotsOverlappingAddress";
+  test.code = std::move(code);
+  test.initial = {0x44332211u, 0, 0, 0, 0, 0, 0, 0};
+  test.expected = {0x44332211u, 0, 0, 0, 0x11u, 0x22u, 0x33u, 0x44u};
+  test.opcodes = {O::VMovB32, O::TBufferLoadFormatXyzw,
+                  O::BufferStoreDword, O::SEndpgm};
+  test.user_data = MakeStructuredStorageBufferData(4, 8);
+  test.has_user_data = true;
+  return test;
+}
+
 TestCase TBufferStoreFormatX8UintWritesOneByte() {
   using O = ShaderOpcode;
 
@@ -13576,23 +13888,36 @@ TestCase DsAppendUsesEncodedGdsSelector() {
   using O = ShaderOpcode;
 
   std::vector<u32> code;
-  AppendSMovLiteral(&code, 124, 0x00000001u);
+  AppendSMovLiteral(&code, 124, 0x00000008u);
   code.push_back(EncodeDs0(0x3e, 0, true));
   code.push_back(EncodeDs1(0, 0, 0));
   code.push_back(EncodeDs0(0x3d, 0, true));
   code.push_back(EncodeDs1(1, 0, 0));
+  code.push_back(EncodeDs0(0x3e, 4, true));
+  code.push_back(EncodeDs1(2, 0, 0));
+  code.push_back(EncodeDs0(0x3d, 4, true));
+  code.push_back(EncodeDs1(3, 0, 0));
+  AppendSMovLiteral(&code, 124, 0x00080008u);
+  code.push_back(EncodeDs0(0x3e, 4, true));
+  code.push_back(EncodeDs1(4, 0, 0));
+  code.push_back(EncodeDs0(0x3d, 4, true));
+  code.push_back(EncodeDs1(5, 0, 0));
   AppendStoreVgpr(&code, 0, 0);
   AppendStoreVgpr(&code, 1, 1);
+  AppendStoreVgpr(&code, 2, 2);
+  AppendStoreVgpr(&code, 3, 3);
+  AppendStoreVgpr(&code, 4, 4);
+  AppendStoreVgpr(&code, 5, 5);
   AppendEnd(&code);
 
   TestCase test{
       "DsAppendGdsSelector",
       code,
       {},
-      {10, 74},
+      {10, 74, 20, 84, 40, 104},
       {O::SMovB32, O::DsAppend, O::DsConsume, O::BufferStoreDword, O::SEndpgm}};
-  test.gds_initial = {10};
-  test.expected_gds = {10};
+  test.gds_initial = {10, 20, 30, 40};
+  test.expected_gds = {10, 20, 30, 40};
   return test;
 }
 
@@ -14763,8 +15088,13 @@ std::vector<TestCase> MakeCases() {
   AddCase(BufferStoreDwordAppliesHostOffset);
   AddCase(BufferOffsetsUsePackedLaneAndStorageFallback);
   AddCase(BufferLoadVariants);
+  AddCase(BufferLoadDwordx2SnapshotsOverlappingAddress);
+  AddCase(BufferLoadDwordx3SnapshotsOverlappingAddress);
+  AddCase(BufferLoadDwordx4SnapshotsOverlappingAddress);
   AddCase(BufferStoreVariants);
   AddCase(BufferFormatVariants);
+  AddCase(BufferLoadFormatXyzwSnapshotsOverlappingAddress);
+  AddCase(BufferLoadFormatXyzwInactiveExecPreservesOverlappingAddress);
   AddCase(BufferFormatStoreVariants);
   AddCase(BufferStoreFormatXResource16UintWritesHalfword);
   AddCase(BufferLoadFormatXResource8UintZeroExtendsByte);
@@ -14778,6 +15108,8 @@ std::vector<TestCase> MakeCases() {
   AddCase(BufferStoreFormatXAddTidUsesLaneIndex);
   AddCase(BufferStoreFormatXDropsOutOfRangeRecord);
   AddCase(TBufferLoadVariants);
+  AddCase(TBufferLoadFormatXyzwSnapshotsOverlappingAddress);
+  AddCase(TBufferLoadFormatXyzwPackedSnapshotsOverlappingAddress);
   AddCase(TBufferLoadFormatX8UintZeroExtendsByte);
   AddCase(TBufferLoadFormatX8888UintExtractsFirstByte);
   AddCase(TBufferLoadFormatXIdxenUsesDescriptorStride);
@@ -15203,7 +15535,7 @@ void CheckRenderTargetFormatContract() {
       resource.kind = ShaderRecompiler::IR::ResourceKind::Image;
     } else if (std::strcmp(kind, "storage-no-write") == 0) {
       resource.written = false;
-    } else if (std::strcmp(kind, "storage-atomic") == 0) {
+    } else if (std::strcmp(kind, "storage-nonuint-atomic") == 0) {
       resource.atomic = true;
     } else if (std::strcmp(kind, "storage-compare") == 0) {
       resource.depth_compare = true;
@@ -15420,6 +15752,12 @@ void CheckSampledColorViews() {
   Require("SampledColorViews", "write-only uint 2D-array storage resource",
           IsSupportedStorageImageResource(storage_resource),
           "basic write-only uint 2D-array storage resource was rejected");
+  storage_resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D;
+  storage_resource.read = true;
+  storage_resource.atomic = true;
+  Require("SampledColorViews", "atomic uint 2D storage resource",
+          IsSupportedStorageImageResource(storage_resource),
+          "atomic uint storage resource was rejected");
 
   char path[MAX_PATH]{};
   Require("SampledColorViews", "host",
@@ -15429,7 +15767,7 @@ void CheckSampledColorViews() {
        {"sampled-invalid-selector", "sampled-incompatible-format",
         "sampled-invalid-high", "sampled-depth-format", "sampled-depth-swizzle",
         "storage-incompatible-format", "storage-kind", "storage-no-write",
-        "storage-atomic", "storage-compare", "storage-mip", "storage-dimension",
+        "storage-nonuint-atomic", "storage-compare", "storage-mip", "storage-dimension",
         "volume-mip-count", "volume-slice-range"}) {
     std::string command =
         std::string("\"") + path + "\" --image-view-death " + kind;
@@ -16138,6 +16476,19 @@ ShaderTextureResource BasicUintVolumeStorageTextureDescriptor() {
            0x00700000u, 0x00000000u, 0x00000000u}};
 }
 
+ShaderRecompiler::IR::ImageResource AtomicStorageTextureResource() {
+  auto resource = BasicLinearStorageTextureResource();
+  resource.kind = ShaderRecompiler::IR::ResourceKind::StorageImageUint;
+  resource.read = true;
+  resource.atomic = true;
+  return resource;
+}
+
+ShaderTextureResource AtomicStorageTextureDescriptor() {
+  return {{0x304bb700u, 0xc1400000u, 0x0000001fu, 0x91b00204u, 0x00000000u,
+           0x00700000u, 0x00000000u, 0x00000000u}};
+}
+
 [[noreturn]] void RunStorageTextureDescriptorDeathCase(const char *kind) {
   auto resource = BasicStorageTextureResource();
   auto descriptor = BasicStorageTextureDescriptor();
@@ -16199,6 +16550,12 @@ ShaderTextureResource BasicUintVolumeStorageTextureDescriptor() {
   } else if (std::strcmp(kind, "uint-resource-float-format") == 0) {
     resource = BasicUintArrayStorageTextureResource();
     descriptor = BasicArrayStorageTextureDescriptor();
+  } else if (std::strcmp(kind, "atomic-format") == 0) {
+    resource = AtomicStorageTextureResource();
+    descriptor = AtomicStorageTextureDescriptor();
+    descriptor.fields[1] =
+        (descriptor.fields[1] & ~0x1ff00000u) |
+        (Prospero::GpuEnumValue(Prospero::BufferFormat::k8UInt) << 20u);
   } else if (std::strcmp(kind, "depth-tile-read") == 0) {
     resource = Ppsa14053DepthTileStorageTextureResource();
     descriptor = Ppsa14053DepthTileStorageTextureDescriptor();
@@ -16282,7 +16639,7 @@ void CheckBasicStorageTextureDescriptor() {
       "PPSA06228 R11G11B10 storage descriptor fixture is malformed");
   ValidateStorageTexture(BasicBgraStorageTextureResource(), r11g11b10,
                          0x870000);
-  ValidateStorageColorView(vk::Format::eB10G11R11UfloatPack32,
+  ValidateStorageColorView(vk::Format::eB8G8R8A8Unorm,
                            vk::Format::eB10G11R11UfloatPack32,
                            r11g11b10.DstSelXYZW());
 
@@ -16573,6 +16930,18 @@ void CheckBasicStorageTextureDescriptor() {
           IsValidImageSwizzle(DstSel(4, 4, 4, 4)),
           "single-channel replicated destination selection was rejected");
 
+  const auto atomic = AtomicStorageTextureDescriptor();
+  Require("BasicStorageTexture", "atomic R32_UINT descriptor",
+          atomic.Width5() + 1u == 128 && atomic.Height5() + 1u == 1 &&
+              atomic.Depth() + 1u == 1 &&
+              atomic.Type() ==
+                  Prospero::GpuEnumValue(Prospero::ImageType::kColor2D) &&
+              atomic.Format() ==
+                  Prospero::GpuEnumValue(Prospero::BufferFormat::k32UInt) &&
+              atomic.DstSelXYZW() == DstSel(4, 0, 0, 1),
+          "PPSA22102 image-atomic descriptor fixture is malformed");
+  ValidateStorageTexture(AtomicStorageTextureResource(), atomic, 0x10000);
+
   char path[MAX_PATH]{};
   Require("BasicStorageTexture", "host",
           GetModuleFileNameA(nullptr, path, MAX_PATH) != 0,
@@ -16581,7 +16950,7 @@ void CheckBasicStorageTextureDescriptor() {
        {"resource", "type", "tile", "mip", "swizzle", "linear-rgb1-read",
         "bgra-read", "r16-float-read", "r8-unorm-read", "yzwx-read",
         "reserved-swizzle", "array-base-out-of-range", "array-mip-view",
-        "reserved", "uint-format", "uint-resource-float-format",
+        "reserved", "uint-format", "uint-resource-float-format", "atomic-format",
         "depth-tile-read", "depth-tile-extent", "depth-tile-fmask"}) {
     std::string command = std::string("\"") + path +
                           "\" --storage-texture-descriptor-death " + kind;
@@ -16880,12 +17249,12 @@ void CheckStandard64RenderTargetTileRoundTrip() {
 void CheckStorageTextureGpuOwnedRebindState() {
   constexpr uintptr_t base = 0x0000000200200000ull;
   constexpr uint64_t size = 0x10000;
-  auto *memory = static_cast<uint8_t *>(
-      VirtualAlloc(reinterpret_cast<void *>(base), size,
-                   MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE));
+  const auto guest_memory = Libs::LibKernel::Memory::AllocateRuntimeMemory(
+      base, size, Common::VirtualMemory::Mode::ReadWrite,
+      "storage_texture_gpu_owned_rebind", true);
+  auto *memory = reinterpret_cast<uint8_t *>(guest_memory);
   Require("StorageTextureGpuOwnedRebind", "allocation",
-          memory == reinterpret_cast<void *>(base),
-          "fixed VirtualAlloc failed");
+          guest_memory == base, "fixed guest-owner allocation failed");
   PageManager page_manager(CacheFault, nullptr);
   MemoryTracker tracker(page_manager);
   page_manager.OnGpuMap(base, size);
@@ -16897,7 +17266,6 @@ void CheckStorageTextureGpuOwnedRebindState() {
   Require(
       "StorageTextureGpuOwnedRebind", "owned",
       tracker.IsRegionGpuModified(base, size) &&
-          page_manager.IsMapped(base, size) &&
           (!HostMemoryQueryReadable(base, size, readable) || readable < size) &&
           HostMemoryQueryRange(base, size, HostMemoryAccess::Mapped, mapped) &&
           mapped == size &&
@@ -16943,7 +17311,8 @@ void CheckStorageTextureGpuOwnedRebindState() {
   tracker.UntrackMemory(base, size);
   page_manager.OnGpuUnmap(base, size);
   Require("StorageTextureGpuOwnedRebind", "free",
-          VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
+          Libs::LibKernel::Memory::FreeGuestMemory(base, size),
+          "guest-owner free failed");
   std::printf("[host]    %-32s ok\n", "StorageTextureGpuOwnedRebind");
 }
 #endif
@@ -17711,6 +18080,11 @@ int main(int argc, char **argv) {
     vulkan.CheckSchedulerTimeline();
     return 0;
   }
+  if (argc == 2 && std::strcmp(argv[1], "--mapped-range-only") == 0) {
+    VulkanHarness vulkan;
+    vulkan.CheckGpuMappedRangeLifecycle();
+    return 0;
+  }
   if (argc == 2 && std::strcmp(argv[1], "--stream-buffer-only") == 0) {
     VulkanHarness vulkan;
     vulkan.CheckStreamBufferRing();
@@ -17866,6 +18240,7 @@ int main(int argc, char **argv) {
   CheckEmbeddedFetchLaneSpill();
   CheckPs5GameExampleImageClearRuntimeShape();
   vulkan.CheckSchedulerTimeline();
+  vulkan.CheckGpuMappedRangeLifecycle();
   vulkan.CheckStreamBufferRing();
   vulkan.CheckCommandPoolGrowth();
   vulkan.CheckGpuTilerCpuParity();

tests/VirtualMemoryAllocationTests.cpp

@@ -1,15 +1,21 @@
 #include "common/commonSubsystem.h"
 #include "common/emulatorConfig.h"
+#include "common/file.h"
 #include "common/logging/log.h"
 #include "common/subsystems.h"
 #include "common/threads.h"
+#include "common/virtualMemory.h"
 #include "kernel/memory.h"
+#include "kernel/pthread.h"
 #include "libs/errno.h"
+#include "loader/runtimeLinker.h"
+#include "loader/systemContent.h"
 
 #include <cinttypes>
 #include <cstdint>
 #include <cstdio>
 #include <cstring>
+#include <filesystem>
 #include <string>
 
 namespace {
@@ -18,11 +24,16 @@ using Libs::LibKernel::Memory::VirtualQueryInfo;
 
 // Prospero ABI?
 constexpr uint64_t SceKernelPageSize             = 0x4000;
+constexpr uint64_t SceKernelTotalPhysicalSize    = 13824ull * 1024ull * 1024ull;
+constexpr uint64_t TestFlexibleMemorySize        = 3072ull * 1024ull * 1024ull;
 constexpr int      SceKernelProtCpuRead          = 0x01;
 constexpr int      SceKernelProtCpuRw            = 0x02;
+constexpr int      SceKernelProtCpuExec          = 0x04;
 constexpr int      SceKernelMapFixed             = 0x10;
 constexpr int      SceKernelMapNoOverwrite       = 0x80;
+constexpr int      SceKernelMapDmemCompat        = 0x400;
 constexpr int      SceKernelMapNoCoalesce        = 0x400000;
+constexpr int      SceKernelMapAligned64Kb       = 16 << 24;
 constexpr int      SceKernelVqFindNext           = 1;
 constexpr int      SceKernelMtypeC               = 11;
 constexpr uint64_t SceKernelDirectMemoryStart    = 0;
@@ -94,6 +105,29 @@ void InitSubsystems() {
 	Config::Load(options);
 
 	slist->Add(log, {core, config});
+	Check("InitSubsystems", slist->InitAll(false), "failed to initialize logging subsystem");
+
+	const auto param_json =
+	    std::filesystem::temp_directory_path() /
+	    ("kyty_virtual_memory_" +
+	     std::to_string(reinterpret_cast<uintptr_t>(&initialized)) + ".json");
+	constexpr char json[] = R"({"kernel":{"flexibleMemorySize":3221225472}})";
+	Common::File  param_file;
+	Check("InitSubsystems", param_file.Create(param_json), "failed to create temporary param.json");
+	uint32_t bytes_written = 0;
+	param_file.Write(json, sizeof(json) - 1, &bytes_written);
+	param_file.Close();
+	Check("InitSubsystems", bytes_written == sizeof(json) - 1,
+	      "failed to write temporary param.json");
+
+	Loader::SystemContentLoadParamSfo(param_json);
+	const auto flexible_memory_size = Loader::SystemContentGetFlexibleMemorySize();
+	Check("InitSubsystems", Common::File::DeleteFile(param_json),
+	      "failed to remove temporary param.json");
+	Check("InitSubsystems", flexible_memory_size == TestFlexibleMemorySize,
+	      "failed to read flexible memory size from param.json");
+	Libs::LibKernel::Memory::SetFlexibleMemorySize(flexible_memory_size);
+
 	slist->Add(memory, {core, log, thread});
 	Check("InitSubsystems", slist->InitAll(false), "failed to initialize memory subsystem");
 
@@ -131,6 +165,13 @@ size_t AvailableFlexibleMemory(const char* test) {
 	return size;
 }
 
+size_t ConfiguredFlexibleMemory(const char* test) {
+	size_t size = 0;
+	CheckOk(test, Libs::LibKernel::Memory::KernelConfiguredFlexibleMemorySize(&size),
+	        "KernelConfiguredFlexibleMemorySize");
+	return size;
+}
+
 uint64_t MapNamedFlexible(const char* test, uint64_t size, int prot, const char* name) {
 	void*     addr = nullptr;
 	const int ret =
@@ -190,6 +231,387 @@ void TestProsperoArgumentAndInfoSizeContracts() {
 	std::printf("[host]    %-48s ok\n", test);
 }
 
+void TestGuestAddressSpaceOwnsReservationsBeforeBacking() {
+	const char* test = "GuestAddressSpaceOwnsReservationsBeforeBacking";
+	void*       addr = nullptr;
+
+	Check(test, Libs::LibKernel::Memory::TestGuestBackingOutsideAddressSpace(),
+	      "boot-time shared backing alias overlaps an owned guest interval");
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelReserveVirtualRange(&addr, SceKernelPageSize, 0,
+	                                                           SceKernelPageSize),
+	        "KernelReserveVirtualRange");
+	const auto base = reinterpret_cast<uint64_t>(addr);
+	Check(test, Libs::LibKernel::Memory::TestGuestAddressRangeIsOwned(base, SceKernelPageSize),
+	      "guest reservation was allocated outside the early owner");
+	Check(test, Libs::LibKernel::Memory::TestPlaceholderRangeIsFree(base, SceKernelPageSize),
+	      "semantic reservation replaced the owner's placeholder");
+	Check(test,
+	      Libs::LibKernel::Memory::ProtectGuestHostMemory(
+	          base, SceKernelPageSize, Common::VirtualMemory::Mode::NoAccess),
+	      "owner rejected a sparse placeholder protection no-op");
+	CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(base, SceKernelPageSize), "KernelMunmap");
+	Check(test, Libs::LibKernel::Memory::TestPlaceholderRangeIsFree(base, SceKernelPageSize),
+	      "released semantic reservation escaped owner control");
+
+	std::printf("[host]    %-48s ok\n", test);
+}
+
+void TestGuestAddressSpaceHasNoFixedFallback() {
+	const char* test            = "GuestAddressSpaceHasNoFixedFallback";
+	const auto  unowned_address = reinterpret_cast<void*>(0x10000);
+	void*       addr            = unowned_address;
+
+	CheckFailed(test,
+	            Libs::LibKernel::Memory::KernelReserveVirtualRange(
+	                &addr, SceKernelPageSize, SceKernelMapFixed | SceKernelMapNoOverwrite,
+	                SceKernelPageSize),
+	            "KernelReserveVirtualRange(unowned fixed address)");
+	Check(test, reinterpret_cast<uint64_t>(addr) == 0x10000,
+	      "failed fixed reservation unexpectedly moved");
+
+	addr = unowned_address;
+	CheckFailed(test,
+	            Libs::LibKernel::Memory::KernelMapNamedFlexibleMemory(
+	                &addr, SceKernelPageSize, SceKernelProtCpuRw,
+	                SceKernelMapFixed | SceKernelMapNoOverwrite, "unowned_flexible"),
+	            "KernelMapNamedFlexibleMemory(unowned fixed address)");
+
+	int64_t phys_addr = -1;
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelAllocateDirectMemory(
+	            0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(), SceKernelPageSize,
+	            SceKernelPageSize, SceKernelMtypeC, &phys_addr),
+	        "KernelAllocateDirectMemory");
+	addr = unowned_address;
+	CheckFailed(test,
+	            Libs::LibKernel::Memory::KernelMapNamedDirectMemory(
+	                &addr, SceKernelPageSize, SceKernelProtCpuRw,
+	                SceKernelMapFixed | SceKernelMapNoOverwrite, phys_addr, SceKernelPageSize,
+	                "unowned_direct"),
+	            "KernelMapNamedDirectMemory(unowned fixed address)");
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelCheckedReleaseDirectMemory(phys_addr, SceKernelPageSize),
+	        "KernelCheckedReleaseDirectMemory");
+
+	std::printf("[host]    %-48s ok\n", test);
+}
+
+void TestGuestFreeRangeSearchDoesNotUnderflow() {
+	const char* test = "GuestFreeRangeSearchDoesNotUnderflow";
+
+	Check(test, Libs::LibKernel::Memory::TestGuestFreeRangeBounds(),
+	      "free-range containment accepted a candidate beyond the range end");
+
+	std::printf("[host]    %-48s ok\n", test);
+}
+
+void TestFlexibleMemoryCapacityIsBootFixed() {
+	const char* test       = "FlexibleMemoryCapacityIsBootFixed";
+	const auto  configured = ConfiguredFlexibleMemory(test);
+	const auto  baseline   = AvailableFlexibleMemory(test);
+	const auto  backing    = Libs::LibKernel::Memory::TestGuestBackingSize();
+
+	Check(test, configured == TestFlexibleMemorySize,
+	      "boot flexible pool did not use the param.json value");
+	Check(test, configured == baseline, "boot flexible pool did not start at configured capacity");
+	Check(test, backing == SceKernelTotalPhysicalSize,
+	      "boot backing is not the single 13.5 GiB physical file");
+	Check(test, backing == Libs::LibKernel::Memory::KernelGetDirectMemorySize() + configured,
+	      "direct and flexible regions do not partition the boot backing");
+
+	const auto address =
+	    MapNamedFlexible(test, SceKernelPageSize, SceKernelProtCpuRw, "boot_fixed_flexible");
+	Check(test, ConfiguredFlexibleMemory(test) == configured,
+	      "configured flexible capacity changed after allocation");
+	Check(test, Libs::LibKernel::Memory::TestGuestBackingSize() == backing,
+	      "shared backing size changed after allocation");
+	Check(test, AvailableFlexibleMemory(test) == baseline - SceKernelPageSize,
+	      "flexible allocation did not consume the boot-time pool");
+
+	CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(address, SceKernelPageSize),
+	        "KernelMunmap");
+	Check(test, ConfiguredFlexibleMemory(test) == configured,
+	      "configured flexible capacity changed after release");
+	Check(test, AvailableFlexibleMemory(test) == baseline,
+	      "flexible release did not restore the boot-time pool");
+
+	std::printf("[host]    %-48s ok\n", test);
+}
+
+void TestFlexibleMemoryUsesSharedBacking() {
+	const char* test     = "FlexibleMemoryUsesSharedBacking";
+	const auto  baseline = AvailableFlexibleMemory(test);
+	void*       address  = nullptr;
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelMapNamedFlexibleMemory(
+	            &address, SceKernelPageSize * 2, SceKernelProtCpuRw, 0, "shared_flexible"),
+	        "KernelMapNamedFlexibleMemory");
+	const auto base = reinterpret_cast<uint64_t>(address);
+	Check(test, Libs::LibKernel::Memory::TestGuestAddressRangeIsOwned(base, SceKernelPageSize * 2),
+	      "flexible mapping escaped the guest owner");
+
+	constexpr uint64_t first_value     = 0x464c45584241434bull; // "FLEXBACK"
+	constexpr uint64_t second_value    = 0x534841524544464cull; // "SHAREDFL"
+	*reinterpret_cast<uint64_t*>(base) = first_value;
+	uint64_t value                     = 0;
+	Check(test, Libs::LibKernel::Memory::TryReadBacking(base, &value, sizeof(value)),
+	      "TryReadBacking did not resolve flexible memory");
+	Check(test, value == first_value, "backing did not observe a flexible-memory CPU write");
+	Check(test,
+	      Libs::LibKernel::Memory::TryWriteBacking(base + SceKernelPageSize, &second_value,
+	                                               sizeof(second_value)),
+	      "TryWriteBacking did not resolve flexible memory");
+	Check(test, *reinterpret_cast<uint64_t*>(base + SceKernelPageSize) == second_value,
+	      "flexible-memory view did not observe a backing write");
+
+	CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(base, SceKernelPageSize * 2),
+	        "KernelMunmap");
+	Check(test, AvailableFlexibleMemory(test) == baseline,
+	      "flexible backing offsets were not returned to the boot-time pool");
+	Check(test, !Libs::LibKernel::Memory::TryReadBacking(base, &value, sizeof(value)),
+	      "unmapped flexible memory remained registered in the backing owner");
+
+	std::printf("[host]    %-48s ok\n", test);
+}
+
+void TestFlexibleDmemCompatAndAlignmentFlags() {
+	const char* test     = "FlexibleDmemCompatAndAlignmentFlags";
+	const auto  baseline = AvailableFlexibleMemory(test);
+	void*       address  = nullptr;
+
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelMapNamedFlexibleMemory(
+	            &address, SceKernelPageSize, SceKernelProtCpuRw,
+	            SceKernelMapDmemCompat | SceKernelMapAligned64Kb, "dmem_compat"),
+	        "KernelMapNamedFlexibleMemory(DMEM_COMPAT|ALIGNED_64KB)");
+	const auto base = reinterpret_cast<uint64_t>(address);
+	Check(test, (base & (0x10000 - 1u)) == 0, "SDK alignment flag was not honored");
+	const auto info = Query(test, base);
+	Check(test, info.is_flexible == 1 && info.is_stack == 0,
+	      "SCE_KERNEL_MAP_DMEM_COMPAT was misclassified as MAP_STACK");
+	Check(test, AvailableFlexibleMemory(test) + SceKernelPageSize == baseline,
+	      "DMEM_COMPAT mapping did not consume boot-time flexible backing");
+
+	void* stack_start = reinterpret_cast<void*>(UINT64_MAX);
+	void* stack_end   = reinterpret_cast<void*>(UINT64_MAX);
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelIsStack(reinterpret_cast<void*>(base), &stack_start,
+	                                               &stack_end),
+	        "KernelIsStack");
+	Check(test, stack_start == nullptr && stack_end == nullptr,
+	      "DMEM_COMPAT flexible mapping was reported as a stack");
+
+	CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(base, SceKernelPageSize),
+	        "KernelMunmap");
+	Check(test, AvailableFlexibleMemory(test) == baseline,
+	      "DMEM_COMPAT cleanup did not restore flexible capacity");
+
+	void* opaque = nullptr;
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelMapNamedFlexibleMemory(
+	            &opaque, SceKernelPageSize, SceKernelProtCpuRw, 0x8000, "opaque_runtime_flag"),
+	        "KernelMapNamedFlexibleMemory(opaque runtime flag)");
+	Check(test, AvailableFlexibleMemory(test) + SceKernelPageSize == baseline,
+	      "opaque runtime flag mapping did not consume boot-time flexible backing");
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelMunmap(reinterpret_cast<uint64_t>(opaque),
+	                                              SceKernelPageSize),
+	        "KernelMunmap(opaque runtime flag)");
+	Check(test, AvailableFlexibleMemory(test) == baseline,
+	      "opaque runtime flag cleanup did not restore flexible capacity");
+
+	void* invalid_flag = nullptr;
+	CheckFailed(
+	    test,
+	    Libs::LibKernel::Memory::KernelMapNamedFlexibleMemory(
+	        &invalid_flag, SceKernelPageSize, SceKernelProtCpuRw, 0x10000, "unsupported_flag"),
+	    "KernelMapNamedFlexibleMemory(unsupported flag)");
+
+	void* invalid_alignment = nullptr;
+	CheckFailed(test,
+	            Libs::LibKernel::Memory::KernelMapNamedFlexibleMemory(
+	                &invalid_alignment, SceKernelPageSize, SceKernelProtCpuRw, 13 << 24,
+	                "invalid_alignment"),
+	            "KernelMapNamedFlexibleMemory(invalid alignment)");
+
+	std::printf("[host]    %-48s ok\n", test);
+}
+
+void TestFlexibleNoCoalescePreservesBoundaries() {
+	const char* test     = "FlexibleNoCoalescePreservesBoundaries";
+	const auto  baseline = AvailableFlexibleMemory(test);
+	void*       reserve  = nullptr;
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelReserveVirtualRange(
+	            &reserve, SceKernelPageSize * 2, 0, SceKernelPageSize),
+	        "KernelReserveVirtualRange");
+	const auto base = reinterpret_cast<uint64_t>(reserve);
+
+	void* left = reinterpret_cast<void*>(base);
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelMapNamedFlexibleMemory(
+	            &left, SceKernelPageSize, SceKernelProtCpuRw,
+	            SceKernelMapFixed | SceKernelMapNoCoalesce, "no_coalesce"),
+	        "KernelMapNamedFlexibleMemory(left)");
+	void* right = reinterpret_cast<void*>(base + SceKernelPageSize);
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelMapNamedFlexibleMemory(
+	            &right, SceKernelPageSize, SceKernelProtCpuRw,
+	            SceKernelMapFixed | SceKernelMapNoCoalesce, "no_coalesce"),
+	        "KernelMapNamedFlexibleMemory(right)");
+
+	ExpectRange(test, Query(test, base), base, base + SceKernelPageSize, SceKernelProtCpuRw, 1, 0,
+	            0, 1, "no_coalesce");
+	ExpectRange(test, Query(test, base + SceKernelPageSize), base + SceKernelPageSize,
+	            base + SceKernelPageSize * 2, SceKernelProtCpuRw, 1, 0, 0, 1, "no_coalesce");
+
+	CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(base, SceKernelPageSize * 2),
+	        "KernelMunmap");
+	Check(test, AvailableFlexibleMemory(test) == baseline,
+	      "NO_COALESCE cleanup did not restore flexible capacity");
+
+	std::printf("[host]    %-48s ok\n", test);
+}
+
+void TestFlexibleMemoryReuseIsZeroFilled() {
+	const char* test     = "FlexibleMemoryReuseIsZeroFilled";
+	const auto  baseline = AvailableFlexibleMemory(test);
+	const auto  first =
+	    MapNamedFlexible(test, SceKernelPageSize, SceKernelProtCpuRw, "flexible_zero_source");
+	std::memset(reinterpret_cast<void*>(first), 0xa5, SceKernelPageSize);
+	CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(first, SceKernelPageSize),
+	        "KernelMunmap(source)");
+
+	const auto reused =
+	    MapNamedFlexible(test, SceKernelPageSize, SceKernelProtCpuRw, "flexible_zero_reuse");
+	const auto* bytes = reinterpret_cast<const uint8_t*>(reused);
+	Check(test,
+	      std::all_of(bytes, bytes + SceKernelPageSize, [](uint8_t value) { return value == 0; }),
+	      "reused flexible backing exposed stale bytes");
+	CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(reused, SceKernelPageSize),
+	        "KernelMunmap(reuse)");
+	Check(test, AvailableFlexibleMemory(test) == baseline,
+	      "zero-fill test leaked flexible backing capacity");
+
+	std::printf("[host]    %-48s ok\n", test);
+}
+
+void TestGuestStackUsesPrivateOwnerMemoryAndCache() {
+	const char* test     = "GuestStackUsesPrivateOwnerMemoryAndCache";
+	const auto  baseline = AvailableFlexibleMemory(test);
+	uint64_t    first    = 0;
+	uint64_t    second   = 0;
+	uint64_t    map_size = 0;
+
+	Check(test, Libs::LibKernel::TestGuestStackOwnerLifecycle(&first, &second, &map_size),
+	      "guest stack owner lifecycle failed");
+	Check(test, first != 0 && first == second, "guest stack cache did not reuse its owner mapping");
+	Check(test, map_size != 0 && (map_size & (SceKernelPageSize - 1u)) == 0,
+	      "guest stack mapping is not 16 KiB aligned");
+	Check(test, AvailableFlexibleMemory(test) == baseline,
+	      "private guest stack changed flexible backing capacity");
+
+	std::printf("[host]    %-48s ok\n", test);
+}
+
+void TestMainEntryUsesGuestStackAndDisablesHostChecks() {
+	const char* test = "MainEntryUsesGuestStackAndDisablesHostChecks";
+
+	Check(test, Loader::TestMainEntryUsesGuestStack(),
+	      "main-entry stack switch did not preserve the guest/host stack invariants");
+
+	std::printf("[host]    %-48s ok\n", test);
+}
+
+void TestFragmentedBackingUnmapRollback() {
+	const char* test     = "FragmentedBackingUnmapRollback";
+	const auto  baseline = AvailableFlexibleMemory(test);
+	const auto  left =
+	    MapNamedFlexible(test, SceKernelPageSize, SceKernelProtCpuRw, "backing_hole_left");
+	const auto blocker =
+	    MapNamedFlexible(test, SceKernelPageSize, SceKernelProtCpuRw, "backing_blocker");
+	const auto right =
+	    MapNamedFlexible(test, SceKernelPageSize, SceKernelProtCpuRw, "backing_hole_right");
+	CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(left, SceKernelPageSize),
+	        "KernelMunmap(left hole)");
+	CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(right, SceKernelPageSize),
+	        "KernelMunmap(right hole)");
+
+	const auto fragmented =
+	    MapNamedFlexible(test, SceKernelPageSize * 2, SceKernelProtCpuRw, "fragmented_backing");
+	auto* first_word = reinterpret_cast<uint64_t*>(fragmented);
+	auto* last_word =
+	    reinterpret_cast<uint64_t*>(fragmented + SceKernelPageSize * 2 - sizeof(uint64_t));
+	*first_word = 0x465241474c454654ull; // "FRAGLEFT"
+	*last_word  = 0x4652414752474854ull; // "FRAGRGHT"
+
+	Libs::LibKernel::Memory::TestFailGuestBackingStoreUnmapAfter(1);
+	CheckFailed(test, Libs::LibKernel::Memory::KernelMunmap(fragmented, SceKernelPageSize * 2),
+	            "KernelMunmap(injected second-view failure)");
+	ExpectRange(test, Query(test, fragmented), fragmented, fragmented + SceKernelPageSize * 2,
+	            SceKernelProtCpuRw, 1, 0, 0, 1, "fragmented_backing");
+	Check(test, *first_word == 0x465241474c454654ull && *last_word == 0x4652414752474854ull,
+	      "transactional backing-unmap rollback lost mapped contents");
+
+	CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(fragmented, SceKernelPageSize * 2),
+	        "KernelMunmap(retry)");
+	CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(blocker, SceKernelPageSize),
+	        "KernelMunmap(blocker)");
+	Check(test, AvailableFlexibleMemory(test) == baseline,
+	      "fragmented backing rollback test leaked flexible capacity");
+
+	std::printf("[host]    %-48s ok\n", test);
+}
+
+void TestRuntimeMemoryOwnerLifecycle() {
+	const char* test = "RuntimeMemoryOwnerLifecycle";
+	Check(test,
+	      Libs::LibKernel::Memory::AllocateRuntimeMemory(0x10000, SceKernelPageSize,
+	                                                     Common::VirtualMemory::Mode::ReadWrite,
+	                                                     "runtime_outside_owner", true) == 0,
+	      "fixed runtime allocation escaped the guest owner");
+
+	const auto base = Libs::LibKernel::Memory::AllocateRuntimeMemory(
+	    0, SceKernelPageSize * 2, Common::VirtualMemory::Mode::ReadWrite, "runtime_lifecycle");
+	Check(test, base != 0, "runtime allocation failed");
+	Check(test, Libs::LibKernel::Memory::TestGuestAddressRangeIsOwned(base, SceKernelPageSize * 2),
+	      "runtime allocation is outside the owner");
+	*reinterpret_cast<uint64_t*>(base) = 0x52554e54494d454full; // "RUNTIMEO"
+	Check(test,
+	      Libs::LibKernel::Memory::ProtectGuestMemory(base, SceKernelPageSize,
+	                                                  Common::VirtualMemory::Mode::Read),
+	      "runtime protection failed");
+	Check(test, Libs::LibKernel::Memory::FreeGuestMemory(base, SceKernelPageSize * 2),
+	      "runtime free failed");
+	Check(test, Libs::LibKernel::Memory::TestPlaceholderRangeIsFree(base, SceKernelPageSize * 2),
+	      "runtime free did not restore the owner placeholder");
+
+	const auto reused = Libs::LibKernel::Memory::AllocateRuntimeMemory(
+	    base, SceKernelPageSize * 2, Common::VirtualMemory::Mode::ReadWrite, "runtime_reuse", true);
+	Check(test, reused == base, "fixed runtime allocation did not reuse the owner placeholder");
+	Check(test, Libs::LibKernel::Memory::FreeGuestMemory(reused, SceKernelPageSize * 2),
+	      "reused runtime free failed");
+
+	const auto adjacent_first = Libs::LibKernel::Memory::AllocateRuntimeMemory(
+	    0, SceKernelPageSize, Common::VirtualMemory::Mode::ReadWrite, "runtime_adjacent_first");
+	Check(test, adjacent_first != 0, "first adjacent runtime allocation failed");
+	const auto adjacent_second = Libs::LibKernel::Memory::AllocateRuntimeMemory(
+	    adjacent_first + SceKernelPageSize, SceKernelPageSize,
+	    Common::VirtualMemory::Mode::ReadWrite, "runtime_adjacent_second", true);
+	Check(test, adjacent_second == adjacent_first + SceKernelPageSize,
+	      "second adjacent runtime allocation failed");
+	Check(test,
+	      Libs::LibKernel::Memory::FreeGuestMemory(adjacent_first, SceKernelPageSize * 2),
+	      "combined adjacent runtime free failed");
+	Check(test,
+	      Libs::LibKernel::Memory::TestPlaceholderRangeIsFree(adjacent_first,
+	                                                          SceKernelPageSize * 2),
+	      "combined adjacent runtime free did not restore one owner placeholder");
+
+	std::printf("[host]    %-48s ok\n", test);
+}
+
 void TestFlexibleMapQueryAndWholeMunmap() {
 	const char* test     = "FlexibleMapQueryAndWholeMunmap";
 	const auto  baseline = AvailableFlexibleMemory(test);
@@ -341,9 +763,11 @@ void TestDirectMapQueryOffsetAndPartialMunmap() {
 	            &addr, SceKernelPageSize * 4, SceKernelProtCpuRw, 0, phys_addr, SceKernelPageSize,
 	            "prospero_direct"),
 	        "KernelMapNamedDirectMemory");
-	const auto base  = reinterpret_cast<uint64_t>(addr);
-	const auto phys  = static_cast<uint64_t>(phys_addr);
-	void*      alias = nullptr;
+	const auto base = reinterpret_cast<uint64_t>(addr);
+	const auto phys = static_cast<uint64_t>(phys_addr);
+	Check(test, Libs::LibKernel::Memory::TestGuestAddressRangeIsOwned(base, SceKernelPageSize * 4),
+	      "direct mapping escaped the guest owner");
+	void* alias = nullptr;
 	CheckOk(test,
 	        Libs::LibKernel::Memory::KernelMapNamedDirectMemory(
 	            &alias, SceKernelPageSize * 4, SceKernelProtCpuRw, 0, phys_addr, SceKernelPageSize,
@@ -405,6 +829,10 @@ void TestDirectMapQueryOffsetAndPartialMunmap() {
 	      "TryReadBacking should reject a range crossing an unmapped span");
 	Check(test, rejected_read == transaction_sentinel,
 	      "failed backing reads must not modify a destination prefix");
+	Check(test,
+	      Libs::LibKernel::Memory::ClampRangeSize(base + SceKernelPageSize - 0xf30, 0x1560) ==
+	          0xf30,
+	      "ClampRangeSize did not stop at an unmapped span");
 
 	info = Query(test, base + SceKernelPageSize, SceKernelVqFindNext);
 	ExpectRange(test, info, base + SceKernelPageSize * 2, base + SceKernelPageSize * 4,
@@ -426,6 +854,192 @@ void TestDirectMapQueryOffsetAndPartialMunmap() {
 	std::printf("[host]    %-48s ok\n", test);
 }
 
+void TestDirectPartialProtectUnmapPreservesNeighbors() {
+	const char* test      = "DirectPartialProtectUnmapPreservesNeighbors";
+	const auto  size      = SceKernelPageSize * 3;
+	int64_t     phys_addr = 0;
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelAllocateDirectMemory(
+	            0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(), size, SceKernelPageSize,
+	            SceKernelMtypeC, &phys_addr),
+	        "KernelAllocateDirectMemory");
+
+	void* address = nullptr;
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelMapNamedDirectMemory(&address, size, SceKernelProtCpuRw,
+	                                                            0, phys_addr, SceKernelPageSize,
+	                                                            "partial_protect_direct"),
+	        "KernelMapNamedDirectMemory");
+	const auto base = reinterpret_cast<uint64_t>(address);
+	CheckOk(
+	    test,
+	    Libs::LibKernel::Memory::KernelMprotect(reinterpret_cast<void*>(base + SceKernelPageSize),
+	                                            SceKernelPageSize, SceKernelProtCpuRead),
+	    "KernelMprotect(middle)");
+	Check(test,
+	      Libs::LibKernel::Memory::ProtectGuestHostMemory(
+	          base, size, Common::VirtualMemory::Mode::Read),
+	      "owner could not protect fragmented backing views");
+	Check(test,
+	      Libs::LibKernel::Memory::ProtectGuestHostMemory(
+	          base, size, Common::VirtualMemory::Mode::ReadWrite),
+	      "owner could not restore fragmented backing views");
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelMunmap(base + SceKernelPageSize, SceKernelPageSize),
+	        "KernelMunmap(middle)");
+
+	Common::VirtualMemory::Mode old_left {};
+	Common::VirtualMemory::Mode old_right {};
+	Check(test,
+	      Common::VirtualMemory::Protect(base, SceKernelPageSize,
+	                                     Common::VirtualMemory::Mode::ReadWrite, &old_left),
+	      "could not inspect left-page protection");
+	Check(test,
+	      Common::VirtualMemory::Protect(base + SceKernelPageSize * 2, SceKernelPageSize,
+	                                     Common::VirtualMemory::Mode::ReadWrite, &old_right),
+	      "could not inspect right-page protection");
+	Check(test, old_left == Common::VirtualMemory::Mode::ReadWrite,
+	      "partial unmap changed the left neighbor protection");
+	Check(test, old_right == Common::VirtualMemory::Mode::ReadWrite,
+	      "partial unmap changed the right neighbor protection");
+	*reinterpret_cast<uint64_t*>(base) = 0x4c45465450524f54ull; // "LEFTPROT"
+	*reinterpret_cast<uint64_t*>(base + SceKernelPageSize * 2) =
+	    0x5247485450524f54ull; // "RGHTPROT"
+
+	CheckOk(test, Libs::LibKernel::Memory::KernelReleaseDirectMemory(phys_addr, size),
+	        "KernelReleaseDirectMemory");
+	ExpectUnmapped(test, base);
+	ExpectUnmapped(test, base + SceKernelPageSize * 2);
+
+	std::printf("[host]    %-48s ok\n", test);
+}
+
+void TestDirectMapValidationBeforeOwnerMutation() {
+	const char* test    = "DirectMapValidationBeforeOwnerMutation";
+	int64_t     invalid = -1;
+	CheckFailed(test,
+	            Libs::LibKernel::Memory::KernelAllocateDirectMemory(
+	                0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(), SceKernelPageSize + 1,
+	                SceKernelPageSize, SceKernelMtypeC, &invalid),
+	            "KernelAllocateDirectMemory(unaligned size)");
+	Check(test, invalid == -1, "invalid direct allocation changed the output address");
+	CheckFailed(test,
+	            Libs::LibKernel::Memory::KernelAllocateDirectMemory(
+	                0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(), SceKernelPageSize,
+	                0x1000, SceKernelMtypeC, &invalid),
+	            "KernelAllocateDirectMemory(sub-page alignment)");
+	Check(test, invalid == -1, "invalid alignment changed the output address");
+
+	int64_t phys_addr = 0;
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelAllocateDirectMemory(
+	            0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(), SceKernelPageSize * 2,
+	            SceKernelPageSize, SceKernelMtypeC, &phys_addr),
+	        "KernelAllocateDirectMemory");
+
+	auto expect_invalid = [&](size_t len, int prot, int flags, int64_t phys, size_t alignment,
+	                          const char* action) {
+		void* address = nullptr;
+		CheckFailed(test,
+		            Libs::LibKernel::Memory::KernelMapDirectMemory(&address, len, prot, flags, phys,
+		                                                           alignment),
+		            action);
+		Check(test, address == nullptr, "invalid direct map changed the output address");
+	};
+	expect_invalid(SceKernelPageSize + 1, SceKernelProtCpuRw, 0, phys_addr, SceKernelPageSize,
+	               "KernelMapDirectMemory(unaligned size)");
+	expect_invalid(SceKernelPageSize, SceKernelProtCpuRw, 0, phys_addr + 1, SceKernelPageSize,
+	               "KernelMapDirectMemory(unaligned physical address)");
+	expect_invalid(SceKernelPageSize, SceKernelProtCpuExec, 0, phys_addr, SceKernelPageSize,
+	               "KernelMapDirectMemory(executable)");
+
+	void* aligned = nullptr;
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelMapDirectMemory(
+	            &aligned, SceKernelPageSize, SceKernelProtCpuRw, 0, phys_addr, 0xc000),
+	        "KernelMapDirectMemory(16K-multiple alignment)");
+	Check(test, reinterpret_cast<uint64_t>(aligned) % 0xc000 == 0,
+	      "non-power-of-two 16K alignment was not honored");
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelMunmap(reinterpret_cast<uint64_t>(aligned),
+	                                              SceKernelPageSize),
+	        "KernelMunmap(16K-multiple alignment)");
+
+	void* ignored_flag = nullptr;
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelMapDirectMemory(&ignored_flag, SceKernelPageSize,
+	                                                       SceKernelProtCpuRw, 0x08, phys_addr,
+	                                                       SceKernelPageSize),
+	        "KernelMapDirectMemory(ignored flag)");
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelMunmap(reinterpret_cast<uint64_t>(ignored_flag),
+	                                              SceKernelPageSize),
+	        "KernelMunmap(ignored flag)");
+
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelReleaseDirectMemory(phys_addr, SceKernelPageSize * 2),
+	        "KernelReleaseDirectMemory");
+	std::printf("[host]    %-48s ok\n", test);
+}
+
+void TestDirectReleaseRollbackRestoresOwnerMapping() {
+	const char* test      = "DirectReleaseRollbackRestoresOwnerMapping";
+	int64_t     phys_addr = 0;
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelAllocateDirectMemory(
+	            0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(), SceKernelPageSize,
+	            SceKernelPageSize, SceKernelMtypeC, &phys_addr),
+	        "KernelAllocateDirectMemory");
+	void* address = nullptr;
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelMapNamedDirectMemory(
+	            &address, SceKernelPageSize, SceKernelProtCpuRw, 0, phys_addr, SceKernelPageSize,
+	            "release_rollback"),
+	        "KernelMapNamedDirectMemory");
+	const auto base                    = reinterpret_cast<uint64_t>(address);
+	*reinterpret_cast<uint64_t*>(base) = 0x52454c524f4c4c42ull; // "RELROLLB"
+
+	Libs::LibKernel::Memory::TestFailNextPhysicalMemoryUnmap();
+	CheckFailed(
+	    test,
+	    Libs::LibKernel::Memory::KernelCheckedReleaseDirectMemory(phys_addr, SceKernelPageSize),
+	    "KernelCheckedReleaseDirectMemory(injected failure)");
+	ExpectRange(test, Query(test, base), base, base + SceKernelPageSize, SceKernelProtCpuRw, 0, 1,
+	            0, 1, "release_rollback", static_cast<uint64_t>(phys_addr));
+	Check(test, *reinterpret_cast<uint64_t*>(base) == 0x52454c524f4c4c42ull,
+	      "release rollback lost the shared-backing contents");
+
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelCheckedReleaseDirectMemory(phys_addr, SceKernelPageSize),
+	        "KernelCheckedReleaseDirectMemory(retry)");
+	ExpectUnmapped(test, base);
+	std::printf("[host]    %-48s ok\n", test);
+}
+
+void TestDirectReleaseContracts() {
+	const char* test = "DirectReleaseContracts";
+	CheckOk(test, Libs::LibKernel::Memory::KernelReleaseDirectMemory(0, 0),
+	        "KernelReleaseDirectMemory(zero length)");
+	CheckOk(test, Libs::LibKernel::Memory::KernelCheckedReleaseDirectMemory(0, 0),
+	        "KernelCheckedReleaseDirectMemory(zero length)");
+	CheckFailed(test, Libs::LibKernel::Memory::KernelReleaseDirectMemory(1, SceKernelPageSize),
+	            "KernelReleaseDirectMemory(unaligned start)");
+	CheckFailed(test, Libs::LibKernel::Memory::KernelReleaseDirectMemory(0, SceKernelPageSize + 1),
+	            "KernelReleaseDirectMemory(unaligned size)");
+
+	const auto free_offset = static_cast<int64_t>(
+	    Libs::LibKernel::Memory::KernelGetDirectMemorySize() - SceKernelPageSize);
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelReleaseDirectMemory(free_offset, SceKernelPageSize),
+	        "KernelReleaseDirectMemory(unallocated range)");
+	Check(test,
+	      Libs::LibKernel::Memory::KernelCheckedReleaseDirectMemory(
+	          free_offset, SceKernelPageSize) == Libs::LibKernel::KERNEL_ERROR_ENOENT,
+	      "checked release did not report an unallocated range");
+
+	std::printf("[host]    %-48s ok\n", test);
+}
+
 void TestReleasedReserveCanBeReused() {
 	const char* test = "ReleasedReserveCanBeReused";
 	void*       addr = nullptr;
@@ -473,17 +1087,24 @@ void TestMunmapAcrossAdjacentFlexibleMappings() {
 	            &right, SceKernelPageSize, SceKernelProtCpuRw, SceKernelMapFixed, "adjacent_right"),
 	        "KernelMapNamedFlexibleMemory(right)");
 
+	Check(test,
+	      Libs::LibKernel::Memory::ClampRangeSize(base + SceKernelPageSize - 0x100, 0x200) == 0x200,
+	      "ClampRangeSize did not cross adjacent committed mappings");
+	Check(test,
+	      Libs::LibKernel::Memory::ProtectGuestHostMemory(
+	          base, SceKernelPageSize * 2, Common::VirtualMemory::Mode::Read),
+	      "owner could not protect adjacent backing mappings");
+	Check(test,
+	      Libs::LibKernel::Memory::ProtectGuestHostMemory(
+	          base, SceKernelPageSize * 2, Common::VirtualMemory::Mode::ReadWrite),
+	      "owner could not restore adjacent backing mappings");
+
 	CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(base, SceKernelPageSize * 2),
 	        "KernelMunmap(adjacent mappings)");
 	Check(test, AvailableFlexibleMemory(test) == baseline,
 	      "multi-range unmap leaked flexible-memory budget");
-	ExpectRange(test, Query(test, base), base, base + SceKernelPageSize, 0, 0, 0, 0, 0,
-	            "adjacent_left");
-	ExpectRange(test, Query(test, base + SceKernelPageSize), base + SceKernelPageSize,
-	            base + SceKernelPageSize * 2, 0, 0, 0, 0, 0, "adjacent_right");
-
-	CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(base, SceKernelPageSize * 2),
-	        "KernelMunmap(restored reserve)");
+	ExpectUnmapped(test, base);
+	ExpectUnmapped(test, base + SceKernelPageSize);
 
 	std::printf("[host]    %-48s ok\n", test);
 }
@@ -539,6 +1160,52 @@ void TestNonzeroDirectOffsetAliasesSharedBacking() {
 	std::printf("[host]    %-48s ok\n", test);
 }
 
+void TestDirectMapAcrossContiguousAllocations() {
+	const char* test   = "DirectMapAcrossContiguousAllocations";
+	const auto  end    = Libs::LibKernel::Memory::KernelGetDirectMemorySize();
+	int64_t     first  = 0;
+	int64_t     second = 0;
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelAllocateDirectMemory(
+	            0, end, SceKernelPageSize, SceKernelPageSize, SceKernelMtypeC, &first),
+	        "KernelAllocateDirectMemory(first)");
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelAllocateDirectMemory(
+	            0, end, SceKernelPageSize, SceKernelPageSize, SceKernelMtypeC, &second),
+	        "KernelAllocateDirectMemory(second)");
+	Check(test, second == first + static_cast<int64_t>(SceKernelPageSize),
+	      "test allocations are not physically contiguous");
+
+	void* mapping = nullptr;
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelMapNamedDirectMemory(
+	            &mapping, SceKernelPageSize * 2, SceKernelProtCpuRw, 0, first, SceKernelPageSize,
+	            "contiguous_allocations"),
+	        "KernelMapNamedDirectMemory");
+	auto* words = reinterpret_cast<uint64_t*>(mapping);
+	words[0]    = 0x434f4e5449474c46ull; // "CONTIGLF"
+	*reinterpret_cast<uint64_t*>(reinterpret_cast<uint64_t>(mapping) + SceKernelPageSize) =
+	    0x434f4e5449475254ull; // "CONTIGRT"
+
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelCheckedReleaseDirectMemory(first, SceKernelPageSize * 2),
+	        "KernelCheckedReleaseDirectMemory(contiguous span)");
+	ExpectUnmapped(test, reinterpret_cast<uint64_t>(mapping));
+
+	int64_t reclaimed = -1;
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelAllocateDirectMemory(
+	            0, end, SceKernelPageSize * 2, SceKernelPageSize, SceKernelMtypeC, &reclaimed),
+	        "KernelAllocateDirectMemory(reclaimed)");
+	Check(test, reclaimed == first, "released contiguous span was not coalesced");
+	CheckOk(
+	    test,
+	    Libs::LibKernel::Memory::KernelCheckedReleaseDirectMemory(reclaimed, SceKernelPageSize * 2),
+	    "KernelCheckedReleaseDirectMemory(reclaimed)");
+
+	std::printf("[host]    %-48s ok\n", test);
+}
+
 void TestDirectPhysicalFreeRangeReuseAndCoalescing() {
 	const char* test = "DirectPhysicalFreeRangeReuseAndCoalescing";
 	const auto  end  = Libs::LibKernel::Memory::KernelGetDirectMemorySize();
@@ -924,38 +1591,6 @@ void TestFixedReserveRollbackConsumesRestoredPlaceholder() {
 	std::printf("[host]    %-48s ok\n", test);
 }
 
-void TestFixedReserveRollbackRestoresDecommittedHostPages() {
-	const char*        test   = "FixedReserveRollbackRestoresDecommittedHostPages";
-	constexpr uint64_t size   = SceKernelPageSize * 3;
-	void*              mapped = nullptr;
-
-	CheckOk(test,
-	        Libs::LibKernel::Memory::KernelMapNamedFlexibleMemory(&mapped, size, SceKernelProtCpuRw,
-	                                                              0, "host_reserve_rollback"),
-	        "KernelMapNamedFlexibleMemory");
-	const auto base                    = reinterpret_cast<uint64_t>(mapped);
-	*reinterpret_cast<uint64_t*>(base) = 0x4b595459484f5354ull; // "KYTYHOST"
-	*reinterpret_cast<uint64_t*>(base + SceKernelPageSize * 2) =
-	    0x4b5954595441494cull; // "KYTYTAIL"
-
-	Libs::LibKernel::Memory::TestFailHostReservationAfter(1);
-	void* replacement = mapped;
-	CheckFailed(
-	    test,
-	    Libs::LibKernel::Memory::KernelReserveVirtualRange(
-	        &replacement, size, SceKernelMapFixed | SceKernelMapNoCoalesce, SceKernelPageSize),
-	    "KernelReserveVirtualRange(partial host reservation)");
-	Check(test, *reinterpret_cast<uint64_t*>(base) == 0x4b595459484f5354ull,
-	      "rollback did not restore the first flexible page");
-	Check(test, *reinterpret_cast<uint64_t*>(base + SceKernelPageSize * 2) == 0x4b5954595441494cull,
-	      "rollback damaged the flexible tail page");
-	ExpectRange(test, Query(test, base), base, base + size, SceKernelProtCpuRw, 1, 0, 0, 1,
-	            "host_reserve_rollback");
-
-	CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(base, size), "KernelMunmap");
-	std::printf("[host]    %-48s ok\n", test);
-}
-
 void TestFixedReserveRangeAddRollbackKeepsPlaceholder() {
 	const char*        test      = "FixedReserveRangeAddRollbackKeepsPlaceholder";
 	constexpr uint64_t size      = SceKernelPageSize * 4;
@@ -1046,8 +1681,10 @@ void TestLargeHintedReserveHostsSmallDirectMap() {
 	CheckOk(test, Libs::LibKernel::Memory::KernelReleaseDirectMemory(phys, SceKernelPageSize * 2),
 	        "KernelReleaseDirectMemory");
 	CheckOk(test,
-	        Libs::LibKernel::Memory::KernelMunmap(reinterpret_cast<uint64_t>(window), window_size),
-	        "KernelMunmap(window reserve)");
+	        Libs::LibKernel::Memory::KernelMunmap(reinterpret_cast<uint64_t>(window) +
+	                                                  SceKernelPageSize * 2,
+	                                              window_size - SceKernelPageSize * 2),
+	        "KernelMunmap(window reserve remainder)");
 	CheckOk(test,
 	        Libs::LibKernel::Memory::KernelMunmap(reinterpret_cast<uint64_t>(arena), arena_size),
 	        "KernelMunmap(arena reserve)");
@@ -1144,18 +1781,25 @@ void TestProsperoSampleMemoryPoolExpandCommit() {
 	                                                        SceKernelProtCpuRw, 0),
 	        "KernelMemoryPoolCommit");
 	ExpectRange(test, Query(test, base), base, base + commit_len, SceKernelProtCpuRw, 0, 0, 1, 1);
+	Check(test, Libs::LibKernel::Memory::TestGuestAddressRangeIsOwned(base, commit_len),
+	      "pooled commit escaped the guest owner");
 	Check(test, AvailableFlexibleMemory(test) == flexible_baseline,
 	      "pooled commit consumed flexible memory instead of expanded direct "
 	      "backing");
 	CheckFailed(test,
-	            Libs::LibKernel::Memory::KernelReleaseDirectMemory(pool_offset,
-	                                                               SceKernelMemoryPoolExpandLen),
-	            "KernelReleaseDirectMemory(committed pool expansion)");
+	            Libs::LibKernel::Memory::KernelCheckedReleaseDirectMemory(
+	                pool_offset, SceKernelMemoryPoolExpandLen),
+	            "KernelCheckedReleaseDirectMemory(committed pool expansion)");
 
 	constexpr uint64_t first_value     = 0x504f4f4c4241434bull; // "POOLBACK"
 	constexpr uint64_t second_value    = 0x5348415245444d45ull; // "SHAREDME"
 	*reinterpret_cast<uint64_t*>(base) = first_value;
 	*reinterpret_cast<uint64_t*>(base + SceKernelMemoryPoolCommitLen) = second_value;
+	uint64_t backing_read                                             = 0;
+	Check(test, Libs::LibKernel::Memory::TryReadBacking(base, &backing_read, sizeof(backing_read)),
+	      "TryReadBacking did not resolve pooled memory");
+	Check(test, backing_read == first_value,
+	      "shared backing did not observe a pooled-memory CPU write");
 
 	CheckOk(
 	    test,
@@ -1267,8 +1911,10 @@ void TestFragmentedMemoryPoolBacking() {
 	        "KernelMemoryPoolCommit(fragmented recommit)");
 	CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(base, commit_len),
 	        "KernelMunmap(fragmented commit)");
-	CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(base, SceKernelMemoryPoolReserveLen),
-	        "KernelMunmap(fragmented reserve cleanup)");
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelMunmap(base + commit_len,
+	                                              SceKernelMemoryPoolReserveLen - commit_len),
+	        "KernelMunmap(fragmented reserve remainder)");
 
 	CheckOk(test,
 	        Libs::LibKernel::Memory::KernelReleaseDirectMemory(first_pool,
@@ -1418,22 +2064,32 @@ void TestMemoryPoolCommitDecommitQueryFlags() {
 	std::printf("[host]    %-48s ok\n", test);
 }
 
-void TestProgramMemoryRegistrationAndProtection() {
-	const char* test = "ProgramMemoryRegistrationAndProtection";
+void TestProgramMemoryAllocationAndProtection() {
+	const char* test = "ProgramMemoryAllocationAndProtection";
 	const auto  size = SceKernelPageSize * 3;
-	const auto base = Common::VirtualMemory::Alloc(0, size, Common::VirtualMemory::Mode::ReadWrite);
-	Check(test, base != 0, "program host allocation failed");
-
-	Libs::LibKernel::Memory::RegisterProgramMemory(
-	    base, size, Common::VirtualMemory::Mode::ReadWrite, "program_test");
+	const auto  base = Libs::LibKernel::Memory::AllocateProgramMemory(
+	    0x900000000, size, Common::VirtualMemory::Mode::ReadWrite, "program_test");
+	Check(test, base != 0, "program guest allocation failed");
+	Check(test, Libs::LibKernel::Memory::TestGuestAddressRangeIsOwned(base, size),
+	      "program allocation escaped the guest owner");
 	ExpectRange(test, Query(test, base), base, base + size,
 	            SceKernelProtCpuRead | SceKernelProtCpuRw, 0, 0, 0, 1, "program_test");
 
-	Libs::LibKernel::Memory::UpdateProgramMemoryProtection(base, SceKernelPageSize,
-	                                                       Common::VirtualMemory::Mode::Read);
+	Check(test,
+	      Libs::LibKernel::Memory::ProtectGuestMemory(base, SceKernelPageSize,
+	                                                  Common::VirtualMemory::Mode::Read),
+	      "ProtectGuestMemory(first page) failed");
 	ExpectRange(test, Query(test, base), base, base + SceKernelPageSize, SceKernelProtCpuRead, 0, 0,
 	            0, 1, "program_test");
 
+	Common::VirtualMemory::Mode previous_mode = Common::VirtualMemory::Mode::NoAccess;
+	Check(test,
+	      Libs::LibKernel::Memory::ProtectGuestMemory(
+	          base, SceKernelPageSize, Common::VirtualMemory::Mode::ReadWrite, &previous_mode),
+	      "ProtectGuestMemory(tracked restore) failed");
+	Check(test, previous_mode == Common::VirtualMemory::Mode::Read,
+	      "semantic guest protection did not preserve its tracked old mode");
+
 	CheckOk(test,
 	        Libs::LibKernel::Memory::KernelMprotect(
 	            reinterpret_cast<void*>(base + SceKernelPageSize - 0x10), 0x20,
@@ -1442,24 +2098,44 @@ void TestProgramMemoryRegistrationAndProtection() {
 	ExpectRange(test, Query(test, base), base, base + size,
 	            SceKernelProtCpuRead | SceKernelProtCpuRw, 0, 0, 0, 1, "program_test");
 
-	Libs::LibKernel::Memory::UpdateProgramMemoryProtection(
-	    base + SceKernelPageSize * 2, SceKernelPageSize, Common::VirtualMemory::Mode::Read);
+	Check(test,
+	      Libs::LibKernel::Memory::ProtectGuestMemory(
+	          base + SceKernelPageSize * 2, SceKernelPageSize, Common::VirtualMemory::Mode::Read),
+	      "ProtectGuestMemory(last page) failed");
 	ExpectRange(test, Query(test, base + SceKernelPageSize * 2), base + SceKernelPageSize * 2,
 	            base + size, SceKernelProtCpuRead, 0, 0, 0, 1, "program_test");
 
-	Libs::LibKernel::Memory::UnregisterProgramMemory(base, size);
+	Check(test, Libs::LibKernel::Memory::FreeGuestMemory(base, size), "program guest free failed");
 	ExpectUnmapped(test, base);
-	Check(test, Common::VirtualMemory::Free(base), "program host free failed");
 
 	std::printf("[host]    %-48s ok\n", test);
 }
 
+void TestModuleRelocationUsesWritableHostMapping() {
+	const char* test = "ModuleRelocationUsesWritableHostMapping";
+	Check(test, Loader::TestModuleRelocationUsesWritableHostMapping(),
+	      "module relocation did not retain writable host memory and semantic guest protection");
+	std::printf("[host]    %-48s ok\n", test);
+}
+
 } // namespace
 
 int main() {
 	InitSubsystems();
 
 	RunTest(TestProsperoArgumentAndInfoSizeContracts);
+	RunTest(TestGuestAddressSpaceOwnsReservationsBeforeBacking);
+	RunTest(TestGuestAddressSpaceHasNoFixedFallback);
+	RunTest(TestGuestFreeRangeSearchDoesNotUnderflow);
+	RunTest(TestFlexibleMemoryCapacityIsBootFixed);
+	RunTest(TestFlexibleMemoryUsesSharedBacking);
+	RunTest(TestFlexibleDmemCompatAndAlignmentFlags);
+	RunTest(TestFlexibleNoCoalescePreservesBoundaries);
+	RunTest(TestFlexibleMemoryReuseIsZeroFilled);
+	RunTest(TestGuestStackUsesPrivateOwnerMemoryAndCache);
+	RunTest(TestMainEntryUsesGuestStackAndDisablesHostChecks);
+	RunTest(TestFragmentedBackingUnmapRollback);
+	RunTest(TestRuntimeMemoryOwnerLifecycle);
 	RunTest(TestFlexibleMapQueryAndWholeMunmap);
 	RunTest(TestPartialFlexibleMunmapAndFindNext);
 	RunTest(TestReserveMapFixedAndNoOverwrite);
@@ -1467,7 +2143,12 @@ int main() {
 	RunTest(TestReleasedReserveCanBeReused);
 	RunTest(TestMunmapAcrossAdjacentFlexibleMappings);
 	RunTest(TestDirectMapQueryOffsetAndPartialMunmap);
+	RunTest(TestDirectPartialProtectUnmapPreservesNeighbors);
+	RunTest(TestDirectMapValidationBeforeOwnerMutation);
+	RunTest(TestDirectReleaseRollbackRestoresOwnerMapping);
+	RunTest(TestDirectReleaseContracts);
 	RunTest(TestNonzeroDirectOffsetAliasesSharedBacking);
+	RunTest(TestDirectMapAcrossContiguousAllocations);
 	RunTest(TestDirectPhysicalFreeRangeReuseAndCoalescing);
 	RunTest(TestDirectAlignmentStaysWithinSearchRange);
 	RunTest(TestDefaultDirectMapUsesSystemAddressRange);
@@ -1476,7 +2157,6 @@ int main() {
 	RunTest(TestFixedReserveReplacesPartialDirectMapping);
 	RunTest(TestFixedReserveRollbackConsumesRestoredPlaceholder);
 	RunTest(TestFixedReserveRollbackSkipsUntouchedChunks);
-	RunTest(TestFixedReserveRollbackRestoresDecommittedHostPages);
 	RunTest(TestFixedReserveRangeAddRollbackKeepsPlaceholder);
 	RunTest(TestLargeHintedReserveHostsSmallDirectMap);
 	RunTest(TestMemoryPoolAlignmentContracts);
@@ -1484,7 +2164,8 @@ int main() {
 	RunTest(TestFragmentedMemoryPoolBacking);
 	RunTest(TestMemoryPoolMultiRangeDecommit);
 	RunTest(TestMemoryPoolCommitDecommitQueryFlags);
-	RunTest(TestProgramMemoryRegistrationAndProtection);
+	RunTest(TestProgramMemoryAllocationAndProtection);
+	RunTest(TestModuleRelocationUsesWritableHostMapping);
 
 	if (g_failed_tests != 0) {
 		std::printf("VirtualMemoryAllocationTests: %d case(s) failed\n", g_failed_tests);

tests/shaderCfgTests.cpp

@@ -3020,6 +3020,37 @@ void TestNewShaderRecompilerImageQueryLowering() {
 	CheckSpirvBinaryValidates(result.spirv);
 }
 
+void TestNewShaderRecompilerCubeSampleCoordinates() {
+	constexpr uint32_t MimgDimCube = 3;
+	const uint32_t shader[] = {
+	    EncodeMimg0(0x20, 0xf, false, MimgDimCube),
+	    EncodeMimg1(0, 0, 1, 0), // image_sample cube
+	    EncodeMimg0(0x60, 0x3, false, MimgDimCube),
+	    EncodeMimg1(8, 0, 1, 4), // image_get_lod cube
+	    0xbf810000u,
+	};
+
+	auto user_data = ImageTestUserData(Prospero::ImageType::kCube);
+	ShaderRecompiler::CompileOptions options;
+	options.stage     = ShaderType::Compute;
+	options.user_data = user_data.data();
+
+	ShaderRecompiler::CompileResult result;
+	std::string                     error;
+	Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str());
+	Check(result.program.info.images.size() == 1 && result.program.info.images[0].cube,
+	      "cube descriptor identity was not preserved through compilation");
+	Check(SpirvInstructionOpcodeCount(result.spirv, 131) == 4,
+	      "cube sample/get-lod did not remove the RDNA2 S/T bias");
+	Check(SpirvInstructionOpcodeCount(result.spirv, 109) == 1 &&
+	          SpirvInstructionOpcodeCount(result.spirv, 112) == 1 &&
+	          SpirvInstructionOpcodeCount(result.spirv, 194) == 1 &&
+	          SpirvInstructionOpcodeCount(result.spirv, 196) == 1 &&
+	          SpirvInstructionOpcodeCount(result.spirv, 130) == 1,
+	      "cube sample did not repack its face ID exactly once, or get-lod used an array layer");
+	CheckSpirvBinaryValidates(result.spirv);
+}
+
 void TestNewShaderRecompilerImageSampleVariants() {
 	const uint32_t shader[] = {
 	    EncodeMimg0(0x24, 0xf),
@@ -6957,6 +6988,7 @@ int main() {
 	TestNewShaderRecompilerScalarBitfieldAlu();
 	TestNewShaderRecompilerMemoryFamilyLowering();
 	TestNewShaderRecompilerImageQueryLowering();
+	TestNewShaderRecompilerCubeSampleCoordinates();
 	TestNewShaderRecompilerImageSampleVariants();
 	TestNewShaderRecompilerImageSampleA16SamplerCoords();
 	TestNewShaderRecompilerImageSampleOpcodeAliases();