Rework guest memory tracking/virtual address space/direct and flexible memory (#135)

* Rework guest memory tracking

* add unknwon flag

* Fix macOS guest address-space reservation

.github/workflows/build.yml

@@ -83,7 +83,12 @@ jobs:
       - name: Build
         shell: cmd
         run: |
-          cmake --build _Build/windows --target launcher --parallel
+          cmake --build _Build/windows --target launcher virtual_memory_allocation_tests --parallel
+
+      - name: Test
+        shell: cmd
+        run: |
+          ctest --test-dir _Build/windows --output-on-failure -R "^virtual_memory_allocation$"
 
       - name: Install
         shell: cmd
@@ -153,7 +158,15 @@ jobs:
       - name: Build
         shell: bash
         run: |
-          cmake --build _Build/macos --target launcher --parallel
+          cmake --build _Build/macos \
+            --target launcher virtual_memory_allocation_tests \
+            --parallel
+
+      - name: Test
+        shell: bash
+        run: |
+          ctest --test-dir _Build/macos --output-on-failure \
+            -R '^virtual_memory_allocation$'
 
       - name: Install
         shell: bash
@@ -284,13 +297,14 @@ jobs:
         run: |
           cmake --build _Build/linux \
             --target launcher page_manager_tests memory_tracker_tests \
+              virtual_memory_allocation_tests \
             --parallel
 
       - name: Test
         shell: bash
         run: |
           ctest --test-dir _Build/linux --output-on-failure \
-            -R '^(page_manager|memory_tracker)$'
+            -R '^(page_manager|memory_tracker|virtual_memory_allocation)$'
 
       - name: Install
         shell: bash

src/CMakeLists.txt

@@ -314,6 +314,16 @@ function(add_kyty_full_emulator_test target source)
 	endif()
 endfunction()
 
+function(configure_macos_guest_address_space target)
+	if(APPLE AND (CMAKE_OSX_ARCHITECTURES STREQUAL "x86_64" OR
+			(NOT CMAKE_OSX_ARCHITECTURES AND CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|AMD64)$")))
+		target_sources(${target} PRIVATE kernel/macosGuestAddressSpace.cpp)
+		target_compile_definitions(${target} PRIVATE KYTY_LINKED_GUEST_ADDRESS_SPACE=1)
+		target_link_options(${target} PRIVATE
+			-Wl,-ld_classic,-no_pie,-no_fixup_chains,-no_huge,-pagezero_size,0x40000,-segaddr,SYSTEM_MANAGED,0x40000,-segaddr,SYSTEM_RESERVED,0x7ffffc000,-segaddr,USER_AREA,0x7000000000,-image_base,0x700000000000)
+	endif()
+endfunction()
+
 add_kyty_full_emulator_test(shader_cfg_tests ../tests/shaderCfgTests.cpp)
 
 add_executable(scalar_provenance_tests EXCLUDE_FROM_ALL
@@ -338,7 +348,6 @@ add_executable(memory_tracker_tests EXCLUDE_FROM_ALL
 )
 target_link_libraries(memory_tracker_tests fmt::fmt common)
 target_include_directories(memory_tracker_tests PRIVATE ${inc_headers})
-target_compile_definitions(memory_tracker_tests PRIVATE KYTY_MEMORY_TRACKER_TESTS=1)
 
 add_executable(shader_vertex_metadata_tests EXCLUDE_FROM_ALL
 	../tests/ShaderVertexMetadataTests.cpp
@@ -421,6 +430,7 @@ target_sources(shader_recompiler_compute_tests PRIVATE
 add_kyty_full_emulator_test(virtual_memory_allocation_tests ../tests/VirtualMemoryAllocationTests.cpp)
 target_compile_definitions(virtual_memory_allocation_tests PRIVATE
 	KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS=1)
+configure_macos_guest_address_space(virtual_memory_allocation_tests)
 
 # These tests use exceptions.
 if(NOT KYTY_CLANG_CL)
@@ -437,6 +447,8 @@ if(BUILD_TESTING)
 	add_test(NAME resource_mutex COMMAND $<TARGET_FILE:resource_mutex_tests>)
 	add_test(NAME event_queue_lifetime COMMAND $<TARGET_FILE:event_queue_lifetime_tests>)
 	add_test(NAME shader_recompiler_compute COMMAND $<TARGET_FILE:shader_recompiler_compute_tests>)
+	add_test(NAME virtual_memory_allocation
+		COMMAND $<TARGET_FILE:virtual_memory_allocation_tests>)
 	add_test(NAME command_scheduler_timeline
 		COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --scheduler-only)
 	add_test(NAME stream_buffer_ring
@@ -470,6 +482,7 @@ endif()
 
 
 add_executable(kyty_emulator main.cpp ${kyty_emulator_src})
+configure_macos_guest_address_space(kyty_emulator)
 
 target_link_libraries(kyty_emulator ${kyty_emulator_link_libraries})
 if (WIN32)

src/common/virtualMemory.cpp

@@ -58,25 +58,6 @@ bool FlushInstructionCache(uint64_t address, uint64_t size) {
 	return SysVirtualFlushInstructionCache(address, size);
 }
 
-bool PatchReplace(uint64_t vaddr, uint64_t value) {
-	Mode old_mode {};
-	Protect(vaddr, 8, Mode::ReadWrite, &old_mode);
-
-	auto* ptr = reinterpret_cast<uint64_t*>(vaddr);
-
-	bool ret = (*ptr != value);
-
-	*ptr = value;
-
-	Protect(vaddr, 8, old_mode);
-
-	if (IsExecute(old_mode)) {
-		FlushInstructionCache(vaddr, 8);
-	}
-
-	return ret;
-}
-
 } // namespace VirtualMemory
 
 } // namespace Common

src/common/virtualMemory.h

@@ -37,7 +37,6 @@ bool     Free(uint64_t address);
 bool     FreeRange(uint64_t address, uint64_t size);
 bool     Protect(uint64_t address, uint64_t size, Mode mode, Mode* old_mode = nullptr);
 bool     FlushInstructionCache(uint64_t address, uint64_t size);
-bool     PatchReplace(uint64_t vaddr, uint64_t value);
 
 } // namespace VirtualMemory
 

src/emulator.cpp

@@ -105,7 +105,7 @@ static void ClearDebugTextureFolder() {
 	}
 }
 
-static void Init(const Config::ConfigOptions& cfg) {
+static void Init(const Config::ConfigOptions& cfg, const std::filesystem::path& param_json) {
 	EXIT_IF(!Common::Thread::IsMainThread());
 
 	auto* slist = Common::SubsystemsList::Instance();
@@ -127,12 +127,21 @@ static void Init(const Config::ConfigOptions& cfg) {
 	slist->InitAll(true);
 
 	Config::Load(cfg);
+	slist->Add(log, {core, config});
+	slist->InitAll(true);
+
+	if (Common::File::IsFileExisting(param_json)) {
+		Loader::SystemContentLoadParamSfo(param_json);
+		if (const auto flexible_memory_size = Loader::SystemContentGetFlexibleMemorySize();
+		    flexible_memory_size != 0) {
+			Libs::LibKernel::Memory::SetFlexibleMemorySize(flexible_memory_size);
+		}
+	}
 
 	slist->Add(audio, {core, log, pthread, memory});
 	slist->Add(controller, {core, log, config});
 	slist->Add(file_system, {core, log, pthread});
 	slist->Add(graphics, {core, log, pthread, memory, config, profiler, controller});
-	slist->Add(log, {core, config});
 	slist->Add(memory, {core, log});
 	slist->Add(network, {core, log, pthread});
 	slist->Add(profiler, {core, config});
@@ -180,7 +189,8 @@ void Run(const RunOptions& options) {
 		EXIT("ELF is required\n");
 	}
 
-	Init(options.config);
+	const auto param_json = options.app0_dir / "sce_sys" / "param.json";
+	Init(options.config, param_json);
 
 	ClearDebugTextureFolder();
 
@@ -192,15 +202,6 @@ void Run(const RunOptions& options) {
 	Libs::LibKernel::FileSystem::Mount(options.app0_dir, "/app0");
 	Libs::LibKernel::FileSystem::Mount(options.app0_dir, "/hostapp");
 
-	auto param_json = options.app0_dir / "sce_sys" / "param.json";
-	if (Common::File::IsFileExisting(param_json)) {
-		Loader::SystemContentLoadParamSfo(param_json);
-		if (auto flexible_memory_size = Loader::SystemContentGetFlexibleMemorySize();
-		    flexible_memory_size != 0) {
-			Libs::LibKernel::Memory::SetFlexibleMemorySize(flexible_memory_size);
-		}
-	}
-
 	MountSandboxDirs();
 
 	auto* rt = Common::Singleton<Loader::RuntimeLinker>::Instance();

src/graphics/host_gpu/memoryTracker.cpp

@@ -4,16 +4,6 @@
 
 namespace Libs::Graphics {
 
-#if defined(KYTY_MEMORY_TRACKER_TESTS)
-namespace {
-std::atomic<MemoryTracker::UnmapContentionHook> g_unmap_contention_hook {nullptr};
-}
-
-void MemoryTracker::SetUnmapContentionHook(UnmapContentionHook hook) noexcept {
-	g_unmap_contention_hook.store(hook, std::memory_order_release);
-}
-#endif
-
 static_assert(std::atomic<void*>::is_always_lock_free);
 
 MemoryTracker::MemoryTracker(PageManager& page_manager, PageWatchMode gpu_watch_mode)
@@ -94,7 +84,6 @@ RegionManager* MemoryTracker::GetOrCreateRegion(uint64_t index) {
 bool MemoryTracker::IsRegionCpuModified(uint64_t vaddr, uint64_t size) {
 	CheckNotInUploadCallback();
 	std::lock_guard access(m_access_mutex);
-	RequireMapped(vaddr, size);
 	return Iterate<true>(vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) {
 		std::scoped_lock lock(manager->lock);
 		return manager->IsModified<DirtySource::Cpu>(offset, bytes);
@@ -104,7 +93,6 @@ bool MemoryTracker::IsRegionCpuModified(uint64_t vaddr, uint64_t size) {
 bool MemoryTracker::IsRegionGpuModified(uint64_t vaddr, uint64_t size) {
 	CheckNotInUploadCallback();
 	std::lock_guard access(m_access_mutex);
-	RequireMapped(vaddr, size);
 	return Iterate<false>(vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) {
 		std::scoped_lock lock(manager->lock);
 		return manager->IsModified<DirtySource::Gpu>(offset, bytes);
@@ -114,7 +102,6 @@ bool MemoryTracker::IsRegionGpuModified(uint64_t vaddr, uint64_t size) {
 void MemoryTracker::MarkRegionAsCpuModified(uint64_t vaddr, uint64_t size) {
 	CheckNotInUploadCallback();
 	std::lock_guard access(m_access_mutex);
-	RequireMapped(vaddr, size);
 	Iterate<true>(vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) {
 		std::scoped_lock lock(manager->lock);
 		const auto       changed =
@@ -126,7 +113,6 @@ void MemoryTracker::MarkRegionAsCpuModified(uint64_t vaddr, uint64_t size) {
 void MemoryTracker::MarkRegionAsGpuModified(uint64_t vaddr, uint64_t size) {
 	CheckNotInUploadCallback();
 	std::lock_guard access(m_access_mutex);
-	RequireMapped(vaddr, size);
 	Iterate<true>(vaddr, size, [this](RegionManager* manager, uint64_t offset, uint64_t bytes) {
 		std::scoped_lock lock(manager->lock);
 		const auto       changed =
@@ -138,7 +124,6 @@ void MemoryTracker::MarkRegionAsGpuModified(uint64_t vaddr, uint64_t size) {
 void MemoryTracker::UnmarkRegionAsGpuModified(uint64_t vaddr, uint64_t size) {
 	CheckNotInUploadCallback();
 	std::lock_guard access(m_access_mutex);
-	RequireMapped(vaddr, size);
 	Iterate<true>(vaddr, size, [this](RegionManager* manager, uint64_t offset, uint64_t bytes) {
 		std::scoped_lock lock(manager->lock);
 		if (!manager->IsFullyModified<DirtySource::Gpu>(offset, bytes)) {
@@ -151,8 +136,6 @@ void MemoryTracker::UnmarkRegionAsGpuModified(uint64_t vaddr, uint64_t size) {
 }
 
 void MemoryTracker::UntrackMemoryLocked(uint64_t vaddr, uint64_t size) {
-	RequireMapped(vaddr, size);
-
 	std::vector<RegionManager*> managers;
 	managers.reserve((vaddr % TRACKER_REGION_SIZE + size + TRACKER_REGION_SIZE - 1) /
 	                 TRACKER_REGION_SIZE);
@@ -185,22 +168,6 @@ void MemoryTracker::UntrackMemory(uint64_t vaddr, uint64_t size) {
 	UntrackMemoryLocked(vaddr, size);
 }
 
-void MemoryTracker::UnmapMemory(uint64_t vaddr, uint64_t size) {
-	CheckNotInUploadCallback();
-	std::unique_lock access(m_access_mutex, std::try_to_lock);
-	if (!access.owns_lock()) {
-#if defined(KYTY_MEMORY_TRACKER_TESTS)
-		if (const auto hook = g_unmap_contention_hook.load(std::memory_order_acquire);
-		    hook != nullptr) {
-			hook();
-		}
-#endif
-		access.lock();
-	}
-	UntrackMemoryLocked(vaddr, size);
-	m_page_manager.OnGpuUnmap(vaddr, size);
-}
-
 bool MemoryTracker::InvalidateRegion(uint64_t vaddr, uint64_t size, PageFaultPhase phase) noexcept {
 	switch (phase) {
 		case PageFaultPhase::Release: return true;

src/graphics/host_gpu/memoryTracker.h

@@ -30,7 +30,6 @@ public:
 	void               MarkRegionAsGpuModified(uint64_t vaddr, uint64_t size);
 	void               UnmarkRegionAsGpuModified(uint64_t vaddr, uint64_t size);
 	void               UntrackMemory(uint64_t vaddr, uint64_t size);
-	void               UnmapMemory(uint64_t vaddr, uint64_t size);
 	[[nodiscard]] CpuFaultAction
 	                   BeginCpuFault(uint64_t vaddr, uint64_t size,
 	                                 PageFaultAccess access = PageFaultAccess::Write) noexcept;
@@ -91,8 +90,7 @@ public:
 		static_assert(std::is_nothrow_invocable_v<Preflight&, uint64_t, uint64_t>);
 		static_assert(std::is_nothrow_invocable_v<Func&, uint64_t, uint64_t>);
 		CheckNotInUploadCallback();
-		std::lock_guard access(m_access_mutex);
+		std::lock_guard             access(m_access_mutex);
-		RequireMapped(vaddr, size);
 		std::vector<RegionManager*> managers;
 		Iterate<false>(vaddr, size, [&](RegionManager* manager, uint64_t, uint64_t) {
 			managers.push_back(manager);
@@ -132,11 +130,6 @@ public:
 		    vaddr, size, [](uint64_t, uint64_t) noexcept {}, std::forward<Func>(func));
 	}
 
-#if defined(KYTY_MEMORY_TRACKER_TESTS)
-	using UnmapContentionHook = void (*)() noexcept;
-	static void SetUnmapContentionHook(UnmapContentionHook hook) noexcept;
-#endif
-
 	template <typename RangeFunc, typename UploadFunc>
 	void ForEachUploadRange(uint64_t vaddr, uint64_t size, bool is_written, RangeFunc&& range_func,
 	                        UploadFunc&& upload_func) {
@@ -144,7 +137,6 @@ public:
 		static_assert(std::is_nothrow_invocable_v<UploadFunc&>);
 		CheckNotInUploadCallback();
 		std::unique_lock access(m_access_mutex);
-		RequireMapped(vaddr, size);
 		Iterate<true>(vaddr, size, [](RegionManager*, uint64_t, uint64_t) {});
 		const auto* previous_upload_owner = std::exchange(s_upload_owner, this);
 		Iterate<false>(vaddr, size, [&](RegionManager* manager, uint64_t offset, uint64_t bytes) {
@@ -209,16 +201,8 @@ private:
 		return false;
 	}
 
-	static void ValidateRange(uint64_t vaddr, uint64_t size);
+	static void    ValidateRange(uint64_t vaddr, uint64_t size);
-	void        UntrackMemoryLocked(uint64_t vaddr, uint64_t size);
+	void           UntrackMemoryLocked(uint64_t vaddr, uint64_t size);
-	void        RequireMapped(uint64_t vaddr, uint64_t size) const {
-		ValidateRange(vaddr, size);
-		if (!m_page_manager.IsMapped(vaddr, size)) {
-			EXIT("memory tracker range [0x%llx, 0x%llx) is not mapped\n",
-			     static_cast<unsigned long long>(vaddr),
-			     static_cast<unsigned long long>(vaddr + size));
-		}
-	}
 	RegionManager* GetOrCreateRegion(uint64_t index);
 
 	std::unique_ptr<std::atomic<RegionManager*>[]> m_regions;

src/graphics/host_gpu/pageManager.cpp

@@ -1,6 +1,7 @@
 #include "graphics/host_gpu/pageManager.h"
 
 #include "graphics/host_gpu/regionDefinitions.h"
+#include "kernel/memory.h"
 
 #include <algorithm>
 #include <array>
@@ -21,16 +22,11 @@
 #undef min
 #undef max
 #elif defined(__APPLE__)
-#include <mach/mach.h>
-#include <mach/mach_vm.h>
 #include <pthread.h>
 #include <sys/mman.h>
 #include <unistd.h>
 #else
-#include <cerrno>
-#include <cstring>
 #include <execinfo.h>
-#include <fcntl.h>
 #include <sys/mman.h>
 #include <sys/syscall.h>
 #include <unistd.h>
@@ -57,45 +53,8 @@ constexpr uint64_t REGION_PAGES = REGION_SIZE / PAGE_SIZE;
 constexpr uint32_t NO_ACCESS_PROTECTION  = PAGE_NOACCESS;
 constexpr uint32_t READ_ONLY_PROTECTION  = PAGE_READONLY;
 constexpr uint32_t READ_WRITE_PROTECTION = PAGE_READWRITE;
-
-#if defined(__APPLE__)
-// Map the tracker's Win32-style protection tags to POSIX mprotect flags.
-static int PageProtToPosix(uint32_t protection) {
-	switch (protection) {
-		case PAGE_NOACCESS: return PROT_NONE;
-		case PAGE_READONLY: return PROT_READ;
-		case PAGE_READWRITE: return PROT_READ | PROT_WRITE;
-		default: return PROT_NONE;
-	}
-}
-
-// Query the current protection of the page containing vaddr via the Mach VM map and
-// collapse it to the tracker's read/write tags (execute is irrelevant to write tracking).
-static uint32_t MachQueryPageProt(uint64_t vaddr) {
-	auto                           region_addr = static_cast<mach_vm_address_t>(vaddr);
-	mach_vm_size_t                 region_size = 0;
-	vm_region_basic_info_data_64_t info {};
-	mach_msg_type_number_t         count       = VM_REGION_BASIC_INFO_COUNT_64;
-	mach_port_t                    object_name = MACH_PORT_NULL;
-
-	kern_return_t kr =
-	    mach_vm_region(mach_task_self(), &region_addr, &region_size, VM_REGION_BASIC_INFO_64,
-	                   reinterpret_cast<vm_region_info_t>(&info), &count, &object_name);
-	if (kr != KERN_SUCCESS || region_addr > vaddr) {
-		return PAGE_NOACCESS; // no region covering vaddr
-	}
-	if ((info.protection & VM_PROT_WRITE) != 0) {
-		return PAGE_READWRITE;
-	}
-	if ((info.protection & VM_PROT_READ) != 0) {
-		return PAGE_READONLY;
-	}
-	return PAGE_NOACCESS;
-}
-#elif defined(__linux__)
 // Zero is the unknown protection sentinel.
 constexpr uint32_t UNKNOWN_PROTECTION = 0;
-#endif
 
 thread_local bool g_in_fault_resolution = false;
 
@@ -136,6 +95,15 @@ thread_local bool g_in_fault_resolution = false;
 	std::_Exit(322);
 }
 
+Common::VirtualMemory::Mode ToMemoryMode(uint32_t protection) {
+	switch (protection) {
+		case NO_ACCESS_PROTECTION: return Common::VirtualMemory::Mode::NoAccess;
+		case READ_ONLY_PROTECTION: return Common::VirtualMemory::Mode::Read;
+		case READ_WRITE_PROTECTION: return Common::VirtualMemory::Mode::ReadWrite;
+		default: Fatal("unmappable protection 0x%08" PRIx32, protection);
+	}
+}
+
 uint32_t CurrentThread() noexcept {
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
 	return GetCurrentThreadId();
@@ -155,130 +123,6 @@ uint32_t CurrentThread() noexcept {
 #endif
 }
 
-#if defined(__linux__)
-int ToHostProtection(uint32_t protection) {
-	switch (protection) {
-		case NO_ACCESS_PROTECTION: return PROT_NONE;
-		case READ_ONLY_PROTECTION: return PROT_READ;
-		case READ_WRITE_PROTECTION: return PROT_READ | PROT_WRITE;
-		default: Fatal("unmappable protection 0x%08" PRIx32, protection);
-	}
-}
-
-struct HostMapping {
-	uint64_t end        = 0;
-	uint32_t protection = UNKNOWN_PROTECTION;
-};
-
-// Async-signal-safe lookup in the address-ordered /proc/self/maps.
-HostMapping QueryHostMapping(uint64_t vaddr) noexcept {
-	int fd = ::open("/proc/self/maps", O_RDONLY | O_CLOEXEC); // NOLINT
-	if (fd < 0) {
-		return {};
-	}
-
-	enum class Field { Start, End, Perms, Rest };
-
-	HostMapping result {};
-	auto        field      = Field::Start;
-	uint64_t    start      = 0;
-	uint64_t    end        = 0;
-	char        perms[4]   = {};
-	uint32_t    perms_len  = 0;
-	bool        line_valid = true;
-
-	char buffer[8192];
-
-	for (bool done = false; !done;) {
-		const auto got = ::read(fd, buffer, sizeof(buffer));
-		if (got < 0) {
-			if (errno == EINTR) {
-				continue;
-			}
-			break;
-		}
-		if (got == 0) {
-			break;
-		}
-
-		for (ssize_t i = 0; i < got && !done; i++) {
-			const char c = buffer[i];
-
-			if (c == '\n') {
-				field      = Field::Start;
-				start      = 0;
-				end        = 0;
-				perms_len  = 0;
-				line_valid = true;
-				continue;
-			}
-
-			if (!line_valid) {
-				continue;
-			}
-
-			switch (field) {
-				case Field::Start:
-				case Field::End: {
-					uint64_t digit = 0;
-					if (c >= '0' && c <= '9') {
-						digit = static_cast<uint64_t>(c - '0');
-					} else if (c >= 'a' && c <= 'f') {
-						digit = static_cast<uint64_t>(c - 'a') + 10;
-					} else if (c == '-' && field == Field::Start) {
-						field = Field::End;
-						break;
-					} else if (c == ' ' && field == Field::End) {
-						field     = Field::Perms;
-						perms_len = 0;
-						break;
-					} else {
-						line_valid = false;
-						break;
-					}
-
-					auto& value = (field == Field::Start ? start : end);
-					value       = (value << 4u) | digit;
-					break;
-				}
-
-				case Field::Perms: {
-					if (c != ' ') {
-						if (perms_len < sizeof(perms)) {
-							perms[perms_len] = c;
-						}
-						perms_len++;
-						break;
-					}
-
-					if (vaddr < start) {
-						done = true;
-					} else if (vaddr < end && perms_len >= 2) {
-						result.end        = end;
-						result.protection = perms[1] == 'w'   ? READ_WRITE_PROTECTION
-						                    : perms[0] == 'r' ? READ_ONLY_PROTECTION
-						                                      : NO_ACCESS_PROTECTION;
-						done              = true;
-					} else {
-						field = Field::Rest;
-					}
-					break;
-				}
-
-				case Field::Rest: break;
-			}
-		}
-	}
-
-	::close(fd);
-	return result;
-}
-
-uint32_t QueryHostProtection(uint64_t vaddr) noexcept {
-	return QueryHostMapping(vaddr).protection;
-}
-#endif
-
 class SpinGuard final {
 public:
 	explicit SpinGuard(std::atomic_flag& lock): m_lock(lock) {
@@ -313,21 +157,16 @@ uint64_t PageEnd(uint64_t vaddr, uint64_t size) {
 struct PageManager::Impl {
 	struct PageState {
 		std::atomic_flag lock                = ATOMIC_FLAG_INIT;
-		uint32_t         mappings            = 0;
-		uint32_t         gpu_read_mappings   = 0;
-		uint32_t         gpu_write_mappings  = 0;
 		uint32_t         write_watchers      = 0;
 		uint32_t         access_watchers     = 0;
 		uint32_t         original_protection = 0;
 		uint32_t         backing_writer      = 0;
-#if defined(__linux__)
 		// Shadow the protection applied through Protect().
-		uint32_t current_protection = UNKNOWN_PROTECTION;
+		uint32_t current_protection   = UNKNOWN_PROTECTION;
-#endif
+		bool     resolving            = false;
-		bool resolving            = false;
+		bool     resolving_read_write = false;
-		bool resolving_read_write = false;
+		bool     late_read_pending    = false;
-		bool late_read_pending    = false;
+		bool     late_write_pending   = false;
-		bool late_write_pending   = false;
 	};
 
 	struct Region {
@@ -356,7 +195,7 @@ struct PageManager::Impl {
 
 	Impl(PageFaultHandler handler, void* context): fault_handler(handler), fault_context(context) {
 		if (fault_handler == nullptr) {
-			Fatal("null fault handler");
+			Fatal("null page-manager fault callback");
 		}
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
 		SYSTEM_INFO info {};
@@ -386,9 +225,8 @@ struct PageManager::Impl {
 		for (const auto& region: region_storage) {
 			for (auto& page: region->pages) {
 				SpinGuard lock(page.lock);
-				if (page.mappings != 0 || page.gpu_read_mappings != 0 ||
+				if (page.write_watchers != 0 || page.access_watchers != 0 ||
-				    page.gpu_write_mappings != 0 || page.write_watchers != 0 ||
+				    page.backing_writer != 0 || page.resolving) {
-				    page.access_watchers != 0 || page.backing_writer != 0 || page.resolving) {
 					FailFast("PageManager destroyed with live page state");
 				}
 			}
@@ -441,179 +279,30 @@ struct PageManager::Impl {
 		}
 	}
 
-	static void ValidateInitialProtection(std::span<PageState*> pages, uint64_t vaddr) {
+	static void InitializeProtection(std::span<PageState*> pages) {
-		const auto end = vaddr + pages.size() * PAGE_SIZE;
-#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
-		for (auto address = vaddr; address < end;) {
-			MEMORY_BASIC_INFORMATION info {};
-			if (VirtualQuery(reinterpret_cast<const void*>(static_cast<uintptr_t>(address)), &info,
-			                 sizeof(info)) == 0 ||
-			    info.State != MEM_COMMIT || info.Protect != PAGE_READWRITE) {
-				Fatal("basic path requires PAGE_READWRITE at 0x%016" PRIx64 " (state=0x%08" PRIx32
-				      ", protection=0x%08" PRIx32 ")",
-				      address, static_cast<uint32_t>(info.State),
-				      static_cast<uint32_t>(info.Protect));
-			}
-			const auto region_end = reinterpret_cast<uint64_t>(info.BaseAddress) + info.RegionSize;
-			if (region_end <= address) {
-				Fatal("VirtualQuery returned an invalid region at 0x%016" PRIx64, address);
-			}
-			address = std::min(end, region_end);
-		}
-#elif defined(__APPLE__)
-		for (auto address = vaddr; address < end; address += PAGE_SIZE) {
-			const uint32_t protection = MachQueryPageProt(address);
-			if (protection != PAGE_READWRITE) {
-				Fatal("basic path requires PAGE_READWRITE at 0x%016" PRIx64
-				      " (protection=0x%08" PRIx32 ")",
-				      address, protection);
-			}
-		}
-#else
-		for (auto address = vaddr; address < end;) {
-			const auto mapping = QueryHostMapping(address);
-			if (mapping.protection != READ_WRITE_PROTECTION || mapping.end <= address) {
-				Fatal("basic path requires a read/write mapping at 0x%016" PRIx64
-				      " (protection=0x%08" PRIx32 ")",
-				      address, mapping.protection);
-			}
-			address = std::min(end, mapping.end);
-		}
-		for (auto* page: pages) {
-			page->current_protection = READ_WRITE_PROTECTION;
-		}
-#endif
 		for (auto* page: pages) {
 			page->original_protection = READ_WRITE_PROTECTION;
+			page->current_protection  = READ_WRITE_PROTECTION;
 		}
 	}
 
-	static bool AllowsAccess([[maybe_unused]] const PageState& page, uint64_t vaddr,
+	static bool AllowsAccess(const PageState& page, [[maybe_unused]] uint64_t vaddr,
 	                         PageFaultAccess access) noexcept {
-#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
-		MEMORY_BASIC_INFORMATION info {};
-		if (VirtualQuery(reinterpret_cast<const void*>(static_cast<uintptr_t>(vaddr)), &info,
-		                 sizeof(info)) == 0 ||
-		    info.State != MEM_COMMIT) {
-			return false;
-		}
 		switch (access) {
 			case PageFaultAccess::Read:
-				return info.Protect == PAGE_READONLY || info.Protect == PAGE_READWRITE;
+				return page.current_protection == READ_ONLY_PROTECTION ||
-			case PageFaultAccess::Write: return info.Protect == PAGE_READWRITE;
+				       page.current_protection == READ_WRITE_PROTECTION;
+			case PageFaultAccess::Write: return page.current_protection == READ_WRITE_PROTECTION;
 			default: return false;
 		}
-#elif defined(__APPLE__)
-		const uint32_t protection = MachQueryPageProt(vaddr);
-		switch (access) {
-			case PageFaultAccess::Read:
-				return protection == PAGE_READONLY || protection == PAGE_READWRITE;
-			case PageFaultAccess::Write: return protection == PAGE_READWRITE;
-			default: return false;
-		}
-#else
-		const auto permitted = [](uint32_t protection, PageFaultAccess wanted) {
-			switch (wanted) {
-				case PageFaultAccess::Read:
-					return protection == READ_ONLY_PROTECTION ||
-					       protection == READ_WRITE_PROTECTION;
-				case PageFaultAccess::Write: return protection == READ_WRITE_PROTECTION;
-				default: return false;
-			}
-		};
-
-		if (!permitted(page.current_protection, access)) {
-			return false;
-		}
-		return permitted(QueryHostProtection(vaddr), access);
-#endif
 	}
 
-	static void ProtectRange(std::span<PageState*> pages, uint64_t vaddr, uint32_t protection,
+	void ProtectRange(std::span<PageState*> pages, uint64_t vaddr, uint32_t protection,
-	                         std::span<const uint32_t> expected_old, bool fault_path) noexcept {
+	                  std::span<const uint32_t> expected_old, bool fault_path) noexcept {
 		const auto size = pages.size() * PAGE_SIZE;
 		if (pages.size() != expected_old.size()) {
 			FailFast("protection range state size mismatch");
 		}
-#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
-		struct HostRange {
-			uint64_t begin = 0;
-			uint64_t end   = 0;
-		};
-		std::vector<HostRange> host_ranges;
-		const auto             end = vaddr + size;
-		for (auto address = vaddr; address < end;) {
-			MEMORY_BASIC_INFORMATION info {};
-			if (VirtualQuery(reinterpret_cast<const void*>(static_cast<uintptr_t>(address)), &info,
-			                 sizeof(info)) == 0 ||
-			    info.State != MEM_COMMIT) {
-				if (fault_path) {
-					FailFast("VirtualProtect fault transition did not match expected protection");
-				}
-				Fatal("invalid protection transition at 0x%016" PRIx64 ", state=0x%08" PRIx32
-				      ", new=0x%08" PRIx32,
-				      address, static_cast<uint32_t>(info.State), protection);
-			}
-			const auto region_end = reinterpret_cast<uint64_t>(info.BaseAddress) + info.RegionSize;
-			const auto query_end  = std::min(end, region_end);
-			if (query_end <= address) {
-				if (fault_path) {
-					FailFast("VirtualQuery returned an invalid fault transition region");
-				}
-				Fatal("VirtualQuery returned an invalid region at 0x%016" PRIx64, address);
-			}
-			const auto first_page = static_cast<size_t>((address - vaddr) / PAGE_SIZE);
-			const auto last_page =
-			    static_cast<size_t>((query_end - vaddr + PAGE_SIZE - 1) / PAGE_SIZE);
-			for (auto page = first_page; page < last_page; page++) {
-				if (info.Protect != expected_old[page]) {
-					if (fault_path) {
-						FailFast(
-						    "VirtualProtect fault transition did not match expected protection");
-					}
-					Fatal("invalid protection transition at 0x%016" PRIx64 ", actual=0x%08" PRIx32
-					      ", expected=0x%08" PRIx32 ", new=0x%08" PRIx32,
-					      vaddr + page * PAGE_SIZE, static_cast<uint32_t>(info.Protect),
-					      expected_old[page], protection);
-				}
-			}
-			const auto allocation = reinterpret_cast<uint64_t>(info.AllocationBase);
-			if (host_ranges.empty() || allocation != host_ranges.back().begin) {
-				host_ranges.push_back({allocation, query_end});
-			} else {
-				host_ranges.back().end = query_end;
-			}
-			address = query_end;
-		}
-		for (auto range: host_ranges) {
-			range.begin               = std::max(range.begin, vaddr);
-			DWORD      old_protection = 0;
-			const auto first_page     = static_cast<size_t>((range.begin - vaddr) / PAGE_SIZE);
-			if (VirtualProtect(reinterpret_cast<void*>(static_cast<uintptr_t>(range.begin)),
-			                   range.end - range.begin, protection, &old_protection) == 0 ||
-			    old_protection != expected_old[first_page]) {
-				if (fault_path) {
-					FailFast("VirtualProtect fault transition did not match expected protection");
-				}
-				Fatal("invalid protection transition at 0x%016" PRIx64 ", old=0x%08" PRIx32
-				      ", expected=0x%08" PRIx32 ", new=0x%08" PRIx32,
-				      range.begin, static_cast<uint32_t>(old_protection), expected_old[first_page],
-				      protection);
-			}
-		}
-#elif defined(__APPLE__)
-		// mprotect cannot report the previous protection, so the expected_old comparison
-		// is dropped; the tracker is the sole mutator of these pages and drives the
-		// transition from its own shadow state.
-		(void)expected_old;
-		if (mprotect(reinterpret_cast<void*>(static_cast<uintptr_t>(vaddr)), size,
-		             PageProtToPosix(protection)) != 0) {
-			if (fault_path) {
-				FailFast("mprotect fault transition failed");
-			}
-			Fatal("mprotect failed at 0x%016" PRIx64 ", new=0x%08" PRIx32, vaddr, protection);
-		}
-#else
 		for (size_t i = 0; i < pages.size(); i++) {
 			const auto actual = pages[i]->current_protection;
 			if (actual != UNKNOWN_PROTECTION && actual != expected_old[i]) {
@@ -625,22 +314,21 @@ struct PageManager::Impl {
 				      vaddr + i * PAGE_SIZE, actual, expected_old[i], protection);
 			}
 		}
-		if (::mprotect(reinterpret_cast<void*>(static_cast<uintptr_t>(vaddr)), size,
+		if (!Libs::LibKernel::Memory::ProtectGuestHostMemory(vaddr, size,
-		               ToHostProtection(protection)) != 0) {
+		                                                     ToMemoryMode(protection))) {
 			if (fault_path) {
-				FailFast("mprotect failed on the fault path");
+				FailFast("address-space fault protection transition failed");
 			}
-			Fatal("mprotect failed at 0x%016" PRIx64 ", new=0x%08" PRIx32 " (%s)", vaddr,
+			Fatal("address-space protection failed at 0x%016" PRIx64 ", new=0x%08" PRIx32, vaddr,
-			      protection, std::strerror(errno));
+			      protection);
 		}
 		for (auto* page: pages) {
 			page->current_protection = protection;
 		}
-#endif
 	}
 
-	static void Protect(PageState& page, uint64_t vaddr, uint32_t protection, uint32_t expected_old,
+	void Protect(PageState& page, uint64_t vaddr, uint32_t protection, uint32_t expected_old,
-	                    bool fault_path) noexcept {
+	             bool fault_path) noexcept {
 		PageState* pages[]    = {&page};
 		uint32_t   expected[] = {expected_old};
 		ProtectRange(pages, vaddr, protection, expected, fault_path);
@@ -680,50 +368,6 @@ bool PageManager::IsTracked(uint64_t vaddr) const noexcept {
 	return page.write_watchers != 0 || page.access_watchers != 0;
 }
 
-bool PageManager::IsMapped(uint64_t vaddr, uint64_t size) const noexcept {
-	if (vaddr == 0 || size == 0 || vaddr >= ADDRESS_SIZE || size > ADDRESS_SIZE - vaddr) {
-		return false;
-	}
-	const auto end = PageStart(vaddr + size - 1) + PAGE_SIZE;
-	for (auto page_vaddr = PageStart(vaddr); page_vaddr < end; page_vaddr += PAGE_SIZE) {
-		auto* region = m_impl->FindRegion(page_vaddr);
-		if (region == nullptr) {
-			return false;
-		}
-		auto&     page = m_impl->GetPage(*region, page_vaddr);
-		SpinGuard lock(page.lock);
-		if (page.mappings == 0) {
-			return false;
-		}
-	}
-	return true;
-}
-
-bool PageManager::HasGpuAccess(uint64_t vaddr, uint64_t size, GpuAccess access) const noexcept {
-	if (access != GpuAccess::Read && access != GpuAccess::Write && access != GpuAccess::ReadWrite) {
-		FailFast("HasGpuAccess received an invalid GPU access mode");
-	}
-	const bool need_read  = access == GpuAccess::Read || access == GpuAccess::ReadWrite;
-	const bool need_write = access == GpuAccess::Write || access == GpuAccess::ReadWrite;
-	if (vaddr == 0 || size == 0 || vaddr >= ADDRESS_SIZE || size > ADDRESS_SIZE - vaddr) {
-		return false;
-	}
-	const auto end = PageEnd(vaddr, size);
-	for (auto addr = PageStart(vaddr); addr < end; addr += PAGE_SIZE) {
-		auto* region = m_impl->FindRegion(addr);
-		if (region == nullptr) {
-			return false;
-		}
-		auto&     page = m_impl->GetPage(*region, addr);
-		SpinGuard lock(page.lock);
-		if ((need_read && page.gpu_read_mappings == 0) ||
-		    (need_write && page.gpu_write_mappings == 0)) {
-			return false;
-		}
-	}
-	return true;
-}
-
 void PageManager::UpdatePageWatchers(bool track, uint64_t vaddr, uint64_t size,
                                      PageWatchMode mode) {
 	if (mode != PageWatchMode::Write && mode != PageWatchMode::ReadWrite) {
@@ -754,9 +398,6 @@ void PageManager::UpdatePageWatchers(bool track, uint64_t vaddr, uint64_t size,
 			if (page.resolving && track) {
 				FailFast("new page watcher raced active fault resolution");
 			}
-			if (page.mappings == 0) {
-				Fatal("watching unmapped page 0x%016" PRIx64, address);
-			}
 			auto& watchers =
 			    (mode == PageWatchMode::ReadWrite ? page.access_watchers : page.write_watchers);
 			if (track) {
@@ -784,8 +425,7 @@ void PageManager::UpdatePageWatchers(bool track, uint64_t vaddr, uint64_t size,
 					last++;
 				}
 				if (first != last) {
-					Impl::ValidateInitialProtection(std::span {pages}.subspan(first, last - first),
+					Impl::InitializeProtection(std::span {pages}.subspan(first, last - first));
-					                                chunk_begin + first * PAGE_SIZE);
 				}
 				first = last;
 			}
@@ -831,9 +471,9 @@ void PageManager::UpdatePageWatchers(bool track, uint64_t vaddr, uint64_t size,
 					last = current + 1;
 				}
 			}
-			Impl::ProtectRange(std::span {pages}.subspan(first, last - first),
+			m_impl->ProtectRange(std::span {pages}.subspan(first, last - first),
-			                   chunk_begin + first * PAGE_SIZE, protection,
+			                     chunk_begin + first * PAGE_SIZE, protection,
-			                   std::span {old_protections}.subspan(first, last - first), false);
+			                     std::span {old_protections}.subspan(first, last - first), false);
 			first = current;
 		}
 
@@ -860,70 +500,9 @@ void PageManager::UpdatePageWatchers(bool track, uint64_t vaddr, uint64_t size,
 	}
 }
 
-void PageManager::OnGpuMap(uint64_t vaddr, uint64_t size, GpuAccess access) {
+void PageManager::OnGpuMap(uint64_t, uint64_t) {}
-	if (g_in_fault_resolution) {
-		FailFast("GPU mapping changed during fault resolution");
-	}
-	if (access != GpuAccess::Read && access != GpuAccess::Write && access != GpuAccess::ReadWrite) {
-		FailFast("GPU map received an invalid access mode");
-	}
-	const bool gpu_read  = access == GpuAccess::Read || access == GpuAccess::ReadWrite;
-	const bool gpu_write = access == GpuAccess::Write || access == GpuAccess::ReadWrite;
-	const auto end       = PageEnd(vaddr, size);
-	for (auto addr = PageStart(vaddr); addr < end; addr += PAGE_SIZE) {
-		auto&     page = m_impl->GetPage(*m_impl->GetOrCreateRegion(addr), addr);
-		SpinGuard lock(page.lock);
-		if (page.resolving || page.mappings == std::numeric_limits<uint32_t>::max() ||
-		    (gpu_read && page.gpu_read_mappings == std::numeric_limits<uint32_t>::max()) ||
-		    (gpu_write && page.gpu_write_mappings == std::numeric_limits<uint32_t>::max())) {
-			Fatal("invalid map state at 0x%016" PRIx64, addr);
-		}
-		page.mappings++;
-		page.gpu_read_mappings += gpu_read ? 1u : 0u;
-		page.gpu_write_mappings += gpu_write ? 1u : 0u;
-#if defined(__linux__)
-		// New guest mappings start read/write.
-		if (page.current_protection == UNKNOWN_PROTECTION) {
-			page.current_protection = READ_WRITE_PROTECTION;
-		}
-#endif
-	}
-}
 
-void PageManager::OnGpuUnmap(uint64_t vaddr, uint64_t size, GpuAccess access) {
+void PageManager::OnGpuUnmap(uint64_t, uint64_t) {}
-	if (g_in_fault_resolution) {
-		FailFast("GPU unmapping changed during fault resolution");
-	}
-	if (access != GpuAccess::Read && access != GpuAccess::Write && access != GpuAccess::ReadWrite) {
-		FailFast("GPU unmap received an invalid access mode");
-	}
-	const bool gpu_read  = access == GpuAccess::Read || access == GpuAccess::ReadWrite;
-	const bool gpu_write = access == GpuAccess::Write || access == GpuAccess::ReadWrite;
-	const auto end       = PageEnd(vaddr, size);
-	for (auto page_vaddr = PageStart(vaddr); page_vaddr < end; page_vaddr += PAGE_SIZE) {
-		auto* region = m_impl->FindRegion(page_vaddr);
-		if (region == nullptr) {
-			Fatal("unmapping unknown page 0x%016" PRIx64, page_vaddr);
-		}
-		auto&     page = m_impl->GetPage(*region, page_vaddr);
-		SpinGuard lock(page.lock);
-		if (page.resolving || page.mappings == 0 || (gpu_read && page.gpu_read_mappings == 0) ||
-		    (gpu_write && page.gpu_write_mappings == 0) ||
-		    (page.mappings == 1 && (page.write_watchers != 0 || page.access_watchers != 0))) {
-			Fatal("invalid unmap state at 0x%016" PRIx64, page_vaddr);
-		}
-		page.mappings--;
-		page.gpu_read_mappings -= gpu_read ? 1u : 0u;
-		page.gpu_write_mappings -= gpu_write ? 1u : 0u;
-		if (page.mappings == 0) {
-			if (page.gpu_read_mappings != 0 || page.gpu_write_mappings != 0) {
-				FailFast("GPU unmap left nonzero GPU mapping counts");
-			}
-			page.late_read_pending  = false;
-			page.late_write_pending = false;
-		}
-	}
-}
 
 PageManager::BackingWrite::BackingWrite(PageManager& manager, uint64_t vaddr,
                                         uint64_t size) noexcept
@@ -979,8 +558,7 @@ void PageManager::BeginBackingWrite(uint64_t vaddr, uint64_t size) noexcept {
 		}
 		auto&     page = m_impl->GetPage(*region, address);
 		SpinGuard lock(page.lock);
-		if (page.mappings == 0 || page.resolving || page.backing_writer != 0 ||
+		if (page.resolving || page.backing_writer != 0 || page.access_watchers == 0) {
-		    page.access_watchers == 0) {
 			Fatal("backing write races page resolution at 0x%016" PRIx64, address);
 		}
 		page.resolving            = true;
@@ -1008,7 +586,7 @@ void PageManager::EndBackingWrite(uint64_t vaddr, uint64_t size) noexcept {
 		const auto old_protection = NO_ACCESS_PROTECTION;
 		const auto new_protection = Impl::WatcherProtection(page);
 		if (new_protection != old_protection) {
-			Impl::Protect(page, address, new_protection, old_protection, false);
+			m_impl->Protect(page, address, new_protection, old_protection, false);
 		}
 		Impl::PublishDelayedFaults(page, old_protection, new_protection);
 		if (page.write_watchers == 0 && page.access_watchers == 0) {
@@ -1109,7 +687,8 @@ bool PageManager::HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noex
 				page.write_watchers = 0;
 			}
 			const auto restored_protection = Impl::WatcherProtection(page);
-			Impl::Protect(page, PageStart(fault_vaddr), restored_protection, old_protection, true);
+			m_impl->Protect(page, PageStart(fault_vaddr), restored_protection, old_protection,
+			                true);
 			if (page.write_watchers == 0) {
 				page.original_protection = 0;
 			}

src/graphics/host_gpu/pageManager.h

@@ -13,11 +13,9 @@ namespace Libs::Graphics {
 enum class PageFaultAccess { Read, Write, Execute, Unknown };
 enum class PageFaultPhase { Invalidate, Complete, Release };
 enum class PageWatchMode { Write, ReadWrite };
-enum class GpuAccess { Read, Write, ReadWrite };
 
 using PageFaultHandler = bool (*)(void* context, PageFaultAccess access, uint64_t vaddr,
                                   uint64_t size, PageFaultPhase phase) noexcept;
-
 class PageManager final {
 public:
 	class BackingWrite final {
@@ -40,13 +38,11 @@ public:
 
 	[[nodiscard]] uint64_t GetPageSize() const;
 	[[nodiscard]] bool     IsTracked(uint64_t vaddr) const noexcept;
-	[[nodiscard]] bool     IsMapped(uint64_t vaddr, uint64_t size) const noexcept;
-	[[nodiscard]] bool HasGpuAccess(uint64_t vaddr, uint64_t size, GpuAccess access) const noexcept;
 
 	void UpdatePageWatchers(bool track, uint64_t vaddr, uint64_t size,
 	                        PageWatchMode mode = PageWatchMode::Write);
-	void OnGpuMap(uint64_t vaddr, uint64_t size, GpuAccess access = GpuAccess::ReadWrite);
+	void OnGpuMap(uint64_t vaddr, uint64_t size);
-	void OnGpuUnmap(uint64_t vaddr, uint64_t size, GpuAccess access = GpuAccess::ReadWrite);
+	void OnGpuUnmap(uint64_t vaddr, uint64_t size);
 
 	[[nodiscard]] bool HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noexcept;
 	[[nodiscard]] std::vector<std::unique_ptr<BackingWrite>>

src/graphics/host_gpu/renderer/cache/bufferCache.cpp

@@ -714,7 +714,6 @@ BufferBinding BufferCache::ObtainBuffer(CommandBuffer& command, uint64_t vaddr,
 	if (command.IsInvalid() || command.IsExecute()) {
 		EXIT("BufferCache: buffer request requires a recording command buffer\n");
 	}
-	ValidateGpuAccess(vaddr, size, is_read, is_written);
 	std::lock_guard transaction(m_resource_mutex);
 	(void)SynchronizeBacking(vaddr, size);
 
@@ -999,7 +998,6 @@ void BufferCache::FillBuffer(uint64_t vaddr, uint64_t size, uint32_t value, bool
 	if (vaddr == 0) {
 		EXIT("BufferCache: invalid fill memory address\n");
 	}
-	ValidateGpuAccess(vaddr, size, false, true);
 	(void)m_texture_cache.ClearMeta(vaddr);
 	{
 		std::lock_guard transaction(m_resource_mutex);
@@ -1041,12 +1039,6 @@ void BufferCache::CopyBuffer(uint64_t dst_vaddr, uint64_t src_vaddr, uint64_t si
 	    (src_gds && (src_vaddr > m_gds_buffer.Size() || size > m_gds_buffer.Size() - src_vaddr))) {
 		EXIT("BufferCache: invalid or overlapping copy range\n");
 	}
-	if (src_memory) {
-		ValidateGpuAccess(src_vaddr, size, true, false);
-	}
-	if (dst_memory) {
-		ValidateGpuAccess(dst_vaddr, size, false, true);
-	}
 	if (src_memory || dst_memory) {
 		std::lock_guard transaction(m_resource_mutex);
 		if (src_memory) {
@@ -1203,19 +1195,6 @@ void BufferCache::PublishImageBuffer(uint64_t vaddr, uint64_t size) {
 	owner->second->tick_accessed_last = m_gc_tick;
 }
 
-void BufferCache::ValidateGpuAccess(uint64_t vaddr, uint64_t size, bool is_read,
-                                    bool is_written) const {
-	if ((!is_read && !is_written) || vaddr == 0 || size == 0 || size > UINT64_MAX - vaddr) {
-		EXIT("BufferCache: invalid GPU access request\n");
-	}
-	if (is_read && !m_page_manager.HasGpuAccess(vaddr, size, GpuAccess::Read)) {
-		EXIT("BufferCache: GPU-read access denied\n");
-	}
-	if (is_written && !m_page_manager.HasGpuAccess(vaddr, size, GpuAccess::Write)) {
-		EXIT("BufferCache: GPU-write access denied\n");
-	}
-}
-
 void BufferCache::RunGarbageCollector() {
 	std::lock_guard transaction(m_resource_mutex);
 	const auto      tick = m_gc_tick++;

src/graphics/host_gpu/renderer/cache/bufferCache.h

@@ -77,8 +77,7 @@ public:
 	void               CompleteBackingPublication(uint64_t vaddr, uint64_t size, uint64_t tick);
 	[[nodiscard]] bool SynchronizeBacking(uint64_t vaddr, uint64_t size);
 	void               PublishImageBuffer(uint64_t vaddr, uint64_t size);
-	void ValidateGpuAccess(uint64_t vaddr, uint64_t size, bool is_read, bool is_written) const;
+	void               RunGarbageCollector();
-	void RunGarbageCollector();
 
 private:
 	friend struct BufferCacheTestAccess;

src/graphics/host_gpu/renderer/cache/gpuResourceManager.cpp

@@ -4,7 +4,6 @@
 #include "graphics/guest_gpu/command_processor/commandProcessor.h"
 #include "graphics/guest_gpu/graphicsRun.h"
 #include "graphics/host_gpu/renderer/commandScheduler.h"
-
 namespace Libs::Graphics {
 
 GpuResourceManager::GpuResourceManager(GraphicContext& graphics, CommandScheduler& scheduler)
@@ -115,22 +114,19 @@ bool GpuResourceManager::IsMapped(uint64_t vaddr, uint64_t size) const noexcept
 	return m_mapped_ranges.Contains(vaddr, size);
 }
 
-void GpuResourceManager::MapMemory(uint64_t vaddr, uint64_t size, GpuAccess access) {
+void GpuResourceManager::MapMemory(uint64_t vaddr, uint64_t size) {
 	{
 		std::lock_guard lock(m_mapped_ranges_mutex);
 		m_mapped_ranges.Add(vaddr, size);
 	}
-	m_page_manager.OnGpuMap(vaddr, size, access);
+	m_page_manager.OnGpuMap(vaddr, size);
 }
 
-void GpuResourceManager::UnmapMemory(uint64_t vaddr, uint64_t size, GpuAccess access) {
+void GpuResourceManager::UnmapMemory(uint64_t vaddr, uint64_t size) {
-	if (!IsMapped(vaddr, size)) {
+	const auto unmap = [this, vaddr, size] {
-		EXIT("cannot unmap an unmapped GPU resource range\n");
-	}
-	const auto unmap = [this, vaddr, size, access] {
-		m_texture_cache.UnmapMemory(vaddr, size);
 		m_buffer_cache.UnmapMemory(vaddr, size);
-		m_page_manager.OnGpuUnmap(vaddr, size, access);
+		m_texture_cache.UnmapMemory(vaddr, size);
+		m_page_manager.OnGpuUnmap(vaddr, size);
 		std::lock_guard lock(m_mapped_ranges_mutex);
 		m_mapped_ranges.Subtract(vaddr, size);
 	};

src/graphics/host_gpu/renderer/cache/gpuResourceManager.h

@@ -29,8 +29,8 @@ public:
 	[[nodiscard]] bool HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noexcept;
 	[[nodiscard]] bool InvalidateMemory(uint64_t vaddr, uint64_t size);
 	[[nodiscard]] bool IsMapped(uint64_t vaddr, uint64_t size) const noexcept;
-	void               MapMemory(uint64_t vaddr, uint64_t size, GpuAccess access);
+	void               MapMemory(uint64_t vaddr, uint64_t size);
-	void               UnmapMemory(uint64_t vaddr, uint64_t size, GpuAccess access);
+	void               UnmapMemory(uint64_t vaddr, uint64_t size);
 	void               RunGarbageCollector();
 
 private:

src/graphics/host_gpu/renderer/cache/textureCache.cpp

@@ -1377,7 +1377,6 @@ bool TextureCache::ClearImageFromBuffer(CommandBuffer& command, uint64_t address
 	if (command.IsInvalid() || !GuestRange {address, size}.Valid()) {
 		EXIT("TextureCache: invalid image clear\n");
 	}
-	m_buffer_cache.ValidateGpuAccess(address, size, false, true);
 	std::lock_guard      transaction(m_resource_mutex);
 	CacheLock            lock(*this, m_lock);
 	ImageId              selected {};

src/graphics/host_gpu/renderer/pipeline/descriptors.cpp

@@ -616,8 +616,6 @@ RenderExecutor::ResolveTexture(const ShaderRecompiler::IR::ImageResource&   reso
 	                     (address & (static_cast<uint64_t>(size.align) - 1u)) != 0);
 	if (storage) {
 		ValidateStorageTexture(resource, descriptor, size.size);
-		m_context.GetBufferCache().ValidateGpuAccess(address, size.size, resource.read,
-		                                             resource.written);
 	}
 
 	const auto              pixel_format        = TextureGetFormat(format);

src/kernel/macosGuestAddressSpace.cpp

@@ -0,0 +1,8 @@
+#if defined(__APPLE__) && defined(__x86_64__)
+
+// Make the process own the guest ranges before any runtime initialization.
+asm(".zerofill SYSTEM_MANAGED,SYSTEM_MANAGED,__kyty_system_managed,0x7fffbc000");
+asm(".zerofill SYSTEM_RESERVED,SYSTEM_RESERVED,__kyty_system_reserved,0x7c0004000");
+asm(".zerofill USER_AREA,USER_AREA,__kyty_user_area,0x8c00000000");
+
+#endif

src/kernel/memory.h

@@ -145,7 +145,7 @@ int KYTY_SYSV_ABI KernelIsStack(void* addr, void** start, void** end);
 int KYTY_SYSV_ABI KernelReserveVirtualRange(void** addr, size_t len, int flags, size_t alignment);
 bool              KernelHandleReservedRangeAccessViolation(uint64_t vaddr);
 int KYTY_SYSV_ABI KernelAvailableFlexibleMemorySize(size_t* size);
-int KYTY_SYSV_ABI KernelConfiguredFlexibleMemorySize(uint64_t* size);
+int KYTY_SYSV_ABI KernelConfiguredFlexibleMemorySize(size_t* size);
 int KYTY_SYSV_ABI KernelMprotect(const void* addr, size_t len, int prot);
 int KYTY_SYSV_ABI KernelMtypeprotect(const void* addr, size_t len, int type, int prot);
 int KYTY_SYSV_ABI KernelBatchMap(KernelBatchMapEntry* entries, int num_entries,
@@ -163,17 +163,30 @@ int KYTY_SYSV_ABI KernelMemoryPoolBatch(const KernelMemoryPoolBatchEntry* entrie
 int KYTY_SYSV_ABI KernelMemoryPoolGetBlockStats(KernelMemoryPoolBlockStats* output,
                                                 size_t                      output_size);
 
-void RegisterProgramMemory(uint64_t vaddr, uint64_t size, Common::VirtualMemory::Mode mode,
+uint64_t AllocateProgramMemory(uint64_t search_addr, uint64_t size,
-                           const char* name);
+                               Common::VirtualMemory::Mode mode, const char* name);
-void UpdateProgramMemoryProtection(uint64_t vaddr, uint64_t size, Common::VirtualMemory::Mode mode);
+void SetProgramMemoryProtection(uint64_t vaddr, uint64_t size, Common::VirtualMemory::Mode mode);
-void UnregisterProgramMemory(uint64_t vaddr, uint64_t size);
+uint64_t AllocateRuntimeMemory(uint64_t search_addr, uint64_t size,
+                               Common::VirtualMemory::Mode mode, const char* name,
+                               bool fixed = false);
+uint64_t AllocateGuestStackMemory(uint64_t search_addr, uint64_t size,
+                                  Common::VirtualMemory::Mode mode, const char* name);
+bool     ProtectGuestMemory(uint64_t vaddr, uint64_t size, Common::VirtualMemory::Mode mode,
+                            Common::VirtualMemory::Mode* old_mode = nullptr);
+// Transient PageManager watch state; does not change the guest mapping's semantic protection.
+bool ProtectGuestHostMemory(uint64_t vaddr, uint64_t size, Common::VirtualMemory::Mode mode);
+bool FreeGuestMemory(uint64_t vaddr, uint64_t size);
 
 #if defined(KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS)
-void TestFailNextPhysicalMemoryUnmap();
+void     TestFailNextPhysicalMemoryUnmap();
-void TestFailPhysicalMemoryUnmapAfter(uint32_t successful_unmaps);
+void     TestFailPhysicalMemoryUnmapAfter(uint32_t successful_unmaps);
-void TestFailHostReservationAfter(uint32_t successful_pages);
+void     TestFailGuestBackingStoreUnmapAfter(uint32_t successful_unmaps);
-void TestFailNextFixedReserveRangeRegistration();
+void     TestFailNextFixedReserveRangeRegistration();
-bool TestPlaceholderRangeIsFree(uint64_t vaddr, uint64_t size);
+bool     TestPlaceholderRangeIsFree(uint64_t vaddr, uint64_t size);
+bool     TestGuestAddressRangeIsOwned(uint64_t vaddr, uint64_t size);
+bool     TestGuestBackingOutsideAddressSpace();
+uint64_t TestGuestBackingSize();
+bool     TestGuestFreeRangeBounds();
 #endif
 
 } // namespace Libs::LibKernel::Memory

src/kernel/pthread.cpp

@@ -75,16 +75,16 @@ LIB_NAME("libkernel", "libkernel");
 #undef PTHREAD_STACK_MIN
 #endif
 
-constexpr int      KEYS_MAX                  = 256;
+constexpr int      KEYS_MAX                = 256;
-constexpr int      DESTRUCTOR_ITERATIONS     = 4;
+constexpr int      DESTRUCTOR_ITERATIONS   = 4;
-constexpr size_t   PTHREAD_STACK_DEFAULT     = 0x100000;
+constexpr size_t   PTHREAD_STACK_DEFAULT   = 0x100000;
-constexpr size_t   GUEST_PTHREAD_STACK_MIN   = 0x4000;
+constexpr size_t   GUEST_PTHREAD_STACK_MIN = 0x4000;
-constexpr size_t   PTHREAD_STACK_PAGE        = 0x4000;
+constexpr size_t   PTHREAD_STACK_PAGE      = 0x4000;
-constexpr size_t   PTHREAD_STACK_GRANULARITY = 0x10000;
+constexpr size_t   PTHREAD_STACK_INITIAL   = 0x200000;
-constexpr size_t   PTHREAD_STACK_INITIAL     = 0x200000;
+constexpr size_t   PTHREAD_STACK_EXTRA     = 0x100000;
-constexpr size_t   PTHREAD_STACK_EXTRA       = 0x100000;
+constexpr uint64_t PTHREAD_STACK_TOP       = 0x7efff8000ull;
-constexpr uint64_t PTHREAD_STACK_TOP         = 0x7efff8000ull;
+constexpr uint64_t PTHREAD_STACK_BOTTOM    = 0x0000040000ull;
-constexpr uint32_t SIGNAL_APC_POLL_MICROS    = 10000;
+constexpr uint32_t SIGNAL_APC_POLL_MICROS  = 10000;
 
 static constexpr KernelClockid KERNEL_CLOCK_REALTIME          = 0;
 static constexpr KernelClockid KERNEL_CLOCK_VIRTUAL           = 1;
@@ -697,16 +697,17 @@ static std::atomic<int32_t> g_pthread_thread_id      = 0;
 
 static Common::Mutex g_guest_stack_mutex;
 static uint64_t      g_guest_stack_last = 0;
+struct CachedGuestStack {
+	uint64_t address;
+	size_t   map_size;
+	size_t   guard_size;
+};
+static std::vector<CachedGuestStack> g_guest_stack_cache;
 
 static size_t RoundStackSize(size_t size) {
 	return ((size + PTHREAD_STACK_PAGE - 1) / PTHREAD_STACK_PAGE) * PTHREAD_STACK_PAGE;
 }
 
-static size_t RoundStackMappingSize(size_t size) {
-	return ((size + PTHREAD_STACK_GRANULARITY - 1) / PTHREAD_STACK_GRANULARITY) *
-	       PTHREAD_STACK_GRANULARITY;
-}
-
 static int CreateGuestStack(PthreadAttr attr) {
 	if (attr == nullptr) {
 		return KERNEL_ERROR_EINVAL;
@@ -722,34 +723,41 @@ static int CreateGuestStack(PthreadAttr attr) {
 
 	const auto stack_size = RoundStackSize(attr->stack_size);
 	const auto guard_size = RoundStackSize(attr->guard_size);
-	const auto map_size   = RoundStackMappingSize(stack_size + guard_size);
+	const auto map_size   = stack_size + guard_size;
 
 	uint64_t stack_addr = 0;
+	bool     cached     = false;
 	{
 		Common::LockGuard lock(g_guest_stack_mutex);
 
-		if (g_guest_stack_last == 0) {
+		auto cached_stack =
-			g_guest_stack_last = (PTHREAD_STACK_TOP - PTHREAD_STACK_INITIAL - PTHREAD_STACK_PAGE) &
+		    std::find_if(g_guest_stack_cache.begin(), g_guest_stack_cache.end(),
-			                     ~(static_cast<uint64_t>(PTHREAD_STACK_GRANULARITY) - 1);
+		                 [map_size, guard_size](const auto& stack) {
+			                 return stack.map_size == map_size && stack.guard_size == guard_size;
+		                 });
+		if (cached_stack != g_guest_stack_cache.end()) {
+			stack_addr = cached_stack->address;
+			g_guest_stack_cache.erase(cached_stack);
+			cached = true;
+		} else {
+			if (g_guest_stack_last == 0) {
+				g_guest_stack_last = PTHREAD_STACK_TOP - PTHREAD_STACK_INITIAL - PTHREAD_STACK_PAGE;
+			}
+			if (map_size > g_guest_stack_last - PTHREAD_STACK_BOTTOM) {
+				return KERNEL_ERROR_EAGAIN;
+			}
+			stack_addr = g_guest_stack_last - map_size;
+			g_guest_stack_last -= map_size;
 		}
-
-		stack_addr = g_guest_stack_last - map_size;
-		g_guest_stack_last -= map_size;
 	}
 
-	void* mapped_addr = reinterpret_cast<void*>(stack_addr);
+	int result = OK;
-
+	if (!cached) {
-	constexpr int GUEST_PROT_READ_WRITE = 0x03;
+		stack_addr = Memory::AllocateGuestStackMemory(
-	constexpr int GUEST_MAP_PRIVATE     = 0x02;
+		    stack_addr, map_size, Common::VirtualMemory::Mode::ReadWrite, "stack");
-	constexpr int GUEST_MAP_FIXED       = 0x10;
+		if (stack_addr == 0) {
-	constexpr int GUEST_MAP_STACK       = 0x400;
+			return KERNEL_ERROR_EAGAIN;
-	constexpr int GUEST_MAP_ANON        = 0x1000;
+		}
-
-	int result = Memory::KernelMapNamedFlexibleMemory(
-	    &mapped_addr, map_size, GUEST_PROT_READ_WRITE,
-	    GUEST_MAP_PRIVATE | GUEST_MAP_FIXED | GUEST_MAP_STACK | GUEST_MAP_ANON, "stack");
-	if (result != OK) {
-		return KERNEL_ERROR_EAGAIN;
 	}
 
 	if (guard_size != 0) {
@@ -761,7 +769,7 @@ static int CreateGuestStack(PthreadAttr attr) {
 	}
 
 	attr->stack_addr     = reinterpret_cast<void*>(stack_addr + guard_size);
-	attr->stack_size     = map_size - guard_size;
+	attr->stack_size     = stack_size;
 	attr->stack_user     = false;
 	attr->stack_map_addr = stack_addr;
 	attr->stack_map_size = map_size;
@@ -777,13 +785,90 @@ static void FreeGuestStack(PthreadAttr attr) {
 		return;
 	}
 
-	Memory::KernelMunmap(attr->stack_map_addr, attr->stack_map_size);
+	const auto guard_size = attr->stack_map_size - attr->stack_size;
+	{
+		Common::LockGuard lock(g_guest_stack_mutex);
+		g_guest_stack_cache.push_back({attr->stack_map_addr, attr->stack_map_size, guard_size});
+	}
 
 	attr->stack_addr     = nullptr;
 	attr->stack_map_addr = 0;
 	attr->stack_map_size = 0;
 }
 
+#if defined(KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS)
+bool TestGuestStackOwnerLifecycle(uint64_t* first_address, uint64_t* second_address,
+                                  uint64_t* map_size) {
+	if (first_address == nullptr || second_address == nullptr || map_size == nullptr) {
+		return false;
+	}
+
+	size_t flexible_before = 0;
+	if (Memory::KernelAvailableFlexibleMemorySize(&flexible_before) != OK) {
+		return false;
+	}
+
+	PthreadAttr attr = nullptr;
+	if (PthreadAttrInit(&attr) != OK) {
+		return false;
+	}
+	if (CreateGuestStack(attr) != OK) {
+		PthreadAttrDestroy(&attr);
+		return false;
+	}
+	*first_address = attr->stack_map_addr;
+	*map_size      = attr->stack_map_size;
+	const bool first_owned =
+	    Memory::TestGuestAddressRangeIsOwned(*first_address, static_cast<uint64_t>(*map_size));
+	uint64_t   backing_value = 0;
+	const bool first_private =
+	    !Memory::TryReadBacking(*first_address, &backing_value, sizeof(backing_value));
+	size_t     flexible_during_first = 0;
+	const bool first_capacity_unchanged =
+	    Memory::KernelAvailableFlexibleMemorySize(&flexible_during_first) == OK &&
+	    flexible_during_first == flexible_before;
+	FreeGuestStack(attr);
+
+	if (CreateGuestStack(attr) != OK) {
+		PthreadAttrDestroy(&attr);
+		return false;
+	}
+	*second_address = attr->stack_map_addr;
+	const bool second_owned =
+	    Memory::TestGuestAddressRangeIsOwned(*second_address, static_cast<uint64_t>(*map_size));
+	const bool second_private =
+	    !Memory::TryReadBacking(*second_address, &backing_value, sizeof(backing_value));
+	size_t     flexible_during_second = 0;
+	const bool second_capacity_unchanged =
+	    Memory::KernelAvailableFlexibleMemorySize(&flexible_during_second) == OK &&
+	    flexible_during_second == flexible_before;
+	FreeGuestStack(attr);
+
+	CachedGuestStack cached {};
+	bool             found = false;
+	{
+		Common::LockGuard lock(g_guest_stack_mutex);
+		const auto        entry = std::find_if(
+		    g_guest_stack_cache.begin(), g_guest_stack_cache.end(),
+		    [second_address](const auto& stack) { return stack.address == *second_address; });
+		if (entry != g_guest_stack_cache.end()) {
+			cached = *entry;
+			g_guest_stack_cache.erase(entry);
+			found = true;
+		}
+	}
+
+	const bool unmapped = found && Memory::KernelMunmap(cached.address, cached.map_size) == OK;
+	size_t     flexible_after = 0;
+	const bool final_capacity_unchanged =
+	    Memory::KernelAvailableFlexibleMemorySize(&flexible_after) == OK &&
+	    flexible_after == flexible_before;
+	return PthreadAttrDestroy(&attr) == OK && first_owned && first_private &&
+	       first_capacity_unchanged && second_owned && second_private &&
+	       second_capacity_unchanged && unmapped && final_capacity_unchanged;
+}
+#endif
+
 static KYTY_SYSV_ABI void* RunOnGuestStack(void* arg, pthread_entry_func_t func, void* stack_top) {
 #if defined(__x86_64__) || defined(_M_X64)
 	void*      ret       = nullptr;

src/kernel/pthread.h

@@ -112,11 +112,15 @@ void                  PthreadQueuePendingSignal(Pthread thread, int signum);
 bool                  PthreadHasPendingSignal(Pthread thread, int signum);
 bool                  PthreadTakePendingSignal(Pthread thread, int signum);
 bool PthreadGetGuestStack(Pthread thread, uint64_t* stack_addr, uint64_t* stack_size);
+#if defined(KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS)
+bool TestGuestStackOwnerLifecycle(uint64_t* first_address, uint64_t* second_address,
+                                  uint64_t* map_size);
+#endif
 #if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS
 bool PthreadKillHost(Pthread thread, int host_signal);
 #endif
-int  PthreadGetPriorityForKernel(Pthread thread);
+int PthreadGetPriorityForKernel(Pthread thread);
-int  PthreadGetCurrentPriorityForKernel();
+int PthreadGetCurrentPriorityForKernel();
 
 int KYTY_SYSV_ABI          KernelUsleep(KernelUseconds microseconds);
 unsigned int KYTY_SYSV_ABI KernelSleep(unsigned int seconds);

src/loader/gamePatch.cpp

@@ -2,6 +2,7 @@
 
 #include "common/stringUtils.h"
 #include "common/virtualMemory.h"
+#include "kernel/memory.h"
 #include "loader/elf.h"
 #include "loader/runtimeLinker.h"
 #include "loader/systemContent.h"
@@ -153,18 +154,6 @@ bool ValidateTarget(const Plan& plan, const Program* program, std::string* error
 	return true;
 }
 
-Common::VirtualMemory::Mode ReadableMode(Elf64_Word flags) {
-	const bool executable = (flags & PF_X) != 0;
-	const bool writable   = (flags & PF_W) != 0;
-	if (executable && writable) {
-		return Common::VirtualMemory::Mode::ExecuteReadWrite;
-	}
-	if (executable) {
-		return Common::VirtualMemory::Mode::ExecuteRead;
-	}
-	return writable ? Common::VirtualMemory::Mode::ReadWrite : Common::VirtualMemory::Mode::Read;
-}
-
 bool ResolveWrite(const Program& program, Write* write, std::string* error) {
 	const auto* ehdr = program.elf->GetEhdr();
 	const auto* phdr = program.elf->GetPhdr();
@@ -178,16 +167,9 @@ bool ResolveWrite(const Program& program, Write* write, std::string* error) {
 			continue;
 		}
 
-		const auto                  segment_address = program.base_vaddr + segment.p_vaddr;
+		const auto  segment_address = program.base_vaddr + segment.p_vaddr;
-		const bool                  add_read        = (segment.p_flags & PF_R) == 0;
+		const auto* begin           = reinterpret_cast<const uint8_t*>(segment_address);
-		Common::VirtualMemory::Mode old_mode {};
+		const auto* end             = begin + segment.p_filesz;
-		if (add_read && !Common::VirtualMemory::Protect(segment_address, segment.p_memsz,
-		                                                ReadableMode(segment.p_flags), &old_mode)) {
-			return Fail(error, "could not read a loaded executable segment");
-		}
-
-		const auto* begin = reinterpret_cast<const uint8_t*>(segment_address);
-		const auto* end   = begin + segment.p_filesz;
 		for (auto* current = begin; current < end;) {
 			const auto* found =
 			    std::search(current, end, write->expected.begin(), write->expected.end());
@@ -202,15 +184,9 @@ bool ResolveWrite(const Program& program, Write* write, std::string* error) {
 			match_count++;
 			current = found + 1;
 		}
-
-		if (add_read &&
-		    !Common::VirtualMemory::Protect(segment_address, segment.p_memsz, old_mode)) {
-			return Fail(error, "could not restore executable segment protection");
-		}
 	}
 
-	::printf("Game patch: found %zu entries for '%s'\n", match_count,
+	::printf("Game patch: found %zu entries for '%s'\n", match_count, write->patch_name.c_str());
-	         write->patch_name.c_str());
 	if (match == 0) {
 		return Fail(error, "original bytes not found for '" + write->patch_name + "'");
 	}
@@ -229,16 +205,9 @@ bool PrepareWrites(Plan* plan, const Program& program, std::string* error) {
 
 bool ApplyWrites(Plan* plan, std::string* error) {
 	for (auto& write: plan->writes) {
-		Common::VirtualMemory::Mode old_mode {};
+		const auto size = write.replacement.size();
-		const auto                  size = write.replacement.size();
-		if (!Common::VirtualMemory::Protect(
-		        write.address, size, Common::VirtualMemory::Mode::ExecuteReadWrite, &old_mode)) {
-			return Fail(error, "could not make patch memory writable");
-		}
-
 		std::memcpy(reinterpret_cast<void*>(write.address), write.replacement.data(), size);
-		if (!Common::VirtualMemory::Protect(write.address, size, old_mode) ||
+		if (!Common::VirtualMemory::FlushInstructionCache(write.address, size)) {
-		    !Common::VirtualMemory::FlushInstructionCache(write.address, size)) {
 			return Fail(error, "could not finalize patch");
 		}
 	}

src/loader/runtimeLinker.cpp

@@ -62,14 +62,16 @@ static void FreeTlsBlock(ThreadLocalStorage::Block* block) {
 	if (block->free_func != nullptr) {
 		block->free_func(block->ptr);
 	} else if (block->vm_alloc) {
-		Common::VirtualMemory::Free(reinterpret_cast<uint64_t>(block->ptr));
+		EXIT_IF(!Libs::LibKernel::Memory::FreeGuestMemory(reinterpret_cast<uint64_t>(block->ptr),
+		                                                  block->alloc_size));
 	} else {
 		delete[] block->ptr;
 	}
 
-	block->ptr       = nullptr;
+	block->ptr        = nullptr;
-	block->free_func = nullptr;
+	block->free_func  = nullptr;
-	block->vm_alloc  = false;
+	block->vm_alloc   = false;
+	block->alloc_size = 0;
 }
 
 static uint64_t AlignUp(uint64_t value, uint64_t alignment) {
@@ -131,17 +133,25 @@ static std::vector<StubbedImportRecord> g_stubbed_imports;
 static std::atomic_uint32_t             g_unresolved_stub_call_log_count {0};
 static std::vector<uint64_t>            g_unresolved_stub_thunk_pages;
 static uint64_t                         g_unresolved_stub_thunk_offset = 0;
+static constexpr uint64_t               UNRESOLVED_STUB_PAGE_SIZE      = 4096;
 
 static KYTY_SYSV_ABI uint64_t ResolveImportStubWithId(uint64_t record_id);
 
+static bool PatchGuestMemory64(uint64_t vaddr, uint64_t value) {
+	auto* ptr     = reinterpret_cast<uint64_t*>(vaddr);
+	bool  changed = (*ptr != value);
+	std::memcpy(ptr, &value, sizeof(value));
+	return changed;
+}
+
 static uint64_t AllocateUnresolvedImportThunk(uint64_t record_id) {
-	constexpr uint64_t page_size  = 4096;
 	constexpr uint64_t thunk_size = 162;
 
 	if (g_unresolved_stub_thunk_pages.empty() ||
-	    g_unresolved_stub_thunk_offset + thunk_size > page_size) {
+	    g_unresolved_stub_thunk_offset + thunk_size > UNRESOLVED_STUB_PAGE_SIZE) {
-		auto page = Common::VirtualMemory::Alloc(0, page_size,
+		auto page = Libs::LibKernel::Memory::AllocateRuntimeMemory(
-		                                         Common::VirtualMemory::Mode::ExecuteReadWrite);
+		    0, UNRESOLVED_STUB_PAGE_SIZE, Common::VirtualMemory::Mode::ExecuteReadWrite,
+		    "unresolved_import_thunk");
 		EXIT_NOT_IMPLEMENTED(page == 0);
 		g_unresolved_stub_thunk_pages.push_back(page);
 		g_unresolved_stub_thunk_offset = 0;
@@ -298,7 +308,7 @@ static KYTY_SYSV_ABI uint64_t ResolveImportStubWithId(uint64_t record_id) {
 			     resolved.name.c_str(), resolved.vaddr);
 
 			if (record.patch_vaddr != 0) {
-				*reinterpret_cast<uint64_t*>(record.patch_vaddr) = resolved.vaddr;
+				PatchGuestMemory64(record.patch_vaddr, resolved.vaddr);
 			}
 
 			return resolved.vaddr;
@@ -360,7 +370,7 @@ static KYTY_SYSV_ABI void RunEntry(uint64_t addr, EntryParams* params, atexit_fu
 		register uintptr_t    guest_rbp_reg asm("r15") = guest_rbp;
 #endif
 
-#if defined(__APPLE__) || KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
+#if defined(__APPLE__)
 		asm volatile(
 		    "pushq %%r12\n\t"
 		    "pushq %%r13\n\t"
@@ -374,16 +384,44 @@ static KYTY_SYSV_ABI void RunEntry(uint64_t addr, EntryParams* params, atexit_fu
 		    "popq %%r13\n\t"
 		    "popq %%r12\n\t"
 		    :
-#if defined(__APPLE__)
 		    : [func] "r"(func_reg), "D"(params),
 		      "S"(atexit_func), [guest_rsp] "r"(guest_rsp_reg), [guest_rbp] "r"(guest_rbp_reg)
-#else
-		    : [func] "r"(func), "D"(params),
-		      "S"(atexit_func), [guest_rsp] "r"(guest_rsp), [guest_rbp] "r"(guest_rbp)
-#endif
 		    : "cc", "memory", "rax", "rcx", "rdx", "r8", "r9", "r10", "r11", "xmm0", "xmm1", "xmm2",
 		      "xmm3", "xmm4", "xmm5", "xmm6", "xmm7", "xmm8", "xmm9", "xmm10", "xmm11", "xmm12",
 		      "xmm13", "xmm14", "xmm15");
+#elif KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
+		// Windows stack probes use the TEB stack limits during the guest stack switch.
+		// bounds, which describe the host stack and are invalid while RSP is in guest memory.
+		register entry_func_t func_reg asm("rbx")     = func;
+		register uintptr_t    guest_rsp_reg asm("r8") = guest_rsp;
+		register uintptr_t    guest_rbp_reg asm("r9") = guest_rbp;
+		asm volatile("pushq %%r12\n\t"
+		             "pushq %%r13\n\t"
+		             "pushq %%r14\n\t"
+		             "pushq %%r15\n\t"
+		             "movq %%gs:0x08, %%r14\n\t"
+		             "movq %%gs:0x10, %%r15\n\t"
+		             "xorq %%rcx, %%rcx\n\t"
+		             "movq %%rcx, %%gs:0x08\n\t"
+		             "movq %%rcx, %%gs:0x10\n\t"
+		             "movq %%rsp, %%r12\n\t"
+		             "movq %%rbp, %%r13\n\t"
+		             "movq %[guest_rsp], %%rsp\n\t"
+		             "movq %[guest_rbp], %%rbp\n\t"
+		             "callq *%[func]\n\t"
+		             "movq %%r13, %%rbp\n\t"
+		             "movq %%r12, %%rsp\n\t"
+		             "movq %%r14, %%gs:0x08\n\t"
+		             "movq %%r15, %%gs:0x10\n\t"
+		             "popq %%r15\n\t"
+		             "popq %%r14\n\t"
+		             "popq %%r13\n\t"
+		             "popq %%r12\n\t"
+		             : [guest_rsp] "+r"(guest_rsp_reg), [guest_rbp] "+r"(guest_rbp_reg)
+		             : [func] "r"(func_reg), "D"(params), "S"(atexit_func)
+		             : "cc", "memory", "rax", "rcx", "rdx", "r10", "r11", "xmm0", "xmm1", "xmm2",
+		               "xmm3", "xmm4", "xmm5", "xmm6", "xmm7", "xmm8", "xmm9", "xmm10", "xmm11",
+		               "xmm12", "xmm13", "xmm14", "xmm15");
 #else
 		// Clobbers prevent inputs from being allocated to r12/r13.
 		asm volatile("movq %%rsp, %%r12\n\t"
@@ -449,6 +487,110 @@ static KYTY_SYSV_ABI void RunEntry(uint64_t addr, EntryParams* params, atexit_fu
 #endif
 }
 
+#if defined(KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS)
+struct MainEntryStackTestState {
+	bool      called = false;
+	uintptr_t rsp    = 0;
+#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
+	uintptr_t teb_stack_base  = UINTPTR_MAX;
+	uintptr_t teb_stack_limit = UINTPTR_MAX;
+#endif
+};
+
+static KYTY_SYSV_ABI void TestMainEntryStackCallback(EntryParams* params,
+                                                     atexit_func_t /*atexit_func*/) {
+	auto* state = reinterpret_cast<MainEntryStackTestState*>(const_cast<char*>(params->argv[0]));
+	asm volatile("movq %%rsp, %0" : "=r"(state->rsp) : : "memory");
+#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
+	asm volatile("movq %%gs:0x08, %0\n\t"
+	             "movq %%gs:0x10, %1\n\t"
+	             : "=r"(state->teb_stack_base), "=r"(state->teb_stack_limit)
+	             :
+	             : "memory");
+#endif
+	state->called = true;
+}
+
+bool TestMainEntryUsesGuestStack() {
+	constexpr uint64_t stack_size = 0x10000;
+	const auto         stack_base = Libs::LibKernel::Memory::AllocateRuntimeMemory(
+	    0, stack_size, Common::VirtualMemory::Mode::ReadWrite, "main_entry_stack_test");
+	if (stack_base == 0) {
+		return false;
+	}
+
+	MainEntryStackTestState state {};
+	EntryParams             params {};
+	params.argv[0] = reinterpret_cast<const char*>(&state);
+
+#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
+	uintptr_t original_teb_stack_base  = 0;
+	uintptr_t original_teb_stack_limit = 0;
+	asm volatile("movq %%gs:0x08, %0\n\t"
+	             "movq %%gs:0x10, %1\n\t"
+	             : "=r"(original_teb_stack_base), "=r"(original_teb_stack_limit)
+	             :
+	             : "memory");
+#endif
+
+	RunEntry(reinterpret_cast<uint64_t>(TestMainEntryStackCallback), &params, nullptr,
+	         reinterpret_cast<void*>(stack_base + stack_size));
+
+#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
+	uintptr_t restored_teb_stack_base  = 0;
+	uintptr_t restored_teb_stack_limit = 0;
+	asm volatile("movq %%gs:0x08, %0\n\t"
+	             "movq %%gs:0x10, %1\n\t"
+	             : "=r"(restored_teb_stack_base), "=r"(restored_teb_stack_limit)
+	             :
+	             : "memory");
+	const bool teb_ok = state.teb_stack_base == 0 && state.teb_stack_limit == 0 &&
+	                    restored_teb_stack_base == original_teb_stack_base &&
+	                    restored_teb_stack_limit == original_teb_stack_limit;
+#else
+	constexpr bool teb_ok = true;
+#endif
+
+	const bool rsp_ok = state.rsp >= stack_base && state.rsp < stack_base + stack_size;
+	const bool freed  = Libs::LibKernel::Memory::FreeGuestMemory(stack_base, stack_size);
+	return state.called && rsp_ok && teb_ok && freed;
+}
+
+bool TestModuleRelocationUsesWritableHostMapping() {
+	constexpr uint64_t page_size = 0x4000;
+	constexpr uint64_t value     = 0x4b59545950415443;
+	const auto         base      = Libs::LibKernel::Memory::AllocateProgramMemory(
+	    0, page_size, Common::VirtualMemory::Mode::ReadWrite, "host_only_patch_test");
+	if (base == 0) {
+		return false;
+	}
+	Libs::LibKernel::Memory::SetProgramMemoryProtection(base, page_size,
+	                                                    Common::VirtualMemory::Mode::Read);
+
+	Libs::LibKernel::Memory::VirtualQueryInfo before {};
+	Libs::LibKernel::Memory::VirtualQueryInfo after {};
+	const bool                                before_ok =
+	    Libs::LibKernel::Memory::KernelVirtualQuery(reinterpret_cast<const void*>(base), 0, &before,
+	                                                sizeof(before)) == 0;
+	const bool changed  = PatchGuestMemory64(base, value);
+	const bool after_ok = Libs::LibKernel::Memory::KernelVirtualQuery(
+	                          reinterpret_cast<const void*>(base), 0, &after, sizeof(after)) == 0;
+	const bool value_ok = *reinterpret_cast<const uint64_t*>(base) == value;
+#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
+	MEMORY_BASIC_INFORMATION mbi {};
+	const bool               host_mode_ok =
+	    VirtualQuery(reinterpret_cast<const void*>(base), &mbi, sizeof(mbi)) != 0 &&
+	    mbi.Protect == PAGE_READWRITE;
+#else
+	constexpr bool host_mode_ok = true;
+#endif
+	const bool freed = Libs::LibKernel::Memory::FreeGuestMemory(base, page_size);
+
+	return before_ok && after_ok && changed && value_ok && host_mode_ok && freed &&
+	       before.protection == after.protection;
+}
+#endif
+
 static uint64_t GetAlignedSize(const Elf64_Phdr* p) {
 	return (p->p_align != 0 ? (p->p_memsz + (p->p_align - 1)) & ~(p->p_align - 1) : p->p_memsz);
 }
@@ -1038,7 +1180,7 @@ static void RelocateRecord(uint32_t index, Elf64_Rela* r, Program* program, bool
 	// KYTY_PROFILER_BLOCK("patch");
 
 	if (ri.resolved) {
-		patched = Common::VirtualMemory::PatchReplace(ri.vaddr, ri.value);
+		patched = PatchGuestMemory64(ri.vaddr, ri.value);
 	} else {
 		uint64_t value = 0;
 		bool     weak  = (ri.bind == BindType::Weak || !program->fail_if_global_not_resolved);
@@ -1056,7 +1198,7 @@ static void RelocateRecord(uint32_t index, Elf64_Rela* r, Program* program, bool
 		}
 
 		if (value != 0) {
-			patched = Common::VirtualMemory::PatchReplace(ri.vaddr, value);
+			patched = PatchGuestMemory64(ri.vaddr, value);
 		} else {
 			auto dbg_str = fmt::format("[{:016x}] <- {:016x}, {}, {}, {}, {}", ri.vaddr, ri.value,
 			                           ri.name.c_str(), Common::EnumName(ri.type).c_str(),
@@ -1079,7 +1221,7 @@ static void RelocateRecord(uint32_t index, Elf64_Rela* r, Program* program, bool
 			}
 
 			if (value != 0) {
-				patched = Common::VirtualMemory::PatchReplace(ri.vaddr, value);
+				patched = PatchGuestMemory64(ri.vaddr, value);
 			}
 		}
 	}
@@ -1459,6 +1601,7 @@ void RuntimeLinker::Execute(const std::filesystem::path& game_patch) {
 
 	PreloadAdjacentPrograms();
 	RelocateAll();
+
 	if (!game_patch.empty()) {
 		GamePatch::Apply(game_patch, m_programs.empty() ? nullptr : m_programs.front());
 	}
@@ -1489,6 +1632,21 @@ void RuntimeLinker::Clear() {
 		DeleteProgram(p);
 	}
 	m_programs.clear();
+	for (const auto page: g_unresolved_stub_thunk_pages) {
+		EXIT_IF(!Libs::LibKernel::Memory::FreeGuestMemory(page, UNRESOLVED_STUB_PAGE_SIZE));
+	}
+	g_unresolved_stub_thunk_pages.clear();
+	g_unresolved_stub_thunk_offset = 0;
+	g_stubbed_imports.clear();
+	g_unresolved_stub_call_log_count.store(0);
+	if (g_invalid_memory != 0) {
+		EXIT_IF(!Libs::LibKernel::Memory::FreeGuestMemory(g_invalid_memory, 4096));
+		g_invalid_memory = 0;
+	}
+	g_tls_main_program        = nullptr;
+	g_tls_cached_main_program = nullptr;
+	g_tls_cached_main_tcb     = nullptr;
+	g_desired_base_addr       = SYSTEM_RESERVED + CODE_BASE_OFFSET;
 	m_symbols.reset();
 	m_relocated = false;
 }
@@ -1926,10 +2084,11 @@ uint8_t* RuntimeLinker::TlsGetAddr(Program* program) {
 		const auto tcb_offset =
 		    program->tls.tcb_offset != 0 ? program->tls.tcb_offset : program->tls.image_size;
 		const auto alloc_size = AlignUp(tcb_offset, TCB_ALIGN) + TCB_SIZE;
-		tls.ptr               = reinterpret_cast<uint8_t*>(
+		tls.ptr        = reinterpret_cast<uint8_t*>(Libs::LibKernel::Memory::AllocateRuntimeMemory(
-		    Common::VirtualMemory::Alloc(0, alloc_size, Common::VirtualMemory::Mode::ReadWrite));
+		    0, alloc_size, Common::VirtualMemory::Mode::ReadWrite, "thread_local_storage"));
-		tls.free_func = nullptr;
+		tls.free_func  = nullptr;
-		tls.vm_alloc  = true;
+		tls.vm_alloc   = true;
+		tls.alloc_size = alloc_size;
 
 		EXIT_IF(tls.ptr == nullptr);
 
@@ -2006,8 +2165,9 @@ void RuntimeLinker::LoadProgramToMemory(Program* program) {
 	EXIT_IF(tls_handler_size > UINT64_MAX - program->base_size_aligned);
 	program->mapped_size = program->base_size_aligned + tls_handler_size;
 
-	program->base_vaddr = Common::VirtualMemory::Alloc(
+	program->base_vaddr = Libs::LibKernel::Memory::AllocateProgramMemory(
-	    g_desired_base_addr, program->mapped_size, Common::VirtualMemory::Mode::ExecuteReadWrite);
+	    g_desired_base_addr, program->mapped_size, Common::VirtualMemory::Mode::ExecuteReadWrite,
+	    Common::PathToString(program->file_name.filename()).c_str());
 
 	if (!is_shared) {
 		program->tls.handler_vaddr = program->base_vaddr + program->base_size_aligned;
@@ -2017,10 +2177,6 @@ void RuntimeLinker::LoadProgramToMemory(Program* program) {
 
 	EXIT_IF(program->base_vaddr == 0);
 	EXIT_IF(program->base_size_aligned < program->base_size);
-	Libs::LibKernel::Memory::RegisterProgramMemory(
-	    program->base_vaddr, program->mapped_size, Common::VirtualMemory::Mode::ExecuteReadWrite,
-	    Common::PathToString(program->file_name.filename()).c_str());
-
 	LOGF("base_vaddr             = 0x%016" PRIx64 "\n"
 	     "base_size              = 0x%016" PRIx64 "\n"
 	     "base_size_aligned      = 0x%016" PRIx64 "\n"
@@ -2060,11 +2216,8 @@ void RuntimeLinker::LoadProgramToMemory(Program* program) {
 			}
 
 			if (!skip_protect) {
-				if (!Common::VirtualMemory::Protect(segment_addr, segment_memory_size, mode)) {
+				Libs::LibKernel::Memory::SetProgramMemoryProtection(segment_addr,
-					EXIT("failed to protect ELF segment %u\n", static_cast<unsigned>(i));
+				                                                    segment_memory_size, mode);
-				}
-				Libs::LibKernel::Memory::UpdateProgramMemoryProtection(segment_addr,
-				                                                       segment_memory_size, mode);
 
 				if (Common::VirtualMemory::IsExecute(mode)) {
 					Common::VirtualMemory::FlushInstructionCache(segment_addr, segment_memory_size);
@@ -2105,15 +2258,29 @@ void RuntimeLinker::LoadProgramToMemory(Program* program) {
 
 void RuntimeLinker::DeleteProgram(Program* p) {
 	auto program = std::unique_ptr<Program>(p);
+	if (g_tls_main_program == program.get()) {
+		g_tls_main_program = nullptr;
+	}
+	if (g_tls_cached_main_program == program.get()) {
+		g_tls_cached_main_program = nullptr;
+		g_tls_cached_main_tcb     = nullptr;
+	}
+	for (auto& record: g_stubbed_imports) {
+		if (record.patch_vaddr >= program->base_vaddr &&
+		    record.patch_vaddr < program->base_vaddr + program->mapped_size) {
+			record.patch_vaddr = 0;
+		}
+	}
 
 	if (program->base_vaddr != 0 || program->mapped_size != 0) {
 		EXIT_IF(program->base_vaddr == 0 || program->mapped_size == 0);
-		Libs::LibKernel::Memory::UnregisterProgramMemory(program->base_vaddr, program->mapped_size);
+		EXIT_IF(
-		EXIT_IF(!Common::VirtualMemory::Free(program->base_vaddr));
+		    !Libs::LibKernel::Memory::FreeGuestMemory(program->base_vaddr, program->mapped_size));
 	}
 
 	if (program->custom_call_plt_vaddr != 0 || program->custom_call_plt_num != 0) {
-		Common::VirtualMemory::Free(program->custom_call_plt_vaddr);
+		const auto size = Jit::CallPlt::GetSize(program->custom_call_plt_num);
+		EXIT_IF(!Libs::LibKernel::Memory::FreeGuestMemory(program->custom_call_plt_vaddr, size));
 	}
 }
 
@@ -2237,13 +2404,13 @@ static void InstallRelocateHandler(Program* program) {
 	void**   pltgot       = reinterpret_cast<void**>(pltgot_vaddr);
 
 	Common::VirtualMemory::Mode old_mode {};
-	Common::VirtualMemory::Protect(pltgot_vaddr, pltgot_size, Common::VirtualMemory::Mode::Write,
+	EXIT_IF(!Libs::LibKernel::Memory::ProtectGuestMemory(
-	                               &old_mode);
+	    pltgot_vaddr, pltgot_size, Common::VirtualMemory::Mode::Write, &old_mode));
 
 	pltgot[1] = program;
 	pltgot[2] = reinterpret_cast<void*>(RelocateHandler);
 
-	Common::VirtualMemory::Protect(pltgot_vaddr, pltgot_size, old_mode);
+	EXIT_IF(!Libs::LibKernel::Memory::ProtectGuestMemory(pltgot_vaddr, pltgot_size, old_mode));
 
 	if (Common::VirtualMemory::IsExecute(old_mode)) {
 		Common::VirtualMemory::FlushInstructionCache(pltgot_vaddr, pltgot_size);
@@ -2253,15 +2420,15 @@ static void InstallRelocateHandler(Program* program) {
 	if (program->custom_call_plt_vaddr == 0) {
 		program->custom_call_plt_num =
 		    program->dynamic_info->jmprela_table_size / sizeof(Elf64_Rela);
-		auto size = Jit::CallPlt::GetSize(program->custom_call_plt_num);
+		auto size                      = Jit::CallPlt::GetSize(program->custom_call_plt_num);
-		program->custom_call_plt_vaddr =
+		program->custom_call_plt_vaddr = Libs::LibKernel::Memory::AllocateRuntimeMemory(
-		    Common::VirtualMemory::Alloc(SYSTEM_RESERVED, size, Common::VirtualMemory::Mode::Write);
+		    SYSTEM_RESERVED, size, Common::VirtualMemory::Mode::Write, "custom_call_plt");
 		EXIT_NOT_IMPLEMENTED(program->custom_call_plt_vaddr == 0);
 		auto* code = new (reinterpret_cast<void*>(program->custom_call_plt_vaddr))
 		    Jit::CallPlt(program->custom_call_plt_num);
 		code->SetPltGot(pltgot_vaddr);
-		Common::VirtualMemory::Protect(program->custom_call_plt_vaddr, size,
+		EXIT_IF(!Libs::LibKernel::Memory::ProtectGuestMemory(program->custom_call_plt_vaddr, size,
-		                               Common::VirtualMemory::Mode::Execute);
+		                                                     Common::VirtualMemory::Mode::Execute));
 		Common::VirtualMemory::FlushInstructionCache(program->custom_call_plt_vaddr, size);
 	}
 }
@@ -2272,8 +2439,8 @@ void RuntimeLinker::Relocate(Program* program) {
 	EXIT_IF(program == nullptr);
 
 	if (g_invalid_memory == 0) {
-		g_invalid_memory = Common::VirtualMemory::Alloc(INVALID_MEMORY, 4096,
+		g_invalid_memory = Libs::LibKernel::Memory::AllocateRuntimeMemory(
-		                                                Common::VirtualMemory::Mode::NoAccess);
+		    INVALID_MEMORY, 4096, Common::VirtualMemory::Mode::NoAccess, "invalid_memory", true);
 		EXIT_NOT_IMPLEMENTED(g_invalid_memory == 0);
 	}
 
@@ -2447,12 +2614,9 @@ void RuntimeLinker::SetupTlsHandler(Program* program) {
 		stub->SetOutputReg(reg);
 	}
 
-	if (!Common::VirtualMemory::Protect(program->tls.handler_vaddr, Jit::SafeCall::GetSize(),
+	EXIT_IF(!Libs::LibKernel::Memory::ProtectGuestMemory(program->tls.handler_vaddr,
-	                                    Common::VirtualMemory::Mode::Execute)) {
+	                                                     Jit::SafeCall::GetSize(),
-		EXIT("failed to protect program TLS handler\n");
+	                                                     Common::VirtualMemory::Mode::Execute));
-	}
-	Libs::LibKernel::Memory::UpdateProgramMemoryProtection(
-	    program->tls.handler_vaddr, Jit::SafeCall::GetSize(), Common::VirtualMemory::Mode::Execute);
 	Common::VirtualMemory::FlushInstructionCache(program->tls.handler_vaddr,
 	                                             Jit::SafeCall::GetSize());
 }

src/loader/runtimeLinker.h

@@ -48,9 +48,10 @@ struct LibraryId {
 
 struct ThreadLocalStorage {
 	struct Block {
-		uint8_t*                     ptr       = nullptr;
+		uint8_t*                     ptr        = nullptr;
-		application_heap_free_func_t free_func = nullptr;
+		application_heap_free_func_t free_func  = nullptr;
-		bool                         vm_alloc  = false;
+		bool                         vm_alloc   = false;
+		uint64_t                     alloc_size = 0;
 	};
 
 	~ThreadLocalStorage();
@@ -204,6 +205,11 @@ private:
 	application_heap_posix_memalign_func_t m_application_heap_posix_memalign = nullptr;
 };
 
+#if defined(KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS)
+bool TestMainEntryUsesGuestStack();
+bool TestModuleRelocationUsesWritableHostMapping();
+#endif
+
 } // namespace Loader
 
 #endif /* EMULATOR_INCLUDE_EMULATOR_LOADER_RUNTIMELINKER_H_ */

tests/MemoryTrackerTests.cpp

@@ -1,6 +1,7 @@
 #include "graphics/host_gpu/memoryTracker.h"
 #include "graphics/host_gpu/rangeSet.h"
 #include "common/assert.h"
+#include "common/virtualMemory.h"
 
 #include <atomic>
 #include <cstdint>
@@ -209,6 +210,20 @@ class SharedPage final {
   HANDLE mapping_ = nullptr;
 };
 #endif
+
+bool ProtectAddressSpace(uint64_t vaddr, uint64_t size,
+                         Common::VirtualMemory::Mode mode) {
+  uint32_t protection = PAGE_NOACCESS;
+  if (mode == Common::VirtualMemory::Mode::Read) {
+    protection = PAGE_READONLY;
+  } else if (mode == Common::VirtualMemory::Mode::ReadWrite) {
+    protection = PAGE_READWRITE;
+  }
+  DWORD old_protection = 0;
+  return VirtualProtect(reinterpret_cast<void *>(vaddr), size, protection,
+                        &old_protection) != 0;
+}
+
 #if 1
 
 bool DummyFault(void *, PageFaultAccess, uint64_t, uint64_t, PageFaultPhase) noexcept {
@@ -353,7 +368,8 @@ struct DownloadTrackerHarness {
     return completed;
   }
 
-  DownloadTrackerHarness() : page_manager(Fault, this), tracker(page_manager) {}
+  DownloadTrackerHarness()
+      : page_manager(Fault, this), tracker(page_manager) {}
 
   PageFaultAccess pending_access = PageFaultAccess::Unknown;
   uint64_t download_address = 0;
@@ -366,11 +382,6 @@ struct DownloadTrackerHarness {
 
 std::atomic<PageManager *> g_native_page_manager{nullptr};
 std::atomic_bool g_native_fault_entered{false};
-std::atomic_bool g_unmap_contended{false};
-
-void UnmapContended() noexcept {
-  g_unmap_contended.store(true, std::memory_order_release);
-}
 
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
 LONG CALLBACK NativeTrackerFaultHandler(EXCEPTION_POINTERS *exception) {
@@ -552,7 +563,7 @@ void TestGpuDownloadFaultOwnership() {
             !harness.tracker.IsRegionGpuModified(address, page_size) &&
             harness.tracker.IsRegionCpuModified(address, page_size) && IsWritable(memory),
         "GPU write fault did not download before granting CPU ownership");
-  harness.tracker.UnmapMemory(address, page_size);
+  harness.tracker.UntrackMemory(address, page_size);
 }
 
 void TestVirtualGpuWriteDiscard() {
@@ -578,7 +589,7 @@ void TestVirtualGpuWriteDiscard() {
   Check(!harness.tracker.IsRegionGpuModified(address, page_size) &&
             harness.tracker.IsRegionCpuModified(address, page_size) && IsWritable(memory),
         "virtual GPU discard did not transfer the page to CPU ownership");
-  harness.tracker.UnmapMemory(address, page_size);
+  harness.tracker.UntrackMemory(address, page_size);
   Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
 }
 
@@ -684,6 +695,15 @@ void TestRangeSet() {
         "range set subtraction did not preserve both exact tails");
 }
 
+void TestQueriesDoNotRequireMappedOwnership() {
+  constexpr uint64_t address = 0x0000000203000000ull;
+  TrackerHarness harness;
+  const auto page_size = harness.page_manager.GetPageSize();
+  Check(harness.tracker.IsRegionCpuModified(address, page_size) &&
+            !harness.tracker.IsRegionGpuModified(address, page_size),
+        "unowned tracker range did not expose its initial CPU-dirty state");
+}
+
 void TestRangeInvalidation() {
   constexpr uintptr_t base = 0x0000000201000000ull;
   TrackerHarness harness;
@@ -767,7 +787,7 @@ void TestCpuDirtyUploadAndFault() {
   Check(IsWritable(memory),
         "explicit CPU dirty transition did not release the rearmed watch");
 
-  tracker.UnmapMemory(address, page_size * 2);
+  tracker.UntrackMemory(address, page_size * 2);
   Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
 }
 
@@ -795,7 +815,7 @@ void TestFaultDuringUploadRemainsDirty() {
       });
   Check(tracker.IsRegionCpuModified(address, page_size) && IsWritable(memory),
         "upload completion erased a racing CPU dirty transition");
-  tracker.UnmapMemory(address, page_size);
+  tracker.UntrackMemory(address, page_size);
   Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
 }
 
@@ -840,7 +860,7 @@ void TestNativeStoreDuringRangeEnumeration() {
             IsWritable(memory),
         "native store during range enumeration was lost");
 
-  tracker.UnmapMemory(address, page_size);
+  tracker.UntrackMemory(address, page_size);
   Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
 }
 
@@ -924,7 +944,7 @@ void TestFaultDuringDownloadSynchronization() {
             page_manager.IsTracked(address + page_size * 2),
         "uncontended dirty page did not retain its clean write watch");
 
-  tracker.UnmapMemory(address, page_size * 3);
+  tracker.UntrackMemory(address, page_size * 3);
 }
 
 void TestFaultAndExplicitDirtyRace() {
@@ -964,7 +984,7 @@ void TestFaultAndExplicitDirtyRace() {
               IsWritable(memory),
           "fault/explicit-dirty race lost dirty state or write access");
   }
-  tracker.UnmapMemory(address, page_size);
+  tracker.UntrackMemory(address, page_size);
   Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
 }
 
@@ -1016,7 +1036,7 @@ void TestSharedTrackersAndConcurrentPageFaults() {
   }
 
   harness.first.UntrackMemory(address, page_size * 2);
-  harness.second.UnmapMemory(address, page_size * 2);
+  harness.second.UntrackMemory(address, page_size * 2);
   Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
 }
 
@@ -1049,7 +1069,7 @@ void TestGpuDirtyBits() {
         "explicit GPU dirty transition did not trap CPU access");
   tracker.UnmarkRegionAsGpuModified(address, page_size);
   tracker.MarkRegionAsCpuModified(address, page_size);
-  tracker.UnmapMemory(address, page_size * 2);
+  tracker.UntrackMemory(address, page_size * 2);
   Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
 }
 
@@ -1084,7 +1104,7 @@ void TestCrossRegionUpload() {
         "cross-region written upload did not mark GPU dirty state");
   tracker.UnmarkRegionAsGpuModified(boundary - page_size, page_size * 2);
   tracker.MarkRegionAsCpuModified(boundary - page_size, page_size * 2);
-  tracker.UnmapMemory(address, region_size * 2);
+  tracker.UntrackMemory(address, region_size * 2);
   Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
 }
 
@@ -1094,9 +1114,6 @@ void TestCrossRegionUpload() {
   auto &tracker = harness.tracker;
   auto &page_manager = harness.page_manager;
   const auto page_size = page_manager.GetPageSize();
-  if (std::strcmp(name, "unmapped") == 0) {
-    (void)tracker.IsRegionCpuModified(base, page_size);
-  }
   const auto allocation_size =
       std::strcmp(name, "missing-download-bytes") == 0 ? page_size * 2
                                                        : page_size;
@@ -1149,20 +1166,6 @@ void TestCrossRegionUpload() {
           }
     });
     fault.join();
-  } else if (std::strcmp(name, "gpu-dirty-unmap-race") == 0) {
-    g_unmap_contended.store(false, std::memory_order_release);
-    MemoryTracker::SetUnmapContentionHook(UnmapContended);
-    std::thread unmap;
-    tracker.ForEachUploadRange(
-        address, page_size, true, [](uint64_t, uint64_t) noexcept {},
-        [&]() noexcept {
-          unmap = std::thread(
-              [&] { tracker.UnmapMemory(address, page_size); });
-          while (!g_unmap_contended.load(std::memory_order_acquire)) {
-            std::this_thread::yield();
-          }
-        });
-    unmap.join();
   } else if (std::strcmp(name, "missing-download-bytes") == 0) {
     tracker.ForEachUploadRange(
         address, allocation_size, true, [](uint64_t, uint64_t) noexcept {},
@@ -1193,8 +1196,7 @@ void TestFatalPaths() {
 #endif
   for (const char *name : {"gpu-dirty-fault", "gpu-dirty-read", "virtual-gpu-read",
                            "gpu-dirty-explicit-cpu",
-                           "unmapped", "reentrant-upload",
+                           "reentrant-upload", "writable-upload-race",
-                           "writable-upload-race", "gpu-dirty-unmap-race",
                            "missing-download-bytes"}) {
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
     std::string command = std::string("\"") + path + "\" --death " + name;
@@ -1236,6 +1238,14 @@ void TestFatalPaths() {
 
 } // namespace
 
+namespace Libs::LibKernel::Memory {
+
+bool ProtectGuestHostMemory(uint64_t vaddr, uint64_t size, Common::VirtualMemory::Mode mode) {
+  return ProtectAddressSpace(vaddr, size, mode);
+}
+
+} // namespace Libs::LibKernel::Memory
+
 int main(int argc, char **argv) {
 #if 1
   if (argc == 3 && std::strcmp(argv[1], "--death") == 0) {
@@ -1249,6 +1259,7 @@ int main(int argc, char **argv) {
   TestSameSlabTrackerArbitration();
   TestSharedMetadataAndImagePageFault();
   TestRangeSet();
+  TestQueriesDoNotRequireMappedOwnership();
   TestRangeInvalidation();
   TestGpuDirtyBits();
   TestCrossRegionUpload();

tests/PageManagerTests.cpp

@@ -1,4 +1,5 @@
 #include "graphics/host_gpu/pageManager.h"
+#include "common/virtualMemory.h"
 
 #include <atomic>
 #include <cstdint>
@@ -28,7 +29,6 @@
 
 namespace {
 
-using Libs::Graphics::GpuAccess;
 using Libs::Graphics::PageFaultAccess;
 using Libs::Graphics::PageManager;
 
@@ -122,6 +122,23 @@ uint32_t Protection(const void *address) {
   return info.Protect;
 }
 #endif
+
+std::atomic_uint64_t g_protection_calls{0};
+
+bool ProtectAddressSpace(uint64_t vaddr, uint64_t size,
+                         Common::VirtualMemory::Mode mode) {
+  uint32_t protection = PAGE_NOACCESS;
+  if (mode == Common::VirtualMemory::Mode::Read) {
+    protection = PAGE_READONLY;
+  } else if (mode == Common::VirtualMemory::Mode::ReadWrite) {
+    protection = PAGE_READWRITE;
+  }
+  DWORD old_protection = 0;
+  g_protection_calls.fetch_add(1, std::memory_order_relaxed);
+  return VirtualProtect(reinterpret_cast<void *>(vaddr), size, protection,
+                        &old_protection) != 0;
+}
+
 #if 1
 
 struct FaultContext {
@@ -254,6 +271,7 @@ uint8_t *Allocate(uint64_t size, uint32_t protection = PAGE_READWRITE) {
 }
 
 void TestWatchFaultAndUnwatch() {
+  g_protection_calls.store(0, std::memory_order_relaxed);
   FaultContext context;
   PageManager manager(InvalidateFault, &context);
   context.manager = &manager;
@@ -266,6 +284,8 @@ void TestWatchFaultAndUnwatch() {
   Check(manager.IsTracked(reinterpret_cast<uint64_t>(memory)) &&
             !IsWritable(memory),
         "watch did not protect the page");
+  Check(g_protection_calls.load(std::memory_order_relaxed) != 0,
+        "watch protection bypassed the address-space owner callback");
   Check(manager.HandleFault(PageFaultAccess::Write,
                             reinterpret_cast<uint64_t>(memory + 32)),
         "tracked write fault was not handled");
@@ -343,14 +363,6 @@ void TestPermittedMappedLateFaultsResume() {
         "second delayed mapped write was not accepted");
   Check(manager.HandleFault(PageFaultAccess::Read, address),
         "delayed mapped read was not accepted on readable backing");
-  DWORD old_protection = 0;
-  Check(VirtualProtect(memory, page_size, PAGE_READONLY, &old_protection) != 0 &&
-            old_protection == PAGE_READWRITE,
-        "failed to prepare intentional read-only protection");
-  Check(!manager.HandleFault(PageFaultAccess::Write, address),
-        "intentional read-only mapping accepted a write fault");
-  Check(VirtualProtect(memory, page_size, PAGE_READWRITE, &old_protection) != 0,
-        "failed to restore writable protection");
   manager.OnGpuUnmap(address, page_size);
   Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
 }
@@ -496,31 +508,6 @@ void TestNativeAccessViolation() {
   Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
 }
 
-void TestInvalidLateWriteTokenIsConsumed() {
-  FaultContext context;
-  PageManager manager(InvalidateFault, &context);
-  context.manager = &manager;
-  const auto page_size = manager.GetPageSize();
-  auto *memory = Allocate(page_size);
-  const auto address = reinterpret_cast<uint64_t>(memory);
-  manager.OnGpuMap(address, page_size);
-  manager.UpdatePageWatchers(true, address, page_size);
-  Check(manager.HandleFault(PageFaultAccess::Write, address),
-        "initial write fault was not handled");
-  DWORD old_protection = 0;
-  Check(VirtualProtect(memory, page_size, PAGE_READONLY, &old_protection) !=
-                0 &&
-            old_protection == PAGE_READWRITE,
-        "failed to create invalid late-write protection state");
-  Check(!manager.HandleFault(PageFaultAccess::Write, address) &&
-            !manager.HandleFault(PageFaultAccess::Write, address),
-        "invalid late-write token was accepted or retained");
-  Check(VirtualProtect(memory, page_size, PAGE_READWRITE, &old_protection) != 0,
-        "failed to restore test protection");
-  manager.OnGpuUnmap(address, page_size);
-  Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
-}
-
 void TestCrossRegionRange() {
   FaultContext context;
   PageManager manager(InvalidateFault, &context);
@@ -627,9 +614,7 @@ void TestBatchedWatcherRanges() {
     manager->UpdatePageWatchers(false, 0x1000, page_size);
   } else {
     const bool two_pages = std::strcmp(name, "cross-reentrant") == 0;
-    auto *memory = Allocate(
+    auto *memory = Allocate(two_pages ? page_size * 2 : page_size);
-        two_pages ? page_size * 2 : page_size,
-        std::strcmp(name, "protection") == 0 ? PAGE_READONLY : PAGE_READWRITE);
     const auto address = reinterpret_cast<uint64_t>(memory);
     manager->OnGpuMap(address, two_pages ? page_size * 2 : page_size);
     manager->UpdatePageWatchers(true, address, page_size);
@@ -659,9 +644,7 @@ void TestBatchedWatcherRanges() {
       }
       (void)manager->HandleFault(PageFaultAccess::Read, address);
       first.join();
-    } else if (std::strcmp(name, "watched-unmap") == 0) {
+    } else {
-      manager->OnGpuUnmap(address, page_size);
-    } else if (std::strcmp(name, "protection") != 0) {
       std::_Exit(0x7f);
     }
   }
@@ -712,7 +695,7 @@ void TestFatalPaths() {
   for (const char *name :
        {"invalid-range", "unknown-untrack", "destructor-watch", "non-write",
         "callback-false", "reentrant", "cross-reentrant",
-        "concurrent-non-write", "watched-unmap", "protection"}) {
+        "concurrent-non-write"}) {
     CheckDeathCase(name);
   }
 }
@@ -773,51 +756,18 @@ void TestExternalDirtyTransferDuringResolution() {
   Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
 }
 
-void TestMappingDoesNotRequireCpuWriteAccess() {
-  FaultContext context;
-  PageManager manager(InvalidateFault, &context);
-  context.manager = &manager;
-  const auto page_size = manager.GetPageSize();
-  auto *memory = Allocate(page_size);
-  DWORD old_protection = 0;
-  Check(VirtualProtect(memory, page_size, PAGE_NOACCESS, &old_protection) != 0 &&
-            old_protection == PAGE_READWRITE,
-        "failed to prepare CPU-inaccessible mapping");
-  const auto address = reinterpret_cast<uint64_t>(memory);
-  manager.OnGpuMap(address, page_size);
-  Check(manager.IsMapped(address, page_size),
-        "CPU-inaccessible committed range was not GPU mapped");
-  manager.OnGpuUnmap(address, page_size);
-  Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
-}
-
-void TestGpuAccessPermissions() {
-  FaultContext context;
-  PageManager manager(InvalidateFault, &context);
-  context.manager = &manager;
-  const auto page_size = manager.GetPageSize();
-  auto *memory = Allocate(page_size);
-  const auto address = reinterpret_cast<uint64_t>(memory);
-  manager.OnGpuMap(address, page_size, GpuAccess::Read);
-  Check(manager.HasGpuAccess(address, page_size, GpuAccess::Read) &&
-            !manager.HasGpuAccess(address, page_size, GpuAccess::Write),
-        "read-only GPU mapping granted write access");
-  manager.OnGpuMap(address, page_size, GpuAccess::Write);
-  Check(manager.HasGpuAccess(address, page_size, GpuAccess::ReadWrite),
-        "overlapping GPU mappings did not combine permissions");
-  manager.OnGpuUnmap(address, page_size, GpuAccess::Read);
-  Check(!manager.HasGpuAccess(address, page_size, GpuAccess::Read) &&
-            manager.HasGpuAccess(address, page_size, GpuAccess::Write),
-        "GPU read unmap removed the wrong permission");
-  manager.OnGpuUnmap(address, page_size, GpuAccess::Write);
-  Check(!manager.IsMapped(address, page_size),
-        "GPU permission mappings were not fully balanced");
-  Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
-}
 #endif
 
 } // namespace
 
+namespace Libs::LibKernel::Memory {
+
+bool ProtectGuestHostMemory(uint64_t vaddr, uint64_t size, Common::VirtualMemory::Mode mode) {
+  return ProtectAddressSpace(vaddr, size, mode);
+}
+
+} // namespace Libs::LibKernel::Memory
+
 int main(int argc, char **argv) {
 #if 1
   if (argc == 3 && std::strcmp(argv[1], "--death") == 0) {
@@ -831,13 +781,10 @@ int main(int argc, char **argv) {
   TestNativeDelayedReadAfterModeDowngrade();
   TestDelayedFaultAfterExplicitUnwatch();
   TestNativeAccessViolation();
-  TestInvalidLateWriteTokenIsConsumed();
   TestCrossRegionRange();
   TestBatchedWatcherRanges();
   TestConcurrentFault();
   TestExternalDirtyTransferDuringResolution();
-  TestMappingDoesNotRequireCpuWriteAccess();
-  TestGpuAccessPermissions();
   TestFatalPaths();
   std::puts("PageManagerTests: all cases passed");
   return 0;

tests/ShaderRecompilerComputeTests.cpp

@@ -1274,6 +1274,59 @@ public:
     std::printf("[host]    %-32s ok\n", "SchedulerTimeline");
   }
 
+  void CheckGpuMappedRangeLifecycle() {
+    EnsureRuntimeContext();
+    CommandScheduler scheduler(Renderer(), m_runtime_context);
+    HW::Context registers{};
+    HW::UserConfig user_config{};
+    HW::Shader shaders{};
+    scheduler.Begin(registers, user_config, shaders);
+    Gpu gpu(Renderer());
+    GpuResourceManager resources(m_runtime_context, scheduler);
+    resources.SetGpu(&gpu);
+
+    constexpr uint64_t base = 0x0000000200000000ull;
+    constexpr uint64_t page = 0x4000;
+    resources.MapMemory(base, page * 4);
+    resources.MapMemory(base + page * 2, page * 4);
+    Require("GpuMappedRangeLifecycle", "union",
+            resources.IsMapped(base, page * 6) &&
+                !resources.IsMapped(base, page * 7),
+            "overlapping maps did not form one interval union");
+
+    resources.UnmapMemory(base + page * 2, page * 2);
+    Require("GpuMappedRangeLifecycle", "subtract",
+            resources.IsMapped(base, page * 2) &&
+                resources.IsMapped(base + page * 4, page * 2) &&
+                !resources.IsMapped(base, page * 6),
+            "partial unmap did not punch the expected interval hole");
+
+    resources.UnmapMemory(base + page * 2, page * 2);
+    Require("GpuMappedRangeLifecycle", "idempotent unmap",
+            resources.IsMapped(base, page * 2) &&
+                resources.IsMapped(base + page * 4, page * 2),
+            "unmapping an absent interval changed neighboring mappings");
+
+    resources.UnmapMemory(base, page * 6);
+    Require("GpuMappedRangeLifecycle", "clear",
+            !resources.IsMapped(base, page * 6),
+            "full unmap did not clear the interval union");
+
+    constexpr uint64_t old_prt = base + page * 8;
+    constexpr uint64_t new_prt = base + page * 16;
+    resources.MapMemory(old_prt, page * 4);
+    resources.UnmapMemory(old_prt, page * 4);
+    resources.MapMemory(new_prt, page * 6);
+    Require("GpuMappedRangeLifecycle", "PRT replacement",
+            !resources.IsMapped(old_prt, page * 4) &&
+                resources.IsMapped(new_prt, page * 6),
+            "old-unmap/new-map did not replace full PRT coverage");
+
+    resources.SetGpu(nullptr);
+    scheduler.Finish();
+    std::printf("[host]    %-32s ok\n", "GpuMappedRangeLifecycle");
+  }
+
   void CheckStreamBufferRing() {
     EnsureRuntimeContext();
     CommandScheduler scheduler(Renderer(), m_runtime_context);
@@ -1526,7 +1579,7 @@ public:
             fault_memory == reinterpret_cast<void *>(fault_base),
             "fixed processor-fault allocation failed");
     auto &resources = context.GetGpuResources();
-    resources.MapMemory(fault_base, fault_size, GpuAccess::ReadWrite);
+    resources.MapMemory(fault_base, fault_size);
 
     constexpr uint64_t immediate_dst = fault_base + 0x1000;
     constexpr uint64_t immediate_memory_dst = fault_base + 0x2000;
@@ -1683,7 +1736,7 @@ public:
               resources.InvalidateMemory(fault_base, sizeof(uint32_t)),
               "processor memory invalidation did not find its mapped range");
     });
-    resources.UnmapMemory(fault_base, fault_size, GpuAccess::ReadWrite);
+    resources.UnmapMemory(fault_base, fault_size);
     Require("GpuCommandLane", "processor fault unmap",
             Libs::LibKernel::Memory::KernelMunmap(fault_base, fault_size) == 0,
             "processor-fault direct-memory mapping release failed");
@@ -1953,7 +2006,7 @@ public:
       GpuResourceManager resources(m_runtime_context, scheduler);
       resources.SetGpu(&gpu);
       auto &cache = resources.GetBufferCache();
-      resources.MapMemory(base, allocation_size, GpuAccess::ReadWrite);
+      resources.MapMemory(base, allocation_size);
 
       const auto MarkGpuWrite = [&](uint64_t address, uint64_t size) {
         auto allocation =
@@ -2376,7 +2429,7 @@ public:
                                  sizeof(reacquire_value));
 
       resources.SetGpu(nullptr);
-      resources.UnmapMemory(base, allocation_size, GpuAccess::ReadWrite);
+      resources.UnmapMemory(base, allocation_size);
       scheduler.Finish();
     }
     gpu.Shutdown();
@@ -2431,7 +2484,7 @@ public:
               narrow_download != nullptr && narrow_download_offset % 4 == 0 &&
                   wide_download != nullptr && wide_download_offset % 16 == 0,
               "wide/block image readback was not aligned to its texel block");
-      resources.MapMemory(base, allocation_size, GpuAccess::ReadWrite);
+      resources.MapMemory(base, allocation_size);
 
       ImageDesc sampled{};
       sampled.type = BindingType::Texture;
@@ -4834,7 +4887,7 @@ public:
       m_device.destroyShaderModule(ms_depth_module, nullptr);
 
       resources.SetGpu(nullptr);
-      resources.UnmapMemory(base, allocation_size, GpuAccess::ReadWrite);
+      resources.UnmapMemory(base, allocation_size);
       scheduler.Finish();
     }
     gpu.Shutdown();
@@ -4879,7 +4932,7 @@ public:
     scheduler.Begin(registers, user_config, shaders);
     {
       GpuResourceManager resources(m_runtime_context, scheduler);
-      resources.MapMemory(base, allocation_size, GpuAccess::ReadWrite);
+      resources.MapMemory(base, allocation_size);
       const uint32_t pitch = TileGetTexturePitch(format, 1, 1, tile);
       TileSizeAlign total{};
       TileSizeOffset mip{};
@@ -4978,7 +5031,7 @@ public:
                   std::vector<u32>{0x40004200u, 0x44003c00u},
               "tiled BGRA16 Buffer mirror changed guest component order");
       DestroyBuffer(&mirror_readback);
-      resources.UnmapMemory(base, allocation_size, GpuAccess::ReadWrite);
+      resources.UnmapMemory(base, allocation_size);
       scheduler.Finish();
     }
     Require(name, "unmap",
@@ -5025,7 +5078,7 @@ public:
       auto &resources = context.GetGpuResources();
       auto &texture_cache = resources.GetTextureCache();
       auto &executor = context.GetRenderExecutor();
-      resources.MapMemory(base, allocation_size, GpuAccess::ReadWrite);
+      resources.MapMemory(base, allocation_size);
 
       constexpr auto stencil_format =
           Prospero::GpuEnumValue(Prospero::BufferFormat::k8UInt);
@@ -6011,10 +6064,8 @@ public:
       const auto stale_ordered_color =
           texture_cache.FindImage(ordered_color_desc);
       RenderExecutorTestAccess::BindRenderTarget(executor, stale_ordered_color);
-      resources.UnmapMemory(ordered_color_address, target_mip_size,
+      resources.UnmapMemory(ordered_color_address, target_mip_size);
-                            GpuAccess::ReadWrite);
+      resources.MapMemory(ordered_color_address, target_mip_size);
-      resources.MapMemory(ordered_color_address, target_mip_size,
-                          GpuAccess::ReadWrite);
 
       auto ordered_depth_desc = depth;
       ordered_depth_desc.info.stencil = {ordered_color_address,
@@ -6167,7 +6218,7 @@ public:
               texture_cache.GetImage(depth_id).usage.storage,
           "storage stencil binding did not acquire the associated depth owner");
       RenderExecutorTestAccess::ResetBindings(executor);
-      resources.UnmapMemory(base, allocation_size, GpuAccess::ReadWrite);
+      resources.UnmapMemory(base, allocation_size);
       scheduler.Finish();
     }
 
@@ -17198,12 +17249,12 @@ void CheckStandard64RenderTargetTileRoundTrip() {
 void CheckStorageTextureGpuOwnedRebindState() {
   constexpr uintptr_t base = 0x0000000200200000ull;
   constexpr uint64_t size = 0x10000;
-  auto *memory = static_cast<uint8_t *>(
+  const auto guest_memory = Libs::LibKernel::Memory::AllocateRuntimeMemory(
-      VirtualAlloc(reinterpret_cast<void *>(base), size,
+      base, size, Common::VirtualMemory::Mode::ReadWrite,
-                   MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE));
+      "storage_texture_gpu_owned_rebind", true);
+  auto *memory = reinterpret_cast<uint8_t *>(guest_memory);
   Require("StorageTextureGpuOwnedRebind", "allocation",
-          memory == reinterpret_cast<void *>(base),
+          guest_memory == base, "fixed guest-owner allocation failed");
-          "fixed VirtualAlloc failed");
   PageManager page_manager(CacheFault, nullptr);
   MemoryTracker tracker(page_manager);
   page_manager.OnGpuMap(base, size);
@@ -17215,7 +17266,6 @@ void CheckStorageTextureGpuOwnedRebindState() {
   Require(
       "StorageTextureGpuOwnedRebind", "owned",
       tracker.IsRegionGpuModified(base, size) &&
-          page_manager.IsMapped(base, size) &&
           (!HostMemoryQueryReadable(base, size, readable) || readable < size) &&
           HostMemoryQueryRange(base, size, HostMemoryAccess::Mapped, mapped) &&
           mapped == size &&
@@ -17261,7 +17311,8 @@ void CheckStorageTextureGpuOwnedRebindState() {
   tracker.UntrackMemory(base, size);
   page_manager.OnGpuUnmap(base, size);
   Require("StorageTextureGpuOwnedRebind", "free",
-          VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
+          Libs::LibKernel::Memory::FreeGuestMemory(base, size),
+          "guest-owner free failed");
   std::printf("[host]    %-32s ok\n", "StorageTextureGpuOwnedRebind");
 }
 #endif
@@ -18029,6 +18080,11 @@ int main(int argc, char **argv) {
     vulkan.CheckSchedulerTimeline();
     return 0;
   }
+  if (argc == 2 && std::strcmp(argv[1], "--mapped-range-only") == 0) {
+    VulkanHarness vulkan;
+    vulkan.CheckGpuMappedRangeLifecycle();
+    return 0;
+  }
   if (argc == 2 && std::strcmp(argv[1], "--stream-buffer-only") == 0) {
     VulkanHarness vulkan;
     vulkan.CheckStreamBufferRing();
@@ -18184,6 +18240,7 @@ int main(int argc, char **argv) {
   CheckEmbeddedFetchLaneSpill();
   CheckPs5GameExampleImageClearRuntimeShape();
   vulkan.CheckSchedulerTimeline();
+  vulkan.CheckGpuMappedRangeLifecycle();
   vulkan.CheckStreamBufferRing();
   vulkan.CheckCommandPoolGrowth();
   vulkan.CheckGpuTilerCpuParity();

tests/VirtualMemoryAllocationTests.cpp

@@ -1,15 +1,21 @@
 #include "common/commonSubsystem.h"
 #include "common/emulatorConfig.h"
+#include "common/file.h"
 #include "common/logging/log.h"
 #include "common/subsystems.h"
 #include "common/threads.h"
+#include "common/virtualMemory.h"
 #include "kernel/memory.h"
+#include "kernel/pthread.h"
 #include "libs/errno.h"
+#include "loader/runtimeLinker.h"
+#include "loader/systemContent.h"
 
 #include <cinttypes>
 #include <cstdint>
 #include <cstdio>
 #include <cstring>
+#include <filesystem>
 #include <string>
 
 namespace {
@@ -18,11 +24,16 @@ using Libs::LibKernel::Memory::VirtualQueryInfo;
 
 // Prospero ABI?
 constexpr uint64_t SceKernelPageSize             = 0x4000;
+constexpr uint64_t SceKernelTotalPhysicalSize    = 13824ull * 1024ull * 1024ull;
+constexpr uint64_t TestFlexibleMemorySize        = 3072ull * 1024ull * 1024ull;
 constexpr int      SceKernelProtCpuRead          = 0x01;
 constexpr int      SceKernelProtCpuRw            = 0x02;
+constexpr int      SceKernelProtCpuExec          = 0x04;
 constexpr int      SceKernelMapFixed             = 0x10;
 constexpr int      SceKernelMapNoOverwrite       = 0x80;
+constexpr int      SceKernelMapDmemCompat        = 0x400;
 constexpr int      SceKernelMapNoCoalesce        = 0x400000;
+constexpr int      SceKernelMapAligned64Kb       = 16 << 24;
 constexpr int      SceKernelVqFindNext           = 1;
 constexpr int      SceKernelMtypeC               = 11;
 constexpr uint64_t SceKernelDirectMemoryStart    = 0;
@@ -94,6 +105,29 @@ void InitSubsystems() {
 	Config::Load(options);
 
 	slist->Add(log, {core, config});
+	Check("InitSubsystems", slist->InitAll(false), "failed to initialize logging subsystem");
+
+	const auto param_json =
+	    std::filesystem::temp_directory_path() /
+	    ("kyty_virtual_memory_" +
+	     std::to_string(reinterpret_cast<uintptr_t>(&initialized)) + ".json");
+	constexpr char json[] = R"({"kernel":{"flexibleMemorySize":3221225472}})";
+	Common::File  param_file;
+	Check("InitSubsystems", param_file.Create(param_json), "failed to create temporary param.json");
+	uint32_t bytes_written = 0;
+	param_file.Write(json, sizeof(json) - 1, &bytes_written);
+	param_file.Close();
+	Check("InitSubsystems", bytes_written == sizeof(json) - 1,
+	      "failed to write temporary param.json");
+
+	Loader::SystemContentLoadParamSfo(param_json);
+	const auto flexible_memory_size = Loader::SystemContentGetFlexibleMemorySize();
+	Check("InitSubsystems", Common::File::DeleteFile(param_json),
+	      "failed to remove temporary param.json");
+	Check("InitSubsystems", flexible_memory_size == TestFlexibleMemorySize,
+	      "failed to read flexible memory size from param.json");
+	Libs::LibKernel::Memory::SetFlexibleMemorySize(flexible_memory_size);
+
 	slist->Add(memory, {core, log, thread});
 	Check("InitSubsystems", slist->InitAll(false), "failed to initialize memory subsystem");
 
@@ -131,6 +165,13 @@ size_t AvailableFlexibleMemory(const char* test) {
 	return size;
 }
 
+size_t ConfiguredFlexibleMemory(const char* test) {
+	size_t size = 0;
+	CheckOk(test, Libs::LibKernel::Memory::KernelConfiguredFlexibleMemorySize(&size),
+	        "KernelConfiguredFlexibleMemorySize");
+	return size;
+}
+
 uint64_t MapNamedFlexible(const char* test, uint64_t size, int prot, const char* name) {
 	void*     addr = nullptr;
 	const int ret =
@@ -190,6 +231,387 @@ void TestProsperoArgumentAndInfoSizeContracts() {
 	std::printf("[host]    %-48s ok\n", test);
 }
 
+void TestGuestAddressSpaceOwnsReservationsBeforeBacking() {
+	const char* test = "GuestAddressSpaceOwnsReservationsBeforeBacking";
+	void*       addr = nullptr;
+
+	Check(test, Libs::LibKernel::Memory::TestGuestBackingOutsideAddressSpace(),
+	      "boot-time shared backing alias overlaps an owned guest interval");
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelReserveVirtualRange(&addr, SceKernelPageSize, 0,
+	                                                           SceKernelPageSize),
+	        "KernelReserveVirtualRange");
+	const auto base = reinterpret_cast<uint64_t>(addr);
+	Check(test, Libs::LibKernel::Memory::TestGuestAddressRangeIsOwned(base, SceKernelPageSize),
+	      "guest reservation was allocated outside the early owner");
+	Check(test, Libs::LibKernel::Memory::TestPlaceholderRangeIsFree(base, SceKernelPageSize),
+	      "semantic reservation replaced the owner's placeholder");
+	Check(test,
+	      Libs::LibKernel::Memory::ProtectGuestHostMemory(
+	          base, SceKernelPageSize, Common::VirtualMemory::Mode::NoAccess),
+	      "owner rejected a sparse placeholder protection no-op");
+	CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(base, SceKernelPageSize), "KernelMunmap");
+	Check(test, Libs::LibKernel::Memory::TestPlaceholderRangeIsFree(base, SceKernelPageSize),
+	      "released semantic reservation escaped owner control");
+
+	std::printf("[host]    %-48s ok\n", test);
+}
+
+void TestGuestAddressSpaceHasNoFixedFallback() {
+	const char* test            = "GuestAddressSpaceHasNoFixedFallback";
+	const auto  unowned_address = reinterpret_cast<void*>(0x10000);
+	void*       addr            = unowned_address;
+
+	CheckFailed(test,
+	            Libs::LibKernel::Memory::KernelReserveVirtualRange(
+	                &addr, SceKernelPageSize, SceKernelMapFixed | SceKernelMapNoOverwrite,
+	                SceKernelPageSize),
+	            "KernelReserveVirtualRange(unowned fixed address)");
+	Check(test, reinterpret_cast<uint64_t>(addr) == 0x10000,
+	      "failed fixed reservation unexpectedly moved");
+
+	addr = unowned_address;
+	CheckFailed(test,
+	            Libs::LibKernel::Memory::KernelMapNamedFlexibleMemory(
+	                &addr, SceKernelPageSize, SceKernelProtCpuRw,
+	                SceKernelMapFixed | SceKernelMapNoOverwrite, "unowned_flexible"),
+	            "KernelMapNamedFlexibleMemory(unowned fixed address)");
+
+	int64_t phys_addr = -1;
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelAllocateDirectMemory(
+	            0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(), SceKernelPageSize,
+	            SceKernelPageSize, SceKernelMtypeC, &phys_addr),
+	        "KernelAllocateDirectMemory");
+	addr = unowned_address;
+	CheckFailed(test,
+	            Libs::LibKernel::Memory::KernelMapNamedDirectMemory(
+	                &addr, SceKernelPageSize, SceKernelProtCpuRw,
+	                SceKernelMapFixed | SceKernelMapNoOverwrite, phys_addr, SceKernelPageSize,
+	                "unowned_direct"),
+	            "KernelMapNamedDirectMemory(unowned fixed address)");
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelCheckedReleaseDirectMemory(phys_addr, SceKernelPageSize),
+	        "KernelCheckedReleaseDirectMemory");
+
+	std::printf("[host]    %-48s ok\n", test);
+}
+
+void TestGuestFreeRangeSearchDoesNotUnderflow() {
+	const char* test = "GuestFreeRangeSearchDoesNotUnderflow";
+
+	Check(test, Libs::LibKernel::Memory::TestGuestFreeRangeBounds(),
+	      "free-range containment accepted a candidate beyond the range end");
+
+	std::printf("[host]    %-48s ok\n", test);
+}
+
+void TestFlexibleMemoryCapacityIsBootFixed() {
+	const char* test       = "FlexibleMemoryCapacityIsBootFixed";
+	const auto  configured = ConfiguredFlexibleMemory(test);
+	const auto  baseline   = AvailableFlexibleMemory(test);
+	const auto  backing    = Libs::LibKernel::Memory::TestGuestBackingSize();
+
+	Check(test, configured == TestFlexibleMemorySize,
+	      "boot flexible pool did not use the param.json value");
+	Check(test, configured == baseline, "boot flexible pool did not start at configured capacity");
+	Check(test, backing == SceKernelTotalPhysicalSize,
+	      "boot backing is not the single 13.5 GiB physical file");
+	Check(test, backing == Libs::LibKernel::Memory::KernelGetDirectMemorySize() + configured,
+	      "direct and flexible regions do not partition the boot backing");
+
+	const auto address =
+	    MapNamedFlexible(test, SceKernelPageSize, SceKernelProtCpuRw, "boot_fixed_flexible");
+	Check(test, ConfiguredFlexibleMemory(test) == configured,
+	      "configured flexible capacity changed after allocation");
+	Check(test, Libs::LibKernel::Memory::TestGuestBackingSize() == backing,
+	      "shared backing size changed after allocation");
+	Check(test, AvailableFlexibleMemory(test) == baseline - SceKernelPageSize,
+	      "flexible allocation did not consume the boot-time pool");
+
+	CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(address, SceKernelPageSize),
+	        "KernelMunmap");
+	Check(test, ConfiguredFlexibleMemory(test) == configured,
+	      "configured flexible capacity changed after release");
+	Check(test, AvailableFlexibleMemory(test) == baseline,
+	      "flexible release did not restore the boot-time pool");
+
+	std::printf("[host]    %-48s ok\n", test);
+}
+
+void TestFlexibleMemoryUsesSharedBacking() {
+	const char* test     = "FlexibleMemoryUsesSharedBacking";
+	const auto  baseline = AvailableFlexibleMemory(test);
+	void*       address  = nullptr;
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelMapNamedFlexibleMemory(
+	            &address, SceKernelPageSize * 2, SceKernelProtCpuRw, 0, "shared_flexible"),
+	        "KernelMapNamedFlexibleMemory");
+	const auto base = reinterpret_cast<uint64_t>(address);
+	Check(test, Libs::LibKernel::Memory::TestGuestAddressRangeIsOwned(base, SceKernelPageSize * 2),
+	      "flexible mapping escaped the guest owner");
+
+	constexpr uint64_t first_value     = 0x464c45584241434bull; // "FLEXBACK"
+	constexpr uint64_t second_value    = 0x534841524544464cull; // "SHAREDFL"
+	*reinterpret_cast<uint64_t*>(base) = first_value;
+	uint64_t value                     = 0;
+	Check(test, Libs::LibKernel::Memory::TryReadBacking(base, &value, sizeof(value)),
+	      "TryReadBacking did not resolve flexible memory");
+	Check(test, value == first_value, "backing did not observe a flexible-memory CPU write");
+	Check(test,
+	      Libs::LibKernel::Memory::TryWriteBacking(base + SceKernelPageSize, &second_value,
+	                                               sizeof(second_value)),
+	      "TryWriteBacking did not resolve flexible memory");
+	Check(test, *reinterpret_cast<uint64_t*>(base + SceKernelPageSize) == second_value,
+	      "flexible-memory view did not observe a backing write");
+
+	CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(base, SceKernelPageSize * 2),
+	        "KernelMunmap");
+	Check(test, AvailableFlexibleMemory(test) == baseline,
+	      "flexible backing offsets were not returned to the boot-time pool");
+	Check(test, !Libs::LibKernel::Memory::TryReadBacking(base, &value, sizeof(value)),
+	      "unmapped flexible memory remained registered in the backing owner");
+
+	std::printf("[host]    %-48s ok\n", test);
+}
+
+void TestFlexibleDmemCompatAndAlignmentFlags() {
+	const char* test     = "FlexibleDmemCompatAndAlignmentFlags";
+	const auto  baseline = AvailableFlexibleMemory(test);
+	void*       address  = nullptr;
+
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelMapNamedFlexibleMemory(
+	            &address, SceKernelPageSize, SceKernelProtCpuRw,
+	            SceKernelMapDmemCompat | SceKernelMapAligned64Kb, "dmem_compat"),
+	        "KernelMapNamedFlexibleMemory(DMEM_COMPAT|ALIGNED_64KB)");
+	const auto base = reinterpret_cast<uint64_t>(address);
+	Check(test, (base & (0x10000 - 1u)) == 0, "SDK alignment flag was not honored");
+	const auto info = Query(test, base);
+	Check(test, info.is_flexible == 1 && info.is_stack == 0,
+	      "SCE_KERNEL_MAP_DMEM_COMPAT was misclassified as MAP_STACK");
+	Check(test, AvailableFlexibleMemory(test) + SceKernelPageSize == baseline,
+	      "DMEM_COMPAT mapping did not consume boot-time flexible backing");
+
+	void* stack_start = reinterpret_cast<void*>(UINT64_MAX);
+	void* stack_end   = reinterpret_cast<void*>(UINT64_MAX);
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelIsStack(reinterpret_cast<void*>(base), &stack_start,
+	                                               &stack_end),
+	        "KernelIsStack");
+	Check(test, stack_start == nullptr && stack_end == nullptr,
+	      "DMEM_COMPAT flexible mapping was reported as a stack");
+
+	CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(base, SceKernelPageSize),
+	        "KernelMunmap");
+	Check(test, AvailableFlexibleMemory(test) == baseline,
+	      "DMEM_COMPAT cleanup did not restore flexible capacity");
+
+	void* opaque = nullptr;
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelMapNamedFlexibleMemory(
+	            &opaque, SceKernelPageSize, SceKernelProtCpuRw, 0x8000, "opaque_runtime_flag"),
+	        "KernelMapNamedFlexibleMemory(opaque runtime flag)");
+	Check(test, AvailableFlexibleMemory(test) + SceKernelPageSize == baseline,
+	      "opaque runtime flag mapping did not consume boot-time flexible backing");
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelMunmap(reinterpret_cast<uint64_t>(opaque),
+	                                              SceKernelPageSize),
+	        "KernelMunmap(opaque runtime flag)");
+	Check(test, AvailableFlexibleMemory(test) == baseline,
+	      "opaque runtime flag cleanup did not restore flexible capacity");
+
+	void* invalid_flag = nullptr;
+	CheckFailed(
+	    test,
+	    Libs::LibKernel::Memory::KernelMapNamedFlexibleMemory(
+	        &invalid_flag, SceKernelPageSize, SceKernelProtCpuRw, 0x10000, "unsupported_flag"),
+	    "KernelMapNamedFlexibleMemory(unsupported flag)");
+
+	void* invalid_alignment = nullptr;
+	CheckFailed(test,
+	            Libs::LibKernel::Memory::KernelMapNamedFlexibleMemory(
+	                &invalid_alignment, SceKernelPageSize, SceKernelProtCpuRw, 13 << 24,
+	                "invalid_alignment"),
+	            "KernelMapNamedFlexibleMemory(invalid alignment)");
+
+	std::printf("[host]    %-48s ok\n", test);
+}
+
+void TestFlexibleNoCoalescePreservesBoundaries() {
+	const char* test     = "FlexibleNoCoalescePreservesBoundaries";
+	const auto  baseline = AvailableFlexibleMemory(test);
+	void*       reserve  = nullptr;
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelReserveVirtualRange(
+	            &reserve, SceKernelPageSize * 2, 0, SceKernelPageSize),
+	        "KernelReserveVirtualRange");
+	const auto base = reinterpret_cast<uint64_t>(reserve);
+
+	void* left = reinterpret_cast<void*>(base);
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelMapNamedFlexibleMemory(
+	            &left, SceKernelPageSize, SceKernelProtCpuRw,
+	            SceKernelMapFixed | SceKernelMapNoCoalesce, "no_coalesce"),
+	        "KernelMapNamedFlexibleMemory(left)");
+	void* right = reinterpret_cast<void*>(base + SceKernelPageSize);
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelMapNamedFlexibleMemory(
+	            &right, SceKernelPageSize, SceKernelProtCpuRw,
+	            SceKernelMapFixed | SceKernelMapNoCoalesce, "no_coalesce"),
+	        "KernelMapNamedFlexibleMemory(right)");
+
+	ExpectRange(test, Query(test, base), base, base + SceKernelPageSize, SceKernelProtCpuRw, 1, 0,
+	            0, 1, "no_coalesce");
+	ExpectRange(test, Query(test, base + SceKernelPageSize), base + SceKernelPageSize,
+	            base + SceKernelPageSize * 2, SceKernelProtCpuRw, 1, 0, 0, 1, "no_coalesce");
+
+	CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(base, SceKernelPageSize * 2),
+	        "KernelMunmap");
+	Check(test, AvailableFlexibleMemory(test) == baseline,
+	      "NO_COALESCE cleanup did not restore flexible capacity");
+
+	std::printf("[host]    %-48s ok\n", test);
+}
+
+void TestFlexibleMemoryReuseIsZeroFilled() {
+	const char* test     = "FlexibleMemoryReuseIsZeroFilled";
+	const auto  baseline = AvailableFlexibleMemory(test);
+	const auto  first =
+	    MapNamedFlexible(test, SceKernelPageSize, SceKernelProtCpuRw, "flexible_zero_source");
+	std::memset(reinterpret_cast<void*>(first), 0xa5, SceKernelPageSize);
+	CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(first, SceKernelPageSize),
+	        "KernelMunmap(source)");
+
+	const auto reused =
+	    MapNamedFlexible(test, SceKernelPageSize, SceKernelProtCpuRw, "flexible_zero_reuse");
+	const auto* bytes = reinterpret_cast<const uint8_t*>(reused);
+	Check(test,
+	      std::all_of(bytes, bytes + SceKernelPageSize, [](uint8_t value) { return value == 0; }),
+	      "reused flexible backing exposed stale bytes");
+	CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(reused, SceKernelPageSize),
+	        "KernelMunmap(reuse)");
+	Check(test, AvailableFlexibleMemory(test) == baseline,
+	      "zero-fill test leaked flexible backing capacity");
+
+	std::printf("[host]    %-48s ok\n", test);
+}
+
+void TestGuestStackUsesPrivateOwnerMemoryAndCache() {
+	const char* test     = "GuestStackUsesPrivateOwnerMemoryAndCache";
+	const auto  baseline = AvailableFlexibleMemory(test);
+	uint64_t    first    = 0;
+	uint64_t    second   = 0;
+	uint64_t    map_size = 0;
+
+	Check(test, Libs::LibKernel::TestGuestStackOwnerLifecycle(&first, &second, &map_size),
+	      "guest stack owner lifecycle failed");
+	Check(test, first != 0 && first == second, "guest stack cache did not reuse its owner mapping");
+	Check(test, map_size != 0 && (map_size & (SceKernelPageSize - 1u)) == 0,
+	      "guest stack mapping is not 16 KiB aligned");
+	Check(test, AvailableFlexibleMemory(test) == baseline,
+	      "private guest stack changed flexible backing capacity");
+
+	std::printf("[host]    %-48s ok\n", test);
+}
+
+void TestMainEntryUsesGuestStackAndDisablesHostChecks() {
+	const char* test = "MainEntryUsesGuestStackAndDisablesHostChecks";
+
+	Check(test, Loader::TestMainEntryUsesGuestStack(),
+	      "main-entry stack switch did not preserve the guest/host stack invariants");
+
+	std::printf("[host]    %-48s ok\n", test);
+}
+
+void TestFragmentedBackingUnmapRollback() {
+	const char* test     = "FragmentedBackingUnmapRollback";
+	const auto  baseline = AvailableFlexibleMemory(test);
+	const auto  left =
+	    MapNamedFlexible(test, SceKernelPageSize, SceKernelProtCpuRw, "backing_hole_left");
+	const auto blocker =
+	    MapNamedFlexible(test, SceKernelPageSize, SceKernelProtCpuRw, "backing_blocker");
+	const auto right =
+	    MapNamedFlexible(test, SceKernelPageSize, SceKernelProtCpuRw, "backing_hole_right");
+	CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(left, SceKernelPageSize),
+	        "KernelMunmap(left hole)");
+	CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(right, SceKernelPageSize),
+	        "KernelMunmap(right hole)");
+
+	const auto fragmented =
+	    MapNamedFlexible(test, SceKernelPageSize * 2, SceKernelProtCpuRw, "fragmented_backing");
+	auto* first_word = reinterpret_cast<uint64_t*>(fragmented);
+	auto* last_word =
+	    reinterpret_cast<uint64_t*>(fragmented + SceKernelPageSize * 2 - sizeof(uint64_t));
+	*first_word = 0x465241474c454654ull; // "FRAGLEFT"
+	*last_word  = 0x4652414752474854ull; // "FRAGRGHT"
+
+	Libs::LibKernel::Memory::TestFailGuestBackingStoreUnmapAfter(1);
+	CheckFailed(test, Libs::LibKernel::Memory::KernelMunmap(fragmented, SceKernelPageSize * 2),
+	            "KernelMunmap(injected second-view failure)");
+	ExpectRange(test, Query(test, fragmented), fragmented, fragmented + SceKernelPageSize * 2,
+	            SceKernelProtCpuRw, 1, 0, 0, 1, "fragmented_backing");
+	Check(test, *first_word == 0x465241474c454654ull && *last_word == 0x4652414752474854ull,
+	      "transactional backing-unmap rollback lost mapped contents");
+
+	CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(fragmented, SceKernelPageSize * 2),
+	        "KernelMunmap(retry)");
+	CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(blocker, SceKernelPageSize),
+	        "KernelMunmap(blocker)");
+	Check(test, AvailableFlexibleMemory(test) == baseline,
+	      "fragmented backing rollback test leaked flexible capacity");
+
+	std::printf("[host]    %-48s ok\n", test);
+}
+
+void TestRuntimeMemoryOwnerLifecycle() {
+	const char* test = "RuntimeMemoryOwnerLifecycle";
+	Check(test,
+	      Libs::LibKernel::Memory::AllocateRuntimeMemory(0x10000, SceKernelPageSize,
+	                                                     Common::VirtualMemory::Mode::ReadWrite,
+	                                                     "runtime_outside_owner", true) == 0,
+	      "fixed runtime allocation escaped the guest owner");
+
+	const auto base = Libs::LibKernel::Memory::AllocateRuntimeMemory(
+	    0, SceKernelPageSize * 2, Common::VirtualMemory::Mode::ReadWrite, "runtime_lifecycle");
+	Check(test, base != 0, "runtime allocation failed");
+	Check(test, Libs::LibKernel::Memory::TestGuestAddressRangeIsOwned(base, SceKernelPageSize * 2),
+	      "runtime allocation is outside the owner");
+	*reinterpret_cast<uint64_t*>(base) = 0x52554e54494d454full; // "RUNTIMEO"
+	Check(test,
+	      Libs::LibKernel::Memory::ProtectGuestMemory(base, SceKernelPageSize,
+	                                                  Common::VirtualMemory::Mode::Read),
+	      "runtime protection failed");
+	Check(test, Libs::LibKernel::Memory::FreeGuestMemory(base, SceKernelPageSize * 2),
+	      "runtime free failed");
+	Check(test, Libs::LibKernel::Memory::TestPlaceholderRangeIsFree(base, SceKernelPageSize * 2),
+	      "runtime free did not restore the owner placeholder");
+
+	const auto reused = Libs::LibKernel::Memory::AllocateRuntimeMemory(
+	    base, SceKernelPageSize * 2, Common::VirtualMemory::Mode::ReadWrite, "runtime_reuse", true);
+	Check(test, reused == base, "fixed runtime allocation did not reuse the owner placeholder");
+	Check(test, Libs::LibKernel::Memory::FreeGuestMemory(reused, SceKernelPageSize * 2),
+	      "reused runtime free failed");
+
+	const auto adjacent_first = Libs::LibKernel::Memory::AllocateRuntimeMemory(
+	    0, SceKernelPageSize, Common::VirtualMemory::Mode::ReadWrite, "runtime_adjacent_first");
+	Check(test, adjacent_first != 0, "first adjacent runtime allocation failed");
+	const auto adjacent_second = Libs::LibKernel::Memory::AllocateRuntimeMemory(
+	    adjacent_first + SceKernelPageSize, SceKernelPageSize,
+	    Common::VirtualMemory::Mode::ReadWrite, "runtime_adjacent_second", true);
+	Check(test, adjacent_second == adjacent_first + SceKernelPageSize,
+	      "second adjacent runtime allocation failed");
+	Check(test,
+	      Libs::LibKernel::Memory::FreeGuestMemory(adjacent_first, SceKernelPageSize * 2),
+	      "combined adjacent runtime free failed");
+	Check(test,
+	      Libs::LibKernel::Memory::TestPlaceholderRangeIsFree(adjacent_first,
+	                                                          SceKernelPageSize * 2),
+	      "combined adjacent runtime free did not restore one owner placeholder");
+
+	std::printf("[host]    %-48s ok\n", test);
+}
+
 void TestFlexibleMapQueryAndWholeMunmap() {
 	const char* test     = "FlexibleMapQueryAndWholeMunmap";
 	const auto  baseline = AvailableFlexibleMemory(test);
@@ -341,9 +763,11 @@ void TestDirectMapQueryOffsetAndPartialMunmap() {
 	            &addr, SceKernelPageSize * 4, SceKernelProtCpuRw, 0, phys_addr, SceKernelPageSize,
 	            "prospero_direct"),
 	        "KernelMapNamedDirectMemory");
-	const auto base  = reinterpret_cast<uint64_t>(addr);
+	const auto base = reinterpret_cast<uint64_t>(addr);
-	const auto phys  = static_cast<uint64_t>(phys_addr);
+	const auto phys = static_cast<uint64_t>(phys_addr);
-	void*      alias = nullptr;
+	Check(test, Libs::LibKernel::Memory::TestGuestAddressRangeIsOwned(base, SceKernelPageSize * 4),
+	      "direct mapping escaped the guest owner");
+	void* alias = nullptr;
 	CheckOk(test,
 	        Libs::LibKernel::Memory::KernelMapNamedDirectMemory(
 	            &alias, SceKernelPageSize * 4, SceKernelProtCpuRw, 0, phys_addr, SceKernelPageSize,
@@ -430,6 +854,192 @@ void TestDirectMapQueryOffsetAndPartialMunmap() {
 	std::printf("[host]    %-48s ok\n", test);
 }
 
+void TestDirectPartialProtectUnmapPreservesNeighbors() {
+	const char* test      = "DirectPartialProtectUnmapPreservesNeighbors";
+	const auto  size      = SceKernelPageSize * 3;
+	int64_t     phys_addr = 0;
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelAllocateDirectMemory(
+	            0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(), size, SceKernelPageSize,
+	            SceKernelMtypeC, &phys_addr),
+	        "KernelAllocateDirectMemory");
+
+	void* address = nullptr;
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelMapNamedDirectMemory(&address, size, SceKernelProtCpuRw,
+	                                                            0, phys_addr, SceKernelPageSize,
+	                                                            "partial_protect_direct"),
+	        "KernelMapNamedDirectMemory");
+	const auto base = reinterpret_cast<uint64_t>(address);
+	CheckOk(
+	    test,
+	    Libs::LibKernel::Memory::KernelMprotect(reinterpret_cast<void*>(base + SceKernelPageSize),
+	                                            SceKernelPageSize, SceKernelProtCpuRead),
+	    "KernelMprotect(middle)");
+	Check(test,
+	      Libs::LibKernel::Memory::ProtectGuestHostMemory(
+	          base, size, Common::VirtualMemory::Mode::Read),
+	      "owner could not protect fragmented backing views");
+	Check(test,
+	      Libs::LibKernel::Memory::ProtectGuestHostMemory(
+	          base, size, Common::VirtualMemory::Mode::ReadWrite),
+	      "owner could not restore fragmented backing views");
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelMunmap(base + SceKernelPageSize, SceKernelPageSize),
+	        "KernelMunmap(middle)");
+
+	Common::VirtualMemory::Mode old_left {};
+	Common::VirtualMemory::Mode old_right {};
+	Check(test,
+	      Common::VirtualMemory::Protect(base, SceKernelPageSize,
+	                                     Common::VirtualMemory::Mode::ReadWrite, &old_left),
+	      "could not inspect left-page protection");
+	Check(test,
+	      Common::VirtualMemory::Protect(base + SceKernelPageSize * 2, SceKernelPageSize,
+	                                     Common::VirtualMemory::Mode::ReadWrite, &old_right),
+	      "could not inspect right-page protection");
+	Check(test, old_left == Common::VirtualMemory::Mode::ReadWrite,
+	      "partial unmap changed the left neighbor protection");
+	Check(test, old_right == Common::VirtualMemory::Mode::ReadWrite,
+	      "partial unmap changed the right neighbor protection");
+	*reinterpret_cast<uint64_t*>(base) = 0x4c45465450524f54ull; // "LEFTPROT"
+	*reinterpret_cast<uint64_t*>(base + SceKernelPageSize * 2) =
+	    0x5247485450524f54ull; // "RGHTPROT"
+
+	CheckOk(test, Libs::LibKernel::Memory::KernelReleaseDirectMemory(phys_addr, size),
+	        "KernelReleaseDirectMemory");
+	ExpectUnmapped(test, base);
+	ExpectUnmapped(test, base + SceKernelPageSize * 2);
+
+	std::printf("[host]    %-48s ok\n", test);
+}
+
+void TestDirectMapValidationBeforeOwnerMutation() {
+	const char* test    = "DirectMapValidationBeforeOwnerMutation";
+	int64_t     invalid = -1;
+	CheckFailed(test,
+	            Libs::LibKernel::Memory::KernelAllocateDirectMemory(
+	                0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(), SceKernelPageSize + 1,
+	                SceKernelPageSize, SceKernelMtypeC, &invalid),
+	            "KernelAllocateDirectMemory(unaligned size)");
+	Check(test, invalid == -1, "invalid direct allocation changed the output address");
+	CheckFailed(test,
+	            Libs::LibKernel::Memory::KernelAllocateDirectMemory(
+	                0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(), SceKernelPageSize,
+	                0x1000, SceKernelMtypeC, &invalid),
+	            "KernelAllocateDirectMemory(sub-page alignment)");
+	Check(test, invalid == -1, "invalid alignment changed the output address");
+
+	int64_t phys_addr = 0;
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelAllocateDirectMemory(
+	            0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(), SceKernelPageSize * 2,
+	            SceKernelPageSize, SceKernelMtypeC, &phys_addr),
+	        "KernelAllocateDirectMemory");
+
+	auto expect_invalid = [&](size_t len, int prot, int flags, int64_t phys, size_t alignment,
+	                          const char* action) {
+		void* address = nullptr;
+		CheckFailed(test,
+		            Libs::LibKernel::Memory::KernelMapDirectMemory(&address, len, prot, flags, phys,
+		                                                           alignment),
+		            action);
+		Check(test, address == nullptr, "invalid direct map changed the output address");
+	};
+	expect_invalid(SceKernelPageSize + 1, SceKernelProtCpuRw, 0, phys_addr, SceKernelPageSize,
+	               "KernelMapDirectMemory(unaligned size)");
+	expect_invalid(SceKernelPageSize, SceKernelProtCpuRw, 0, phys_addr + 1, SceKernelPageSize,
+	               "KernelMapDirectMemory(unaligned physical address)");
+	expect_invalid(SceKernelPageSize, SceKernelProtCpuExec, 0, phys_addr, SceKernelPageSize,
+	               "KernelMapDirectMemory(executable)");
+
+	void* aligned = nullptr;
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelMapDirectMemory(
+	            &aligned, SceKernelPageSize, SceKernelProtCpuRw, 0, phys_addr, 0xc000),
+	        "KernelMapDirectMemory(16K-multiple alignment)");
+	Check(test, reinterpret_cast<uint64_t>(aligned) % 0xc000 == 0,
+	      "non-power-of-two 16K alignment was not honored");
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelMunmap(reinterpret_cast<uint64_t>(aligned),
+	                                              SceKernelPageSize),
+	        "KernelMunmap(16K-multiple alignment)");
+
+	void* ignored_flag = nullptr;
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelMapDirectMemory(&ignored_flag, SceKernelPageSize,
+	                                                       SceKernelProtCpuRw, 0x08, phys_addr,
+	                                                       SceKernelPageSize),
+	        "KernelMapDirectMemory(ignored flag)");
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelMunmap(reinterpret_cast<uint64_t>(ignored_flag),
+	                                              SceKernelPageSize),
+	        "KernelMunmap(ignored flag)");
+
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelReleaseDirectMemory(phys_addr, SceKernelPageSize * 2),
+	        "KernelReleaseDirectMemory");
+	std::printf("[host]    %-48s ok\n", test);
+}
+
+void TestDirectReleaseRollbackRestoresOwnerMapping() {
+	const char* test      = "DirectReleaseRollbackRestoresOwnerMapping";
+	int64_t     phys_addr = 0;
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelAllocateDirectMemory(
+	            0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(), SceKernelPageSize,
+	            SceKernelPageSize, SceKernelMtypeC, &phys_addr),
+	        "KernelAllocateDirectMemory");
+	void* address = nullptr;
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelMapNamedDirectMemory(
+	            &address, SceKernelPageSize, SceKernelProtCpuRw, 0, phys_addr, SceKernelPageSize,
+	            "release_rollback"),
+	        "KernelMapNamedDirectMemory");
+	const auto base                    = reinterpret_cast<uint64_t>(address);
+	*reinterpret_cast<uint64_t*>(base) = 0x52454c524f4c4c42ull; // "RELROLLB"
+
+	Libs::LibKernel::Memory::TestFailNextPhysicalMemoryUnmap();
+	CheckFailed(
+	    test,
+	    Libs::LibKernel::Memory::KernelCheckedReleaseDirectMemory(phys_addr, SceKernelPageSize),
+	    "KernelCheckedReleaseDirectMemory(injected failure)");
+	ExpectRange(test, Query(test, base), base, base + SceKernelPageSize, SceKernelProtCpuRw, 0, 1,
+	            0, 1, "release_rollback", static_cast<uint64_t>(phys_addr));
+	Check(test, *reinterpret_cast<uint64_t*>(base) == 0x52454c524f4c4c42ull,
+	      "release rollback lost the shared-backing contents");
+
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelCheckedReleaseDirectMemory(phys_addr, SceKernelPageSize),
+	        "KernelCheckedReleaseDirectMemory(retry)");
+	ExpectUnmapped(test, base);
+	std::printf("[host]    %-48s ok\n", test);
+}
+
+void TestDirectReleaseContracts() {
+	const char* test = "DirectReleaseContracts";
+	CheckOk(test, Libs::LibKernel::Memory::KernelReleaseDirectMemory(0, 0),
+	        "KernelReleaseDirectMemory(zero length)");
+	CheckOk(test, Libs::LibKernel::Memory::KernelCheckedReleaseDirectMemory(0, 0),
+	        "KernelCheckedReleaseDirectMemory(zero length)");
+	CheckFailed(test, Libs::LibKernel::Memory::KernelReleaseDirectMemory(1, SceKernelPageSize),
+	            "KernelReleaseDirectMemory(unaligned start)");
+	CheckFailed(test, Libs::LibKernel::Memory::KernelReleaseDirectMemory(0, SceKernelPageSize + 1),
+	            "KernelReleaseDirectMemory(unaligned size)");
+
+	const auto free_offset = static_cast<int64_t>(
+	    Libs::LibKernel::Memory::KernelGetDirectMemorySize() - SceKernelPageSize);
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelReleaseDirectMemory(free_offset, SceKernelPageSize),
+	        "KernelReleaseDirectMemory(unallocated range)");
+	Check(test,
+	      Libs::LibKernel::Memory::KernelCheckedReleaseDirectMemory(
+	          free_offset, SceKernelPageSize) == Libs::LibKernel::KERNEL_ERROR_ENOENT,
+	      "checked release did not report an unallocated range");
+
+	std::printf("[host]    %-48s ok\n", test);
+}
+
 void TestReleasedReserveCanBeReused() {
 	const char* test = "ReleasedReserveCanBeReused";
 	void*       addr = nullptr;
@@ -478,21 +1088,23 @@ void TestMunmapAcrossAdjacentFlexibleMappings() {
 	        "KernelMapNamedFlexibleMemory(right)");
 
 	Check(test,
-	      Libs::LibKernel::Memory::ClampRangeSize(base + SceKernelPageSize - 0x100, 0x200) ==
+	      Libs::LibKernel::Memory::ClampRangeSize(base + SceKernelPageSize - 0x100, 0x200) == 0x200,
-	          0x200,
 	      "ClampRangeSize did not cross adjacent committed mappings");
+	Check(test,
+	      Libs::LibKernel::Memory::ProtectGuestHostMemory(
+	          base, SceKernelPageSize * 2, Common::VirtualMemory::Mode::Read),
+	      "owner could not protect adjacent backing mappings");
+	Check(test,
+	      Libs::LibKernel::Memory::ProtectGuestHostMemory(
+	          base, SceKernelPageSize * 2, Common::VirtualMemory::Mode::ReadWrite),
+	      "owner could not restore adjacent backing mappings");
 
 	CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(base, SceKernelPageSize * 2),
 	        "KernelMunmap(adjacent mappings)");
 	Check(test, AvailableFlexibleMemory(test) == baseline,
 	      "multi-range unmap leaked flexible-memory budget");
-	ExpectRange(test, Query(test, base), base, base + SceKernelPageSize, 0, 0, 0, 0, 0,
+	ExpectUnmapped(test, base);
-	            "adjacent_left");
+	ExpectUnmapped(test, base + SceKernelPageSize);
-	ExpectRange(test, Query(test, base + SceKernelPageSize), base + SceKernelPageSize,
-	            base + SceKernelPageSize * 2, 0, 0, 0, 0, 0, "adjacent_right");
-
-	CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(base, SceKernelPageSize * 2),
-	        "KernelMunmap(restored reserve)");
 
 	std::printf("[host]    %-48s ok\n", test);
 }
@@ -548,6 +1160,52 @@ void TestNonzeroDirectOffsetAliasesSharedBacking() {
 	std::printf("[host]    %-48s ok\n", test);
 }
 
+void TestDirectMapAcrossContiguousAllocations() {
+	const char* test   = "DirectMapAcrossContiguousAllocations";
+	const auto  end    = Libs::LibKernel::Memory::KernelGetDirectMemorySize();
+	int64_t     first  = 0;
+	int64_t     second = 0;
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelAllocateDirectMemory(
+	            0, end, SceKernelPageSize, SceKernelPageSize, SceKernelMtypeC, &first),
+	        "KernelAllocateDirectMemory(first)");
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelAllocateDirectMemory(
+	            0, end, SceKernelPageSize, SceKernelPageSize, SceKernelMtypeC, &second),
+	        "KernelAllocateDirectMemory(second)");
+	Check(test, second == first + static_cast<int64_t>(SceKernelPageSize),
+	      "test allocations are not physically contiguous");
+
+	void* mapping = nullptr;
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelMapNamedDirectMemory(
+	            &mapping, SceKernelPageSize * 2, SceKernelProtCpuRw, 0, first, SceKernelPageSize,
+	            "contiguous_allocations"),
+	        "KernelMapNamedDirectMemory");
+	auto* words = reinterpret_cast<uint64_t*>(mapping);
+	words[0]    = 0x434f4e5449474c46ull; // "CONTIGLF"
+	*reinterpret_cast<uint64_t*>(reinterpret_cast<uint64_t>(mapping) + SceKernelPageSize) =
+	    0x434f4e5449475254ull; // "CONTIGRT"
+
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelCheckedReleaseDirectMemory(first, SceKernelPageSize * 2),
+	        "KernelCheckedReleaseDirectMemory(contiguous span)");
+	ExpectUnmapped(test, reinterpret_cast<uint64_t>(mapping));
+
+	int64_t reclaimed = -1;
+	CheckOk(test,
+	        Libs::LibKernel::Memory::KernelAllocateDirectMemory(
+	            0, end, SceKernelPageSize * 2, SceKernelPageSize, SceKernelMtypeC, &reclaimed),
+	        "KernelAllocateDirectMemory(reclaimed)");
+	Check(test, reclaimed == first, "released contiguous span was not coalesced");
+	CheckOk(
+	    test,
+	    Libs::LibKernel::Memory::KernelCheckedReleaseDirectMemory(reclaimed, SceKernelPageSize * 2),
+	    "KernelCheckedReleaseDirectMemory(reclaimed)");
+
+	std::printf("[host]    %-48s ok\n", test);
+}
+
 void TestDirectPhysicalFreeRangeReuseAndCoalescing() {
 	const char* test = "DirectPhysicalFreeRangeReuseAndCoalescing";
 	const auto  end  = Libs::LibKernel::Memory::KernelGetDirectMemorySize();
@@ -933,38 +1591,6 @@ void TestFixedReserveRollbackConsumesRestoredPlaceholder() {
 	std::printf("[host]    %-48s ok\n", test);
 }
 
-void TestFixedReserveRollbackRestoresDecommittedHostPages() {
-	const char*        test   = "FixedReserveRollbackRestoresDecommittedHostPages";
-	constexpr uint64_t size   = SceKernelPageSize * 3;
-	void*              mapped = nullptr;
-
-	CheckOk(test,
-	        Libs::LibKernel::Memory::KernelMapNamedFlexibleMemory(&mapped, size, SceKernelProtCpuRw,
-	                                                              0, "host_reserve_rollback"),
-	        "KernelMapNamedFlexibleMemory");
-	const auto base                    = reinterpret_cast<uint64_t>(mapped);
-	*reinterpret_cast<uint64_t*>(base) = 0x4b595459484f5354ull; // "KYTYHOST"
-	*reinterpret_cast<uint64_t*>(base + SceKernelPageSize * 2) =
-	    0x4b5954595441494cull; // "KYTYTAIL"
-
-	Libs::LibKernel::Memory::TestFailHostReservationAfter(1);
-	void* replacement = mapped;
-	CheckFailed(
-	    test,
-	    Libs::LibKernel::Memory::KernelReserveVirtualRange(
-	        &replacement, size, SceKernelMapFixed | SceKernelMapNoCoalesce, SceKernelPageSize),
-	    "KernelReserveVirtualRange(partial host reservation)");
-	Check(test, *reinterpret_cast<uint64_t*>(base) == 0x4b595459484f5354ull,
-	      "rollback did not restore the first flexible page");
-	Check(test, *reinterpret_cast<uint64_t*>(base + SceKernelPageSize * 2) == 0x4b5954595441494cull,
-	      "rollback damaged the flexible tail page");
-	ExpectRange(test, Query(test, base), base, base + size, SceKernelProtCpuRw, 1, 0, 0, 1,
-	            "host_reserve_rollback");
-
-	CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(base, size), "KernelMunmap");
-	std::printf("[host]    %-48s ok\n", test);
-}
-
 void TestFixedReserveRangeAddRollbackKeepsPlaceholder() {
 	const char*        test      = "FixedReserveRangeAddRollbackKeepsPlaceholder";
 	constexpr uint64_t size      = SceKernelPageSize * 4;
@@ -1055,8 +1681,10 @@ void TestLargeHintedReserveHostsSmallDirectMap() {
 	CheckOk(test, Libs::LibKernel::Memory::KernelReleaseDirectMemory(phys, SceKernelPageSize * 2),
 	        "KernelReleaseDirectMemory");
 	CheckOk(test,
-	        Libs::LibKernel::Memory::KernelMunmap(reinterpret_cast<uint64_t>(window), window_size),
+	        Libs::LibKernel::Memory::KernelMunmap(reinterpret_cast<uint64_t>(window) +
-	        "KernelMunmap(window reserve)");
+	                                                  SceKernelPageSize * 2,
+	                                              window_size - SceKernelPageSize * 2),
+	        "KernelMunmap(window reserve remainder)");
 	CheckOk(test,
 	        Libs::LibKernel::Memory::KernelMunmap(reinterpret_cast<uint64_t>(arena), arena_size),
 	        "KernelMunmap(arena reserve)");
@@ -1153,18 +1781,25 @@ void TestProsperoSampleMemoryPoolExpandCommit() {
 	                                                        SceKernelProtCpuRw, 0),
 	        "KernelMemoryPoolCommit");
 	ExpectRange(test, Query(test, base), base, base + commit_len, SceKernelProtCpuRw, 0, 0, 1, 1);
+	Check(test, Libs::LibKernel::Memory::TestGuestAddressRangeIsOwned(base, commit_len),
+	      "pooled commit escaped the guest owner");
 	Check(test, AvailableFlexibleMemory(test) == flexible_baseline,
 	      "pooled commit consumed flexible memory instead of expanded direct "
 	      "backing");
 	CheckFailed(test,
-	            Libs::LibKernel::Memory::KernelReleaseDirectMemory(pool_offset,
+	            Libs::LibKernel::Memory::KernelCheckedReleaseDirectMemory(
-	                                                               SceKernelMemoryPoolExpandLen),
+	                pool_offset, SceKernelMemoryPoolExpandLen),
-	            "KernelReleaseDirectMemory(committed pool expansion)");
+	            "KernelCheckedReleaseDirectMemory(committed pool expansion)");
 
 	constexpr uint64_t first_value     = 0x504f4f4c4241434bull; // "POOLBACK"
 	constexpr uint64_t second_value    = 0x5348415245444d45ull; // "SHAREDME"
 	*reinterpret_cast<uint64_t*>(base) = first_value;
 	*reinterpret_cast<uint64_t*>(base + SceKernelMemoryPoolCommitLen) = second_value;
+	uint64_t backing_read                                             = 0;
+	Check(test, Libs::LibKernel::Memory::TryReadBacking(base, &backing_read, sizeof(backing_read)),
+	      "TryReadBacking did not resolve pooled memory");
+	Check(test, backing_read == first_value,
+	      "shared backing did not observe a pooled-memory CPU write");
 
 	CheckOk(
 	    test,
@@ -1276,8 +1911,10 @@ void TestFragmentedMemoryPoolBacking() {
 	        "KernelMemoryPoolCommit(fragmented recommit)");
 	CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(base, commit_len),
 	        "KernelMunmap(fragmented commit)");
-	CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(base, SceKernelMemoryPoolReserveLen),
+	CheckOk(test,
-	        "KernelMunmap(fragmented reserve cleanup)");
+	        Libs::LibKernel::Memory::KernelMunmap(base + commit_len,
+	                                              SceKernelMemoryPoolReserveLen - commit_len),
+	        "KernelMunmap(fragmented reserve remainder)");
 
 	CheckOk(test,
 	        Libs::LibKernel::Memory::KernelReleaseDirectMemory(first_pool,
@@ -1427,22 +2064,32 @@ void TestMemoryPoolCommitDecommitQueryFlags() {
 	std::printf("[host]    %-48s ok\n", test);
 }
 
-void TestProgramMemoryRegistrationAndProtection() {
+void TestProgramMemoryAllocationAndProtection() {
-	const char* test = "ProgramMemoryRegistrationAndProtection";
+	const char* test = "ProgramMemoryAllocationAndProtection";
 	const auto  size = SceKernelPageSize * 3;
-	const auto base = Common::VirtualMemory::Alloc(0, size, Common::VirtualMemory::Mode::ReadWrite);
+	const auto  base = Libs::LibKernel::Memory::AllocateProgramMemory(
-	Check(test, base != 0, "program host allocation failed");
+	    0x900000000, size, Common::VirtualMemory::Mode::ReadWrite, "program_test");
-
+	Check(test, base != 0, "program guest allocation failed");
-	Libs::LibKernel::Memory::RegisterProgramMemory(
+	Check(test, Libs::LibKernel::Memory::TestGuestAddressRangeIsOwned(base, size),
-	    base, size, Common::VirtualMemory::Mode::ReadWrite, "program_test");
+	      "program allocation escaped the guest owner");
 	ExpectRange(test, Query(test, base), base, base + size,
 	            SceKernelProtCpuRead | SceKernelProtCpuRw, 0, 0, 0, 1, "program_test");
 
-	Libs::LibKernel::Memory::UpdateProgramMemoryProtection(base, SceKernelPageSize,
+	Check(test,
-	                                                       Common::VirtualMemory::Mode::Read);
+	      Libs::LibKernel::Memory::ProtectGuestMemory(base, SceKernelPageSize,
+	                                                  Common::VirtualMemory::Mode::Read),
+	      "ProtectGuestMemory(first page) failed");
 	ExpectRange(test, Query(test, base), base, base + SceKernelPageSize, SceKernelProtCpuRead, 0, 0,
 	            0, 1, "program_test");
 
+	Common::VirtualMemory::Mode previous_mode = Common::VirtualMemory::Mode::NoAccess;
+	Check(test,
+	      Libs::LibKernel::Memory::ProtectGuestMemory(
+	          base, SceKernelPageSize, Common::VirtualMemory::Mode::ReadWrite, &previous_mode),
+	      "ProtectGuestMemory(tracked restore) failed");
+	Check(test, previous_mode == Common::VirtualMemory::Mode::Read,
+	      "semantic guest protection did not preserve its tracked old mode");
+
 	CheckOk(test,
 	        Libs::LibKernel::Memory::KernelMprotect(
 	            reinterpret_cast<void*>(base + SceKernelPageSize - 0x10), 0x20,
@@ -1451,24 +2098,44 @@ void TestProgramMemoryRegistrationAndProtection() {
 	ExpectRange(test, Query(test, base), base, base + size,
 	            SceKernelProtCpuRead | SceKernelProtCpuRw, 0, 0, 0, 1, "program_test");
 
-	Libs::LibKernel::Memory::UpdateProgramMemoryProtection(
+	Check(test,
-	    base + SceKernelPageSize * 2, SceKernelPageSize, Common::VirtualMemory::Mode::Read);
+	      Libs::LibKernel::Memory::ProtectGuestMemory(
+	          base + SceKernelPageSize * 2, SceKernelPageSize, Common::VirtualMemory::Mode::Read),
+	      "ProtectGuestMemory(last page) failed");
 	ExpectRange(test, Query(test, base + SceKernelPageSize * 2), base + SceKernelPageSize * 2,
 	            base + size, SceKernelProtCpuRead, 0, 0, 0, 1, "program_test");
 
-	Libs::LibKernel::Memory::UnregisterProgramMemory(base, size);
+	Check(test, Libs::LibKernel::Memory::FreeGuestMemory(base, size), "program guest free failed");
 	ExpectUnmapped(test, base);
-	Check(test, Common::VirtualMemory::Free(base), "program host free failed");
 
 	std::printf("[host]    %-48s ok\n", test);
 }
 
+void TestModuleRelocationUsesWritableHostMapping() {
+	const char* test = "ModuleRelocationUsesWritableHostMapping";
+	Check(test, Loader::TestModuleRelocationUsesWritableHostMapping(),
+	      "module relocation did not retain writable host memory and semantic guest protection");
+	std::printf("[host]    %-48s ok\n", test);
+}
+
 } // namespace
 
 int main() {
 	InitSubsystems();
 
 	RunTest(TestProsperoArgumentAndInfoSizeContracts);
+	RunTest(TestGuestAddressSpaceOwnsReservationsBeforeBacking);
+	RunTest(TestGuestAddressSpaceHasNoFixedFallback);
+	RunTest(TestGuestFreeRangeSearchDoesNotUnderflow);
+	RunTest(TestFlexibleMemoryCapacityIsBootFixed);
+	RunTest(TestFlexibleMemoryUsesSharedBacking);
+	RunTest(TestFlexibleDmemCompatAndAlignmentFlags);
+	RunTest(TestFlexibleNoCoalescePreservesBoundaries);
+	RunTest(TestFlexibleMemoryReuseIsZeroFilled);
+	RunTest(TestGuestStackUsesPrivateOwnerMemoryAndCache);
+	RunTest(TestMainEntryUsesGuestStackAndDisablesHostChecks);
+	RunTest(TestFragmentedBackingUnmapRollback);
+	RunTest(TestRuntimeMemoryOwnerLifecycle);
 	RunTest(TestFlexibleMapQueryAndWholeMunmap);
 	RunTest(TestPartialFlexibleMunmapAndFindNext);
 	RunTest(TestReserveMapFixedAndNoOverwrite);
@@ -1476,7 +2143,12 @@ int main() {
 	RunTest(TestReleasedReserveCanBeReused);
 	RunTest(TestMunmapAcrossAdjacentFlexibleMappings);
 	RunTest(TestDirectMapQueryOffsetAndPartialMunmap);
+	RunTest(TestDirectPartialProtectUnmapPreservesNeighbors);
+	RunTest(TestDirectMapValidationBeforeOwnerMutation);
+	RunTest(TestDirectReleaseRollbackRestoresOwnerMapping);
+	RunTest(TestDirectReleaseContracts);
 	RunTest(TestNonzeroDirectOffsetAliasesSharedBacking);
+	RunTest(TestDirectMapAcrossContiguousAllocations);
 	RunTest(TestDirectPhysicalFreeRangeReuseAndCoalescing);
 	RunTest(TestDirectAlignmentStaysWithinSearchRange);
 	RunTest(TestDefaultDirectMapUsesSystemAddressRange);
@@ -1485,7 +2157,6 @@ int main() {
 	RunTest(TestFixedReserveReplacesPartialDirectMapping);
 	RunTest(TestFixedReserveRollbackConsumesRestoredPlaceholder);
 	RunTest(TestFixedReserveRollbackSkipsUntouchedChunks);
-	RunTest(TestFixedReserveRollbackRestoresDecommittedHostPages);
 	RunTest(TestFixedReserveRangeAddRollbackKeepsPlaceholder);
 	RunTest(TestLargeHintedReserveHostsSmallDirectMap);
 	RunTest(TestMemoryPoolAlignmentContracts);
@@ -1493,7 +2164,8 @@ int main() {
 	RunTest(TestFragmentedMemoryPoolBacking);
 	RunTest(TestMemoryPoolMultiRangeDecommit);
 	RunTest(TestMemoryPoolCommitDecommitQueryFlags);
-	RunTest(TestProgramMemoryRegistrationAndProtection);
+	RunTest(TestProgramMemoryAllocationAndProtection);
+	RunTest(TestModuleRelocationUsesWritableHostMapping);
 
 	if (g_failed_tests != 0) {
 		std::printf("VirtualMemoryAllocationTests: %d case(s) failed\n", g_failed_tests);