From 2f5396c6a536374869c9edc4ff32102464616930 Mon Sep 17 00:00:00 2001 From: nmzik Date: Fri, 31 Jul 2026 03:07:17 +0200 Subject: [PATCH] Rework guest memory tracking/virtual address space/direct and flexible memory (#135) * Rework guest memory tracking * add unknwon flag * Fix macOS guest address-space reservation --- .github/workflows/build.yml | 20 +- src/CMakeLists.txt | 15 +- src/common/virtualMemory.cpp | 19 - src/common/virtualMemory.h | 1 - src/emulator.cpp | 25 +- src/graphics/host_gpu/memoryTracker.cpp | 33 - src/graphics/host_gpu/memoryTracker.h | 22 +- src/graphics/host_gpu/pageManager.cpp | 507 +--- src/graphics/host_gpu/pageManager.h | 8 +- .../host_gpu/renderer/cache/bufferCache.cpp | 21 - .../host_gpu/renderer/cache/bufferCache.h | 3 +- .../renderer/cache/gpuResourceManager.cpp | 16 +- .../renderer/cache/gpuResourceManager.h | 4 +- .../host_gpu/renderer/cache/textureCache.cpp | 1 - .../renderer/pipeline/descriptors.cpp | 2 - src/kernel/macosGuestAddressSpace.cpp | 8 + src/kernel/memory.cpp | 2323 +++++++++-------- src/kernel/memory.h | 33 +- src/kernel/memoryAddressSpace.inc | 1388 +++++----- src/kernel/pthread.cpp | 159 +- src/kernel/pthread.h | 8 +- src/loader/gamePatch.cpp | 45 +- src/loader/runtimeLinker.cpp | 268 +- src/loader/runtimeLinker.h | 12 +- tests/MemoryTrackerTests.cpp | 81 +- tests/PageManagerTests.cpp | 117 +- tests/ShaderRecompilerComputeTests.cpp | 99 +- tests/VirtualMemoryAllocationTests.cpp | 804 +++++- 28 files changed, 3155 insertions(+), 2887 deletions(-) create mode 100644 src/kernel/macosGuestAddressSpace.cpp diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6e59910..0216615 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -83,7 +83,12 @@ jobs: - name: Build shell: cmd run: | - cmake --build _Build/windows --target launcher --parallel + cmake --build _Build/windows --target launcher virtual_memory_allocation_tests --parallel + + - name: Test + shell: cmd + run: | + ctest --test-dir _Build/windows --output-on-failure -R "^virtual_memory_allocation$" - name: Install shell: cmd @@ -153,7 +158,15 @@ jobs: - name: Build shell: bash run: | - cmake --build _Build/macos --target launcher --parallel + cmake --build _Build/macos \ + --target launcher virtual_memory_allocation_tests \ + --parallel + + - name: Test + shell: bash + run: | + ctest --test-dir _Build/macos --output-on-failure \ + -R '^virtual_memory_allocation$' - name: Install shell: bash @@ -284,13 +297,14 @@ jobs: run: | cmake --build _Build/linux \ --target launcher page_manager_tests memory_tracker_tests \ + virtual_memory_allocation_tests \ --parallel - name: Test shell: bash run: | ctest --test-dir _Build/linux --output-on-failure \ - -R '^(page_manager|memory_tracker)$' + -R '^(page_manager|memory_tracker|virtual_memory_allocation)$' - name: Install shell: bash diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 6043609..8b14d27 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -314,6 +314,16 @@ function(add_kyty_full_emulator_test target source) endif() endfunction() +function(configure_macos_guest_address_space target) + if(APPLE AND (CMAKE_OSX_ARCHITECTURES STREQUAL "x86_64" OR + (NOT CMAKE_OSX_ARCHITECTURES AND CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|AMD64)$"))) + target_sources(${target} PRIVATE kernel/macosGuestAddressSpace.cpp) + target_compile_definitions(${target} PRIVATE KYTY_LINKED_GUEST_ADDRESS_SPACE=1) + target_link_options(${target} PRIVATE + -Wl,-ld_classic,-no_pie,-no_fixup_chains,-no_huge,-pagezero_size,0x40000,-segaddr,SYSTEM_MANAGED,0x40000,-segaddr,SYSTEM_RESERVED,0x7ffffc000,-segaddr,USER_AREA,0x7000000000,-image_base,0x700000000000) + endif() +endfunction() + add_kyty_full_emulator_test(shader_cfg_tests ../tests/shaderCfgTests.cpp) add_executable(scalar_provenance_tests EXCLUDE_FROM_ALL @@ -338,7 +348,6 @@ add_executable(memory_tracker_tests EXCLUDE_FROM_ALL ) target_link_libraries(memory_tracker_tests fmt::fmt common) target_include_directories(memory_tracker_tests PRIVATE ${inc_headers}) -target_compile_definitions(memory_tracker_tests PRIVATE KYTY_MEMORY_TRACKER_TESTS=1) add_executable(shader_vertex_metadata_tests EXCLUDE_FROM_ALL ../tests/ShaderVertexMetadataTests.cpp @@ -421,6 +430,7 @@ target_sources(shader_recompiler_compute_tests PRIVATE add_kyty_full_emulator_test(virtual_memory_allocation_tests ../tests/VirtualMemoryAllocationTests.cpp) target_compile_definitions(virtual_memory_allocation_tests PRIVATE KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS=1) +configure_macos_guest_address_space(virtual_memory_allocation_tests) # These tests use exceptions. if(NOT KYTY_CLANG_CL) @@ -437,6 +447,8 @@ if(BUILD_TESTING) add_test(NAME resource_mutex COMMAND $) add_test(NAME event_queue_lifetime COMMAND $) add_test(NAME shader_recompiler_compute COMMAND $) + add_test(NAME virtual_memory_allocation + COMMAND $) add_test(NAME command_scheduler_timeline COMMAND $ --scheduler-only) add_test(NAME stream_buffer_ring @@ -470,6 +482,7 @@ endif() add_executable(kyty_emulator main.cpp ${kyty_emulator_src}) +configure_macos_guest_address_space(kyty_emulator) target_link_libraries(kyty_emulator ${kyty_emulator_link_libraries}) if (WIN32) diff --git a/src/common/virtualMemory.cpp b/src/common/virtualMemory.cpp index 99af5c8..1855b53 100644 --- a/src/common/virtualMemory.cpp +++ b/src/common/virtualMemory.cpp @@ -58,25 +58,6 @@ bool FlushInstructionCache(uint64_t address, uint64_t size) { return SysVirtualFlushInstructionCache(address, size); } -bool PatchReplace(uint64_t vaddr, uint64_t value) { - Mode old_mode {}; - Protect(vaddr, 8, Mode::ReadWrite, &old_mode); - - auto* ptr = reinterpret_cast(vaddr); - - bool ret = (*ptr != value); - - *ptr = value; - - Protect(vaddr, 8, old_mode); - - if (IsExecute(old_mode)) { - FlushInstructionCache(vaddr, 8); - } - - return ret; -} - } // namespace VirtualMemory } // namespace Common diff --git a/src/common/virtualMemory.h b/src/common/virtualMemory.h index a36d596..8c3af73 100644 --- a/src/common/virtualMemory.h +++ b/src/common/virtualMemory.h @@ -37,7 +37,6 @@ bool Free(uint64_t address); bool FreeRange(uint64_t address, uint64_t size); bool Protect(uint64_t address, uint64_t size, Mode mode, Mode* old_mode = nullptr); bool FlushInstructionCache(uint64_t address, uint64_t size); -bool PatchReplace(uint64_t vaddr, uint64_t value); } // namespace VirtualMemory diff --git a/src/emulator.cpp b/src/emulator.cpp index 60854ee..cea35f6 100644 --- a/src/emulator.cpp +++ b/src/emulator.cpp @@ -105,7 +105,7 @@ static void ClearDebugTextureFolder() { } } -static void Init(const Config::ConfigOptions& cfg) { +static void Init(const Config::ConfigOptions& cfg, const std::filesystem::path& param_json) { EXIT_IF(!Common::Thread::IsMainThread()); auto* slist = Common::SubsystemsList::Instance(); @@ -127,12 +127,21 @@ static void Init(const Config::ConfigOptions& cfg) { slist->InitAll(true); Config::Load(cfg); + slist->Add(log, {core, config}); + slist->InitAll(true); + + if (Common::File::IsFileExisting(param_json)) { + Loader::SystemContentLoadParamSfo(param_json); + if (const auto flexible_memory_size = Loader::SystemContentGetFlexibleMemorySize(); + flexible_memory_size != 0) { + Libs::LibKernel::Memory::SetFlexibleMemorySize(flexible_memory_size); + } + } slist->Add(audio, {core, log, pthread, memory}); slist->Add(controller, {core, log, config}); slist->Add(file_system, {core, log, pthread}); slist->Add(graphics, {core, log, pthread, memory, config, profiler, controller}); - slist->Add(log, {core, config}); slist->Add(memory, {core, log}); slist->Add(network, {core, log, pthread}); slist->Add(profiler, {core, config}); @@ -180,7 +189,8 @@ void Run(const RunOptions& options) { EXIT("ELF is required\n"); } - Init(options.config); + const auto param_json = options.app0_dir / "sce_sys" / "param.json"; + Init(options.config, param_json); ClearDebugTextureFolder(); @@ -192,15 +202,6 @@ void Run(const RunOptions& options) { Libs::LibKernel::FileSystem::Mount(options.app0_dir, "/app0"); Libs::LibKernel::FileSystem::Mount(options.app0_dir, "/hostapp"); - auto param_json = options.app0_dir / "sce_sys" / "param.json"; - if (Common::File::IsFileExisting(param_json)) { - Loader::SystemContentLoadParamSfo(param_json); - if (auto flexible_memory_size = Loader::SystemContentGetFlexibleMemorySize(); - flexible_memory_size != 0) { - Libs::LibKernel::Memory::SetFlexibleMemorySize(flexible_memory_size); - } - } - MountSandboxDirs(); auto* rt = Common::Singleton::Instance(); diff --git a/src/graphics/host_gpu/memoryTracker.cpp b/src/graphics/host_gpu/memoryTracker.cpp index fd78122..0898f8a 100644 --- a/src/graphics/host_gpu/memoryTracker.cpp +++ b/src/graphics/host_gpu/memoryTracker.cpp @@ -4,16 +4,6 @@ namespace Libs::Graphics { -#if defined(KYTY_MEMORY_TRACKER_TESTS) -namespace { -std::atomic g_unmap_contention_hook {nullptr}; -} - -void MemoryTracker::SetUnmapContentionHook(UnmapContentionHook hook) noexcept { - g_unmap_contention_hook.store(hook, std::memory_order_release); -} -#endif - static_assert(std::atomic::is_always_lock_free); MemoryTracker::MemoryTracker(PageManager& page_manager, PageWatchMode gpu_watch_mode) @@ -94,7 +84,6 @@ RegionManager* MemoryTracker::GetOrCreateRegion(uint64_t index) { bool MemoryTracker::IsRegionCpuModified(uint64_t vaddr, uint64_t size) { CheckNotInUploadCallback(); std::lock_guard access(m_access_mutex); - RequireMapped(vaddr, size); return Iterate(vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) { std::scoped_lock lock(manager->lock); return manager->IsModified(offset, bytes); @@ -104,7 +93,6 @@ bool MemoryTracker::IsRegionCpuModified(uint64_t vaddr, uint64_t size) { bool MemoryTracker::IsRegionGpuModified(uint64_t vaddr, uint64_t size) { CheckNotInUploadCallback(); std::lock_guard access(m_access_mutex); - RequireMapped(vaddr, size); return Iterate(vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) { std::scoped_lock lock(manager->lock); return manager->IsModified(offset, bytes); @@ -114,7 +102,6 @@ bool MemoryTracker::IsRegionGpuModified(uint64_t vaddr, uint64_t size) { void MemoryTracker::MarkRegionAsCpuModified(uint64_t vaddr, uint64_t size) { CheckNotInUploadCallback(); std::lock_guard access(m_access_mutex); - RequireMapped(vaddr, size); Iterate(vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) { std::scoped_lock lock(manager->lock); const auto changed = @@ -126,7 +113,6 @@ void MemoryTracker::MarkRegionAsCpuModified(uint64_t vaddr, uint64_t size) { void MemoryTracker::MarkRegionAsGpuModified(uint64_t vaddr, uint64_t size) { CheckNotInUploadCallback(); std::lock_guard access(m_access_mutex); - RequireMapped(vaddr, size); Iterate(vaddr, size, [this](RegionManager* manager, uint64_t offset, uint64_t bytes) { std::scoped_lock lock(manager->lock); const auto changed = @@ -138,7 +124,6 @@ void MemoryTracker::MarkRegionAsGpuModified(uint64_t vaddr, uint64_t size) { void MemoryTracker::UnmarkRegionAsGpuModified(uint64_t vaddr, uint64_t size) { CheckNotInUploadCallback(); std::lock_guard access(m_access_mutex); - RequireMapped(vaddr, size); Iterate(vaddr, size, [this](RegionManager* manager, uint64_t offset, uint64_t bytes) { std::scoped_lock lock(manager->lock); if (!manager->IsFullyModified(offset, bytes)) { @@ -151,8 +136,6 @@ void MemoryTracker::UnmarkRegionAsGpuModified(uint64_t vaddr, uint64_t size) { } void MemoryTracker::UntrackMemoryLocked(uint64_t vaddr, uint64_t size) { - RequireMapped(vaddr, size); - std::vector managers; managers.reserve((vaddr % TRACKER_REGION_SIZE + size + TRACKER_REGION_SIZE - 1) / TRACKER_REGION_SIZE); @@ -185,22 +168,6 @@ void MemoryTracker::UntrackMemory(uint64_t vaddr, uint64_t size) { UntrackMemoryLocked(vaddr, size); } -void MemoryTracker::UnmapMemory(uint64_t vaddr, uint64_t size) { - CheckNotInUploadCallback(); - std::unique_lock access(m_access_mutex, std::try_to_lock); - if (!access.owns_lock()) { -#if defined(KYTY_MEMORY_TRACKER_TESTS) - if (const auto hook = g_unmap_contention_hook.load(std::memory_order_acquire); - hook != nullptr) { - hook(); - } -#endif - access.lock(); - } - UntrackMemoryLocked(vaddr, size); - m_page_manager.OnGpuUnmap(vaddr, size); -} - bool MemoryTracker::InvalidateRegion(uint64_t vaddr, uint64_t size, PageFaultPhase phase) noexcept { switch (phase) { case PageFaultPhase::Release: return true; diff --git a/src/graphics/host_gpu/memoryTracker.h b/src/graphics/host_gpu/memoryTracker.h index 9053e4f..f814da9 100644 --- a/src/graphics/host_gpu/memoryTracker.h +++ b/src/graphics/host_gpu/memoryTracker.h @@ -30,7 +30,6 @@ public: void MarkRegionAsGpuModified(uint64_t vaddr, uint64_t size); void UnmarkRegionAsGpuModified(uint64_t vaddr, uint64_t size); void UntrackMemory(uint64_t vaddr, uint64_t size); - void UnmapMemory(uint64_t vaddr, uint64_t size); [[nodiscard]] CpuFaultAction BeginCpuFault(uint64_t vaddr, uint64_t size, PageFaultAccess access = PageFaultAccess::Write) noexcept; @@ -91,8 +90,7 @@ public: static_assert(std::is_nothrow_invocable_v); static_assert(std::is_nothrow_invocable_v); CheckNotInUploadCallback(); - std::lock_guard access(m_access_mutex); - RequireMapped(vaddr, size); + std::lock_guard access(m_access_mutex); std::vector managers; Iterate(vaddr, size, [&](RegionManager* manager, uint64_t, uint64_t) { managers.push_back(manager); @@ -132,11 +130,6 @@ public: vaddr, size, [](uint64_t, uint64_t) noexcept {}, std::forward(func)); } -#if defined(KYTY_MEMORY_TRACKER_TESTS) - using UnmapContentionHook = void (*)() noexcept; - static void SetUnmapContentionHook(UnmapContentionHook hook) noexcept; -#endif - template void ForEachUploadRange(uint64_t vaddr, uint64_t size, bool is_written, RangeFunc&& range_func, UploadFunc&& upload_func) { @@ -144,7 +137,6 @@ public: static_assert(std::is_nothrow_invocable_v); CheckNotInUploadCallback(); std::unique_lock access(m_access_mutex); - RequireMapped(vaddr, size); Iterate(vaddr, size, [](RegionManager*, uint64_t, uint64_t) {}); const auto* previous_upload_owner = std::exchange(s_upload_owner, this); Iterate(vaddr, size, [&](RegionManager* manager, uint64_t offset, uint64_t bytes) { @@ -209,16 +201,8 @@ private: return false; } - static void ValidateRange(uint64_t vaddr, uint64_t size); - void UntrackMemoryLocked(uint64_t vaddr, uint64_t size); - void RequireMapped(uint64_t vaddr, uint64_t size) const { - ValidateRange(vaddr, size); - if (!m_page_manager.IsMapped(vaddr, size)) { - EXIT("memory tracker range [0x%llx, 0x%llx) is not mapped\n", - static_cast(vaddr), - static_cast(vaddr + size)); - } - } + static void ValidateRange(uint64_t vaddr, uint64_t size); + void UntrackMemoryLocked(uint64_t vaddr, uint64_t size); RegionManager* GetOrCreateRegion(uint64_t index); std::unique_ptr[]> m_regions; diff --git a/src/graphics/host_gpu/pageManager.cpp b/src/graphics/host_gpu/pageManager.cpp index 7c3ab6d..88eb637 100644 --- a/src/graphics/host_gpu/pageManager.cpp +++ b/src/graphics/host_gpu/pageManager.cpp @@ -1,6 +1,7 @@ #include "graphics/host_gpu/pageManager.h" #include "graphics/host_gpu/regionDefinitions.h" +#include "kernel/memory.h" #include #include @@ -21,16 +22,11 @@ #undef min #undef max #elif defined(__APPLE__) -#include -#include #include #include #include #else -#include -#include #include -#include #include #include #include @@ -57,45 +53,8 @@ constexpr uint64_t REGION_PAGES = REGION_SIZE / PAGE_SIZE; constexpr uint32_t NO_ACCESS_PROTECTION = PAGE_NOACCESS; constexpr uint32_t READ_ONLY_PROTECTION = PAGE_READONLY; constexpr uint32_t READ_WRITE_PROTECTION = PAGE_READWRITE; - -#if defined(__APPLE__) -// Map the tracker's Win32-style protection tags to POSIX mprotect flags. -static int PageProtToPosix(uint32_t protection) { - switch (protection) { - case PAGE_NOACCESS: return PROT_NONE; - case PAGE_READONLY: return PROT_READ; - case PAGE_READWRITE: return PROT_READ | PROT_WRITE; - default: return PROT_NONE; - } -} - -// Query the current protection of the page containing vaddr via the Mach VM map and -// collapse it to the tracker's read/write tags (execute is irrelevant to write tracking). -static uint32_t MachQueryPageProt(uint64_t vaddr) { - auto region_addr = static_cast(vaddr); - mach_vm_size_t region_size = 0; - vm_region_basic_info_data_64_t info {}; - mach_msg_type_number_t count = VM_REGION_BASIC_INFO_COUNT_64; - mach_port_t object_name = MACH_PORT_NULL; - - kern_return_t kr = - mach_vm_region(mach_task_self(), ®ion_addr, ®ion_size, VM_REGION_BASIC_INFO_64, - reinterpret_cast(&info), &count, &object_name); - if (kr != KERN_SUCCESS || region_addr > vaddr) { - return PAGE_NOACCESS; // no region covering vaddr - } - if ((info.protection & VM_PROT_WRITE) != 0) { - return PAGE_READWRITE; - } - if ((info.protection & VM_PROT_READ) != 0) { - return PAGE_READONLY; - } - return PAGE_NOACCESS; -} -#elif defined(__linux__) // Zero is the unknown protection sentinel. constexpr uint32_t UNKNOWN_PROTECTION = 0; -#endif thread_local bool g_in_fault_resolution = false; @@ -136,6 +95,15 @@ thread_local bool g_in_fault_resolution = false; std::_Exit(322); } +Common::VirtualMemory::Mode ToMemoryMode(uint32_t protection) { + switch (protection) { + case NO_ACCESS_PROTECTION: return Common::VirtualMemory::Mode::NoAccess; + case READ_ONLY_PROTECTION: return Common::VirtualMemory::Mode::Read; + case READ_WRITE_PROTECTION: return Common::VirtualMemory::Mode::ReadWrite; + default: Fatal("unmappable protection 0x%08" PRIx32, protection); + } +} + uint32_t CurrentThread() noexcept { #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS return GetCurrentThreadId(); @@ -155,130 +123,6 @@ uint32_t CurrentThread() noexcept { #endif } -#if defined(__linux__) -int ToHostProtection(uint32_t protection) { - switch (protection) { - case NO_ACCESS_PROTECTION: return PROT_NONE; - case READ_ONLY_PROTECTION: return PROT_READ; - case READ_WRITE_PROTECTION: return PROT_READ | PROT_WRITE; - default: Fatal("unmappable protection 0x%08" PRIx32, protection); - } -} - -struct HostMapping { - uint64_t end = 0; - uint32_t protection = UNKNOWN_PROTECTION; -}; - -// Async-signal-safe lookup in the address-ordered /proc/self/maps. -HostMapping QueryHostMapping(uint64_t vaddr) noexcept { - int fd = ::open("/proc/self/maps", O_RDONLY | O_CLOEXEC); // NOLINT - if (fd < 0) { - return {}; - } - - enum class Field { Start, End, Perms, Rest }; - - HostMapping result {}; - auto field = Field::Start; - uint64_t start = 0; - uint64_t end = 0; - char perms[4] = {}; - uint32_t perms_len = 0; - bool line_valid = true; - - char buffer[8192]; - - for (bool done = false; !done;) { - const auto got = ::read(fd, buffer, sizeof(buffer)); - if (got < 0) { - if (errno == EINTR) { - continue; - } - break; - } - if (got == 0) { - break; - } - - for (ssize_t i = 0; i < got && !done; i++) { - const char c = buffer[i]; - - if (c == '\n') { - field = Field::Start; - start = 0; - end = 0; - perms_len = 0; - line_valid = true; - continue; - } - - if (!line_valid) { - continue; - } - - switch (field) { - case Field::Start: - case Field::End: { - uint64_t digit = 0; - if (c >= '0' && c <= '9') { - digit = static_cast(c - '0'); - } else if (c >= 'a' && c <= 'f') { - digit = static_cast(c - 'a') + 10; - } else if (c == '-' && field == Field::Start) { - field = Field::End; - break; - } else if (c == ' ' && field == Field::End) { - field = Field::Perms; - perms_len = 0; - break; - } else { - line_valid = false; - break; - } - - auto& value = (field == Field::Start ? start : end); - value = (value << 4u) | digit; - break; - } - - case Field::Perms: { - if (c != ' ') { - if (perms_len < sizeof(perms)) { - perms[perms_len] = c; - } - perms_len++; - break; - } - - if (vaddr < start) { - done = true; - } else if (vaddr < end && perms_len >= 2) { - result.end = end; - result.protection = perms[1] == 'w' ? READ_WRITE_PROTECTION - : perms[0] == 'r' ? READ_ONLY_PROTECTION - : NO_ACCESS_PROTECTION; - done = true; - } else { - field = Field::Rest; - } - break; - } - - case Field::Rest: break; - } - } - } - - ::close(fd); - return result; -} - -uint32_t QueryHostProtection(uint64_t vaddr) noexcept { - return QueryHostMapping(vaddr).protection; -} -#endif - class SpinGuard final { public: explicit SpinGuard(std::atomic_flag& lock): m_lock(lock) { @@ -313,21 +157,16 @@ uint64_t PageEnd(uint64_t vaddr, uint64_t size) { struct PageManager::Impl { struct PageState { std::atomic_flag lock = ATOMIC_FLAG_INIT; - uint32_t mappings = 0; - uint32_t gpu_read_mappings = 0; - uint32_t gpu_write_mappings = 0; uint32_t write_watchers = 0; uint32_t access_watchers = 0; uint32_t original_protection = 0; uint32_t backing_writer = 0; -#if defined(__linux__) // Shadow the protection applied through Protect(). - uint32_t current_protection = UNKNOWN_PROTECTION; -#endif - bool resolving = false; - bool resolving_read_write = false; - bool late_read_pending = false; - bool late_write_pending = false; + uint32_t current_protection = UNKNOWN_PROTECTION; + bool resolving = false; + bool resolving_read_write = false; + bool late_read_pending = false; + bool late_write_pending = false; }; struct Region { @@ -356,7 +195,7 @@ struct PageManager::Impl { Impl(PageFaultHandler handler, void* context): fault_handler(handler), fault_context(context) { if (fault_handler == nullptr) { - Fatal("null fault handler"); + Fatal("null page-manager fault callback"); } #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS SYSTEM_INFO info {}; @@ -386,9 +225,8 @@ struct PageManager::Impl { for (const auto& region: region_storage) { for (auto& page: region->pages) { SpinGuard lock(page.lock); - if (page.mappings != 0 || page.gpu_read_mappings != 0 || - page.gpu_write_mappings != 0 || page.write_watchers != 0 || - page.access_watchers != 0 || page.backing_writer != 0 || page.resolving) { + if (page.write_watchers != 0 || page.access_watchers != 0 || + page.backing_writer != 0 || page.resolving) { FailFast("PageManager destroyed with live page state"); } } @@ -441,179 +279,30 @@ struct PageManager::Impl { } } - static void ValidateInitialProtection(std::span pages, uint64_t vaddr) { - const auto end = vaddr + pages.size() * PAGE_SIZE; -#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS - for (auto address = vaddr; address < end;) { - MEMORY_BASIC_INFORMATION info {}; - if (VirtualQuery(reinterpret_cast(static_cast(address)), &info, - sizeof(info)) == 0 || - info.State != MEM_COMMIT || info.Protect != PAGE_READWRITE) { - Fatal("basic path requires PAGE_READWRITE at 0x%016" PRIx64 " (state=0x%08" PRIx32 - ", protection=0x%08" PRIx32 ")", - address, static_cast(info.State), - static_cast(info.Protect)); - } - const auto region_end = reinterpret_cast(info.BaseAddress) + info.RegionSize; - if (region_end <= address) { - Fatal("VirtualQuery returned an invalid region at 0x%016" PRIx64, address); - } - address = std::min(end, region_end); - } -#elif defined(__APPLE__) - for (auto address = vaddr; address < end; address += PAGE_SIZE) { - const uint32_t protection = MachQueryPageProt(address); - if (protection != PAGE_READWRITE) { - Fatal("basic path requires PAGE_READWRITE at 0x%016" PRIx64 - " (protection=0x%08" PRIx32 ")", - address, protection); - } - } -#else - for (auto address = vaddr; address < end;) { - const auto mapping = QueryHostMapping(address); - if (mapping.protection != READ_WRITE_PROTECTION || mapping.end <= address) { - Fatal("basic path requires a read/write mapping at 0x%016" PRIx64 - " (protection=0x%08" PRIx32 ")", - address, mapping.protection); - } - address = std::min(end, mapping.end); - } - for (auto* page: pages) { - page->current_protection = READ_WRITE_PROTECTION; - } -#endif + static void InitializeProtection(std::span pages) { for (auto* page: pages) { page->original_protection = READ_WRITE_PROTECTION; + page->current_protection = READ_WRITE_PROTECTION; } } - static bool AllowsAccess([[maybe_unused]] const PageState& page, uint64_t vaddr, + static bool AllowsAccess(const PageState& page, [[maybe_unused]] uint64_t vaddr, PageFaultAccess access) noexcept { -#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS - MEMORY_BASIC_INFORMATION info {}; - if (VirtualQuery(reinterpret_cast(static_cast(vaddr)), &info, - sizeof(info)) == 0 || - info.State != MEM_COMMIT) { - return false; - } switch (access) { case PageFaultAccess::Read: - return info.Protect == PAGE_READONLY || info.Protect == PAGE_READWRITE; - case PageFaultAccess::Write: return info.Protect == PAGE_READWRITE; + return page.current_protection == READ_ONLY_PROTECTION || + page.current_protection == READ_WRITE_PROTECTION; + case PageFaultAccess::Write: return page.current_protection == READ_WRITE_PROTECTION; default: return false; } -#elif defined(__APPLE__) - const uint32_t protection = MachQueryPageProt(vaddr); - switch (access) { - case PageFaultAccess::Read: - return protection == PAGE_READONLY || protection == PAGE_READWRITE; - case PageFaultAccess::Write: return protection == PAGE_READWRITE; - default: return false; - } -#else - const auto permitted = [](uint32_t protection, PageFaultAccess wanted) { - switch (wanted) { - case PageFaultAccess::Read: - return protection == READ_ONLY_PROTECTION || - protection == READ_WRITE_PROTECTION; - case PageFaultAccess::Write: return protection == READ_WRITE_PROTECTION; - default: return false; - } - }; - - if (!permitted(page.current_protection, access)) { - return false; - } - return permitted(QueryHostProtection(vaddr), access); -#endif } - static void ProtectRange(std::span pages, uint64_t vaddr, uint32_t protection, - std::span expected_old, bool fault_path) noexcept { + void ProtectRange(std::span pages, uint64_t vaddr, uint32_t protection, + std::span expected_old, bool fault_path) noexcept { const auto size = pages.size() * PAGE_SIZE; if (pages.size() != expected_old.size()) { FailFast("protection range state size mismatch"); } -#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS - struct HostRange { - uint64_t begin = 0; - uint64_t end = 0; - }; - std::vector host_ranges; - const auto end = vaddr + size; - for (auto address = vaddr; address < end;) { - MEMORY_BASIC_INFORMATION info {}; - if (VirtualQuery(reinterpret_cast(static_cast(address)), &info, - sizeof(info)) == 0 || - info.State != MEM_COMMIT) { - if (fault_path) { - FailFast("VirtualProtect fault transition did not match expected protection"); - } - Fatal("invalid protection transition at 0x%016" PRIx64 ", state=0x%08" PRIx32 - ", new=0x%08" PRIx32, - address, static_cast(info.State), protection); - } - const auto region_end = reinterpret_cast(info.BaseAddress) + info.RegionSize; - const auto query_end = std::min(end, region_end); - if (query_end <= address) { - if (fault_path) { - FailFast("VirtualQuery returned an invalid fault transition region"); - } - Fatal("VirtualQuery returned an invalid region at 0x%016" PRIx64, address); - } - const auto first_page = static_cast((address - vaddr) / PAGE_SIZE); - const auto last_page = - static_cast((query_end - vaddr + PAGE_SIZE - 1) / PAGE_SIZE); - for (auto page = first_page; page < last_page; page++) { - if (info.Protect != expected_old[page]) { - if (fault_path) { - FailFast( - "VirtualProtect fault transition did not match expected protection"); - } - Fatal("invalid protection transition at 0x%016" PRIx64 ", actual=0x%08" PRIx32 - ", expected=0x%08" PRIx32 ", new=0x%08" PRIx32, - vaddr + page * PAGE_SIZE, static_cast(info.Protect), - expected_old[page], protection); - } - } - const auto allocation = reinterpret_cast(info.AllocationBase); - if (host_ranges.empty() || allocation != host_ranges.back().begin) { - host_ranges.push_back({allocation, query_end}); - } else { - host_ranges.back().end = query_end; - } - address = query_end; - } - for (auto range: host_ranges) { - range.begin = std::max(range.begin, vaddr); - DWORD old_protection = 0; - const auto first_page = static_cast((range.begin - vaddr) / PAGE_SIZE); - if (VirtualProtect(reinterpret_cast(static_cast(range.begin)), - range.end - range.begin, protection, &old_protection) == 0 || - old_protection != expected_old[first_page]) { - if (fault_path) { - FailFast("VirtualProtect fault transition did not match expected protection"); - } - Fatal("invalid protection transition at 0x%016" PRIx64 ", old=0x%08" PRIx32 - ", expected=0x%08" PRIx32 ", new=0x%08" PRIx32, - range.begin, static_cast(old_protection), expected_old[first_page], - protection); - } - } -#elif defined(__APPLE__) - // mprotect cannot report the previous protection, so the expected_old comparison - // is dropped; the tracker is the sole mutator of these pages and drives the - // transition from its own shadow state. - (void)expected_old; - if (mprotect(reinterpret_cast(static_cast(vaddr)), size, - PageProtToPosix(protection)) != 0) { - if (fault_path) { - FailFast("mprotect fault transition failed"); - } - Fatal("mprotect failed at 0x%016" PRIx64 ", new=0x%08" PRIx32, vaddr, protection); - } -#else for (size_t i = 0; i < pages.size(); i++) { const auto actual = pages[i]->current_protection; if (actual != UNKNOWN_PROTECTION && actual != expected_old[i]) { @@ -625,22 +314,21 @@ struct PageManager::Impl { vaddr + i * PAGE_SIZE, actual, expected_old[i], protection); } } - if (::mprotect(reinterpret_cast(static_cast(vaddr)), size, - ToHostProtection(protection)) != 0) { + if (!Libs::LibKernel::Memory::ProtectGuestHostMemory(vaddr, size, + ToMemoryMode(protection))) { if (fault_path) { - FailFast("mprotect failed on the fault path"); + FailFast("address-space fault protection transition failed"); } - Fatal("mprotect failed at 0x%016" PRIx64 ", new=0x%08" PRIx32 " (%s)", vaddr, - protection, std::strerror(errno)); + Fatal("address-space protection failed at 0x%016" PRIx64 ", new=0x%08" PRIx32, vaddr, + protection); } for (auto* page: pages) { page->current_protection = protection; } -#endif } - static void Protect(PageState& page, uint64_t vaddr, uint32_t protection, uint32_t expected_old, - bool fault_path) noexcept { + void Protect(PageState& page, uint64_t vaddr, uint32_t protection, uint32_t expected_old, + bool fault_path) noexcept { PageState* pages[] = {&page}; uint32_t expected[] = {expected_old}; ProtectRange(pages, vaddr, protection, expected, fault_path); @@ -680,50 +368,6 @@ bool PageManager::IsTracked(uint64_t vaddr) const noexcept { return page.write_watchers != 0 || page.access_watchers != 0; } -bool PageManager::IsMapped(uint64_t vaddr, uint64_t size) const noexcept { - if (vaddr == 0 || size == 0 || vaddr >= ADDRESS_SIZE || size > ADDRESS_SIZE - vaddr) { - return false; - } - const auto end = PageStart(vaddr + size - 1) + PAGE_SIZE; - for (auto page_vaddr = PageStart(vaddr); page_vaddr < end; page_vaddr += PAGE_SIZE) { - auto* region = m_impl->FindRegion(page_vaddr); - if (region == nullptr) { - return false; - } - auto& page = m_impl->GetPage(*region, page_vaddr); - SpinGuard lock(page.lock); - if (page.mappings == 0) { - return false; - } - } - return true; -} - -bool PageManager::HasGpuAccess(uint64_t vaddr, uint64_t size, GpuAccess access) const noexcept { - if (access != GpuAccess::Read && access != GpuAccess::Write && access != GpuAccess::ReadWrite) { - FailFast("HasGpuAccess received an invalid GPU access mode"); - } - const bool need_read = access == GpuAccess::Read || access == GpuAccess::ReadWrite; - const bool need_write = access == GpuAccess::Write || access == GpuAccess::ReadWrite; - if (vaddr == 0 || size == 0 || vaddr >= ADDRESS_SIZE || size > ADDRESS_SIZE - vaddr) { - return false; - } - const auto end = PageEnd(vaddr, size); - for (auto addr = PageStart(vaddr); addr < end; addr += PAGE_SIZE) { - auto* region = m_impl->FindRegion(addr); - if (region == nullptr) { - return false; - } - auto& page = m_impl->GetPage(*region, addr); - SpinGuard lock(page.lock); - if ((need_read && page.gpu_read_mappings == 0) || - (need_write && page.gpu_write_mappings == 0)) { - return false; - } - } - return true; -} - void PageManager::UpdatePageWatchers(bool track, uint64_t vaddr, uint64_t size, PageWatchMode mode) { if (mode != PageWatchMode::Write && mode != PageWatchMode::ReadWrite) { @@ -754,9 +398,6 @@ void PageManager::UpdatePageWatchers(bool track, uint64_t vaddr, uint64_t size, if (page.resolving && track) { FailFast("new page watcher raced active fault resolution"); } - if (page.mappings == 0) { - Fatal("watching unmapped page 0x%016" PRIx64, address); - } auto& watchers = (mode == PageWatchMode::ReadWrite ? page.access_watchers : page.write_watchers); if (track) { @@ -784,8 +425,7 @@ void PageManager::UpdatePageWatchers(bool track, uint64_t vaddr, uint64_t size, last++; } if (first != last) { - Impl::ValidateInitialProtection(std::span {pages}.subspan(first, last - first), - chunk_begin + first * PAGE_SIZE); + Impl::InitializeProtection(std::span {pages}.subspan(first, last - first)); } first = last; } @@ -831,9 +471,9 @@ void PageManager::UpdatePageWatchers(bool track, uint64_t vaddr, uint64_t size, last = current + 1; } } - Impl::ProtectRange(std::span {pages}.subspan(first, last - first), - chunk_begin + first * PAGE_SIZE, protection, - std::span {old_protections}.subspan(first, last - first), false); + m_impl->ProtectRange(std::span {pages}.subspan(first, last - first), + chunk_begin + first * PAGE_SIZE, protection, + std::span {old_protections}.subspan(first, last - first), false); first = current; } @@ -860,70 +500,9 @@ void PageManager::UpdatePageWatchers(bool track, uint64_t vaddr, uint64_t size, } } -void PageManager::OnGpuMap(uint64_t vaddr, uint64_t size, GpuAccess access) { - if (g_in_fault_resolution) { - FailFast("GPU mapping changed during fault resolution"); - } - if (access != GpuAccess::Read && access != GpuAccess::Write && access != GpuAccess::ReadWrite) { - FailFast("GPU map received an invalid access mode"); - } - const bool gpu_read = access == GpuAccess::Read || access == GpuAccess::ReadWrite; - const bool gpu_write = access == GpuAccess::Write || access == GpuAccess::ReadWrite; - const auto end = PageEnd(vaddr, size); - for (auto addr = PageStart(vaddr); addr < end; addr += PAGE_SIZE) { - auto& page = m_impl->GetPage(*m_impl->GetOrCreateRegion(addr), addr); - SpinGuard lock(page.lock); - if (page.resolving || page.mappings == std::numeric_limits::max() || - (gpu_read && page.gpu_read_mappings == std::numeric_limits::max()) || - (gpu_write && page.gpu_write_mappings == std::numeric_limits::max())) { - Fatal("invalid map state at 0x%016" PRIx64, addr); - } - page.mappings++; - page.gpu_read_mappings += gpu_read ? 1u : 0u; - page.gpu_write_mappings += gpu_write ? 1u : 0u; -#if defined(__linux__) - // New guest mappings start read/write. - if (page.current_protection == UNKNOWN_PROTECTION) { - page.current_protection = READ_WRITE_PROTECTION; - } -#endif - } -} +void PageManager::OnGpuMap(uint64_t, uint64_t) {} -void PageManager::OnGpuUnmap(uint64_t vaddr, uint64_t size, GpuAccess access) { - if (g_in_fault_resolution) { - FailFast("GPU unmapping changed during fault resolution"); - } - if (access != GpuAccess::Read && access != GpuAccess::Write && access != GpuAccess::ReadWrite) { - FailFast("GPU unmap received an invalid access mode"); - } - const bool gpu_read = access == GpuAccess::Read || access == GpuAccess::ReadWrite; - const bool gpu_write = access == GpuAccess::Write || access == GpuAccess::ReadWrite; - const auto end = PageEnd(vaddr, size); - for (auto page_vaddr = PageStart(vaddr); page_vaddr < end; page_vaddr += PAGE_SIZE) { - auto* region = m_impl->FindRegion(page_vaddr); - if (region == nullptr) { - Fatal("unmapping unknown page 0x%016" PRIx64, page_vaddr); - } - auto& page = m_impl->GetPage(*region, page_vaddr); - SpinGuard lock(page.lock); - if (page.resolving || page.mappings == 0 || (gpu_read && page.gpu_read_mappings == 0) || - (gpu_write && page.gpu_write_mappings == 0) || - (page.mappings == 1 && (page.write_watchers != 0 || page.access_watchers != 0))) { - Fatal("invalid unmap state at 0x%016" PRIx64, page_vaddr); - } - page.mappings--; - page.gpu_read_mappings -= gpu_read ? 1u : 0u; - page.gpu_write_mappings -= gpu_write ? 1u : 0u; - if (page.mappings == 0) { - if (page.gpu_read_mappings != 0 || page.gpu_write_mappings != 0) { - FailFast("GPU unmap left nonzero GPU mapping counts"); - } - page.late_read_pending = false; - page.late_write_pending = false; - } - } -} +void PageManager::OnGpuUnmap(uint64_t, uint64_t) {} PageManager::BackingWrite::BackingWrite(PageManager& manager, uint64_t vaddr, uint64_t size) noexcept @@ -979,8 +558,7 @@ void PageManager::BeginBackingWrite(uint64_t vaddr, uint64_t size) noexcept { } auto& page = m_impl->GetPage(*region, address); SpinGuard lock(page.lock); - if (page.mappings == 0 || page.resolving || page.backing_writer != 0 || - page.access_watchers == 0) { + if (page.resolving || page.backing_writer != 0 || page.access_watchers == 0) { Fatal("backing write races page resolution at 0x%016" PRIx64, address); } page.resolving = true; @@ -1008,7 +586,7 @@ void PageManager::EndBackingWrite(uint64_t vaddr, uint64_t size) noexcept { const auto old_protection = NO_ACCESS_PROTECTION; const auto new_protection = Impl::WatcherProtection(page); if (new_protection != old_protection) { - Impl::Protect(page, address, new_protection, old_protection, false); + m_impl->Protect(page, address, new_protection, old_protection, false); } Impl::PublishDelayedFaults(page, old_protection, new_protection); if (page.write_watchers == 0 && page.access_watchers == 0) { @@ -1109,7 +687,8 @@ bool PageManager::HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noex page.write_watchers = 0; } const auto restored_protection = Impl::WatcherProtection(page); - Impl::Protect(page, PageStart(fault_vaddr), restored_protection, old_protection, true); + m_impl->Protect(page, PageStart(fault_vaddr), restored_protection, old_protection, + true); if (page.write_watchers == 0) { page.original_protection = 0; } diff --git a/src/graphics/host_gpu/pageManager.h b/src/graphics/host_gpu/pageManager.h index 5143b42..6e98e86 100644 --- a/src/graphics/host_gpu/pageManager.h +++ b/src/graphics/host_gpu/pageManager.h @@ -13,11 +13,9 @@ namespace Libs::Graphics { enum class PageFaultAccess { Read, Write, Execute, Unknown }; enum class PageFaultPhase { Invalidate, Complete, Release }; enum class PageWatchMode { Write, ReadWrite }; -enum class GpuAccess { Read, Write, ReadWrite }; using PageFaultHandler = bool (*)(void* context, PageFaultAccess access, uint64_t vaddr, uint64_t size, PageFaultPhase phase) noexcept; - class PageManager final { public: class BackingWrite final { @@ -40,13 +38,11 @@ public: [[nodiscard]] uint64_t GetPageSize() const; [[nodiscard]] bool IsTracked(uint64_t vaddr) const noexcept; - [[nodiscard]] bool IsMapped(uint64_t vaddr, uint64_t size) const noexcept; - [[nodiscard]] bool HasGpuAccess(uint64_t vaddr, uint64_t size, GpuAccess access) const noexcept; void UpdatePageWatchers(bool track, uint64_t vaddr, uint64_t size, PageWatchMode mode = PageWatchMode::Write); - void OnGpuMap(uint64_t vaddr, uint64_t size, GpuAccess access = GpuAccess::ReadWrite); - void OnGpuUnmap(uint64_t vaddr, uint64_t size, GpuAccess access = GpuAccess::ReadWrite); + void OnGpuMap(uint64_t vaddr, uint64_t size); + void OnGpuUnmap(uint64_t vaddr, uint64_t size); [[nodiscard]] bool HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noexcept; [[nodiscard]] std::vector> diff --git a/src/graphics/host_gpu/renderer/cache/bufferCache.cpp b/src/graphics/host_gpu/renderer/cache/bufferCache.cpp index 3e5bf9a..2d234f5 100644 --- a/src/graphics/host_gpu/renderer/cache/bufferCache.cpp +++ b/src/graphics/host_gpu/renderer/cache/bufferCache.cpp @@ -714,7 +714,6 @@ BufferBinding BufferCache::ObtainBuffer(CommandBuffer& command, uint64_t vaddr, if (command.IsInvalid() || command.IsExecute()) { EXIT("BufferCache: buffer request requires a recording command buffer\n"); } - ValidateGpuAccess(vaddr, size, is_read, is_written); std::lock_guard transaction(m_resource_mutex); (void)SynchronizeBacking(vaddr, size); @@ -999,7 +998,6 @@ void BufferCache::FillBuffer(uint64_t vaddr, uint64_t size, uint32_t value, bool if (vaddr == 0) { EXIT("BufferCache: invalid fill memory address\n"); } - ValidateGpuAccess(vaddr, size, false, true); (void)m_texture_cache.ClearMeta(vaddr); { std::lock_guard transaction(m_resource_mutex); @@ -1041,12 +1039,6 @@ void BufferCache::CopyBuffer(uint64_t dst_vaddr, uint64_t src_vaddr, uint64_t si (src_gds && (src_vaddr > m_gds_buffer.Size() || size > m_gds_buffer.Size() - src_vaddr))) { EXIT("BufferCache: invalid or overlapping copy range\n"); } - if (src_memory) { - ValidateGpuAccess(src_vaddr, size, true, false); - } - if (dst_memory) { - ValidateGpuAccess(dst_vaddr, size, false, true); - } if (src_memory || dst_memory) { std::lock_guard transaction(m_resource_mutex); if (src_memory) { @@ -1203,19 +1195,6 @@ void BufferCache::PublishImageBuffer(uint64_t vaddr, uint64_t size) { owner->second->tick_accessed_last = m_gc_tick; } -void BufferCache::ValidateGpuAccess(uint64_t vaddr, uint64_t size, bool is_read, - bool is_written) const { - if ((!is_read && !is_written) || vaddr == 0 || size == 0 || size > UINT64_MAX - vaddr) { - EXIT("BufferCache: invalid GPU access request\n"); - } - if (is_read && !m_page_manager.HasGpuAccess(vaddr, size, GpuAccess::Read)) { - EXIT("BufferCache: GPU-read access denied\n"); - } - if (is_written && !m_page_manager.HasGpuAccess(vaddr, size, GpuAccess::Write)) { - EXIT("BufferCache: GPU-write access denied\n"); - } -} - void BufferCache::RunGarbageCollector() { std::lock_guard transaction(m_resource_mutex); const auto tick = m_gc_tick++; diff --git a/src/graphics/host_gpu/renderer/cache/bufferCache.h b/src/graphics/host_gpu/renderer/cache/bufferCache.h index 9ce4310..aaefec5 100644 --- a/src/graphics/host_gpu/renderer/cache/bufferCache.h +++ b/src/graphics/host_gpu/renderer/cache/bufferCache.h @@ -77,8 +77,7 @@ public: void CompleteBackingPublication(uint64_t vaddr, uint64_t size, uint64_t tick); [[nodiscard]] bool SynchronizeBacking(uint64_t vaddr, uint64_t size); void PublishImageBuffer(uint64_t vaddr, uint64_t size); - void ValidateGpuAccess(uint64_t vaddr, uint64_t size, bool is_read, bool is_written) const; - void RunGarbageCollector(); + void RunGarbageCollector(); private: friend struct BufferCacheTestAccess; diff --git a/src/graphics/host_gpu/renderer/cache/gpuResourceManager.cpp b/src/graphics/host_gpu/renderer/cache/gpuResourceManager.cpp index 1f7f669..f72de2a 100644 --- a/src/graphics/host_gpu/renderer/cache/gpuResourceManager.cpp +++ b/src/graphics/host_gpu/renderer/cache/gpuResourceManager.cpp @@ -4,7 +4,6 @@ #include "graphics/guest_gpu/command_processor/commandProcessor.h" #include "graphics/guest_gpu/graphicsRun.h" #include "graphics/host_gpu/renderer/commandScheduler.h" - namespace Libs::Graphics { GpuResourceManager::GpuResourceManager(GraphicContext& graphics, CommandScheduler& scheduler) @@ -115,22 +114,19 @@ bool GpuResourceManager::IsMapped(uint64_t vaddr, uint64_t size) const noexcept return m_mapped_ranges.Contains(vaddr, size); } -void GpuResourceManager::MapMemory(uint64_t vaddr, uint64_t size, GpuAccess access) { +void GpuResourceManager::MapMemory(uint64_t vaddr, uint64_t size) { { std::lock_guard lock(m_mapped_ranges_mutex); m_mapped_ranges.Add(vaddr, size); } - m_page_manager.OnGpuMap(vaddr, size, access); + m_page_manager.OnGpuMap(vaddr, size); } -void GpuResourceManager::UnmapMemory(uint64_t vaddr, uint64_t size, GpuAccess access) { - if (!IsMapped(vaddr, size)) { - EXIT("cannot unmap an unmapped GPU resource range\n"); - } - const auto unmap = [this, vaddr, size, access] { - m_texture_cache.UnmapMemory(vaddr, size); +void GpuResourceManager::UnmapMemory(uint64_t vaddr, uint64_t size) { + const auto unmap = [this, vaddr, size] { m_buffer_cache.UnmapMemory(vaddr, size); - m_page_manager.OnGpuUnmap(vaddr, size, access); + m_texture_cache.UnmapMemory(vaddr, size); + m_page_manager.OnGpuUnmap(vaddr, size); std::lock_guard lock(m_mapped_ranges_mutex); m_mapped_ranges.Subtract(vaddr, size); }; diff --git a/src/graphics/host_gpu/renderer/cache/gpuResourceManager.h b/src/graphics/host_gpu/renderer/cache/gpuResourceManager.h index f2dc2f3..db2d822 100644 --- a/src/graphics/host_gpu/renderer/cache/gpuResourceManager.h +++ b/src/graphics/host_gpu/renderer/cache/gpuResourceManager.h @@ -29,8 +29,8 @@ public: [[nodiscard]] bool HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noexcept; [[nodiscard]] bool InvalidateMemory(uint64_t vaddr, uint64_t size); [[nodiscard]] bool IsMapped(uint64_t vaddr, uint64_t size) const noexcept; - void MapMemory(uint64_t vaddr, uint64_t size, GpuAccess access); - void UnmapMemory(uint64_t vaddr, uint64_t size, GpuAccess access); + void MapMemory(uint64_t vaddr, uint64_t size); + void UnmapMemory(uint64_t vaddr, uint64_t size); void RunGarbageCollector(); private: diff --git a/src/graphics/host_gpu/renderer/cache/textureCache.cpp b/src/graphics/host_gpu/renderer/cache/textureCache.cpp index 8d086ef..f95bf54 100644 --- a/src/graphics/host_gpu/renderer/cache/textureCache.cpp +++ b/src/graphics/host_gpu/renderer/cache/textureCache.cpp @@ -1377,7 +1377,6 @@ bool TextureCache::ClearImageFromBuffer(CommandBuffer& command, uint64_t address if (command.IsInvalid() || !GuestRange {address, size}.Valid()) { EXIT("TextureCache: invalid image clear\n"); } - m_buffer_cache.ValidateGpuAccess(address, size, false, true); std::lock_guard transaction(m_resource_mutex); CacheLock lock(*this, m_lock); ImageId selected {}; diff --git a/src/graphics/host_gpu/renderer/pipeline/descriptors.cpp b/src/graphics/host_gpu/renderer/pipeline/descriptors.cpp index 7b90476..597d7fc 100644 --- a/src/graphics/host_gpu/renderer/pipeline/descriptors.cpp +++ b/src/graphics/host_gpu/renderer/pipeline/descriptors.cpp @@ -616,8 +616,6 @@ RenderExecutor::ResolveTexture(const ShaderRecompiler::IR::ImageResource& reso (address & (static_cast(size.align) - 1u)) != 0); if (storage) { ValidateStorageTexture(resource, descriptor, size.size); - m_context.GetBufferCache().ValidateGpuAccess(address, size.size, resource.read, - resource.written); } const auto pixel_format = TextureGetFormat(format); diff --git a/src/kernel/macosGuestAddressSpace.cpp b/src/kernel/macosGuestAddressSpace.cpp new file mode 100644 index 0000000..892e84f --- /dev/null +++ b/src/kernel/macosGuestAddressSpace.cpp @@ -0,0 +1,8 @@ +#if defined(__APPLE__) && defined(__x86_64__) + +// Make the process own the guest ranges before any runtime initialization. +asm(".zerofill SYSTEM_MANAGED,SYSTEM_MANAGED,__kyty_system_managed,0x7fffbc000"); +asm(".zerofill SYSTEM_RESERVED,SYSTEM_RESERVED,__kyty_system_reserved,0x7c0004000"); +asm(".zerofill USER_AREA,USER_AREA,__kyty_user_area,0x8c00000000"); + +#endif diff --git a/src/kernel/memory.cpp b/src/kernel/memory.cpp index ab8f932..bd5ed1f 100644 --- a/src/kernel/memory.cpp +++ b/src/kernel/memory.cpp @@ -66,8 +66,9 @@ constexpr int PAGE_TABLE_POOL_ENTRIES = static_cast(PAGE_TABLE_POOL_SIZE / PAGE_TABLE_GRANULARITY); constexpr uint64_t DEFAULT_FLEXIBLE_MEMORY_SIZE = 4ull * 1024ull * 1024ull * 1024ull; -static uint64_t g_flexible_memory_size = DEFAULT_FLEXIBLE_MEMORY_SIZE; -static Graphics::GpuResourceManager* g_gpu_resources = nullptr; +static uint64_t g_flexible_memory_size = DEFAULT_FLEXIBLE_MEMORY_SIZE; +static bool g_flexible_memory_size_frozen = false; +static Graphics::GpuResourceManager* g_gpu_resources = nullptr; static Graphics::GpuResourceManager& GetGpuResources() { EXIT_IF(g_gpu_resources == nullptr); @@ -76,34 +77,21 @@ static Graphics::GpuResourceManager& GetGpuResources() { static bool IsGpuAddressRange(uint64_t vaddr, uint64_t size) { constexpr uint64_t GPU_ADDRESS_LIMIT = 1ull << 40u; - return vaddr != 0 && size != 0 && vaddr < GPU_ADDRESS_LIMIT && - size <= GPU_ADDRESS_LIMIT - vaddr; + return vaddr != 0 && size != 0 && vaddr < GPU_ADDRESS_LIMIT && size < GPU_ADDRESS_LIMIT - vaddr; } -static void MapGpuRange(uint64_t vaddr, uint64_t size, GpuAccessMode mode) { - if (mode == GpuAccessMode::NoAccess) { +static void MapGpuRange(uint64_t vaddr, uint64_t size) { + if (g_gpu_resources == nullptr || !IsGpuAddressRange(vaddr, size)) { return; } - if (!IsGpuAddressRange(vaddr, size)) { - EXIT("invalid GPU map range: addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", vaddr, size); - } - const auto access = mode == GpuAccessMode::Read ? Graphics::GpuAccess::Read - : mode == GpuAccessMode::Write ? Graphics::GpuAccess::Write - : Graphics::GpuAccess::ReadWrite; - GetGpuResources().MapMemory(vaddr, size, access); + GetGpuResources().MapMemory(vaddr, size); } -static void UnmapGpuRange(uint64_t vaddr, uint64_t size, GpuAccessMode mode) { - if (mode == GpuAccessMode::NoAccess) { +static void UnmapGpuRange(uint64_t vaddr, uint64_t size) { + if (g_gpu_resources == nullptr || !IsGpuAddressRange(vaddr, size)) { return; } - if (!IsGpuAddressRange(vaddr, size)) { - EXIT("invalid GPU unmap range: addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", vaddr, size); - } - const auto access = mode == GpuAccessMode::Read ? Graphics::GpuAccess::Read - : mode == GpuAccessMode::Write ? Graphics::GpuAccess::Write - : Graphics::GpuAccess::ReadWrite; - GetGpuResources().UnmapMemory(vaddr, size, access); + GetGpuResources().UnmapMemory(vaddr, size); } static bool DecodeMemoryProtection(int prot, VirtualMemory::Mode* mode, GpuAccessMode* gpu_mode) { @@ -153,69 +141,6 @@ static bool DecodeMemoryProtection(int prot, VirtualMemory::Mode* mode, GpuAcces return true; } -static GpuAccessMode GetGpuAccessMode(int prot) { - VirtualMemory::Mode mode {}; - GpuAccessMode gpu_mode {}; - if (!DecodeMemoryProtection(prot, &mode, &gpu_mode)) { - EXIT("unsupported GPU memory protection: 0x%08x\n", prot); - } - return gpu_mode; -} - -static bool ProtectCommittedHostMemory(uint64_t start, uint64_t size, VirtualMemory::Mode mode, - VirtualMemory::Mode* old_mode) { - if (size == 0) { - if (old_mode != nullptr) { - *old_mode = VirtualMemory::Mode::NoAccess; - } - return true; - } - -#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS - const auto end = start + size; - auto cur = start; - bool protected_any = false; - - while (cur < end) { - MEMORY_BASIC_INFORMATION mbi {}; - if (VirtualQuery(reinterpret_cast(cur), &mbi, sizeof(mbi)) == 0) { - return false; - } - - const auto region_start = reinterpret_cast(mbi.BaseAddress); - const auto region_end = region_start + static_cast(mbi.RegionSize); - const auto chunk_start = std::max(cur, region_start); - const auto chunk_end = std::min(end, region_end); - - if (chunk_end <= chunk_start) { - return false; - } - - if (mbi.State == MEM_COMMIT) { - VirtualMemory::Mode chunk_old {}; - if (!VirtualMemory::Protect(chunk_start, chunk_end - chunk_start, mode, &chunk_old)) { - return false; - } - - if (!protected_any && old_mode != nullptr) { - *old_mode = chunk_old; - } - protected_any = true; - } - - cur = chunk_end; - } - - if (!protected_any && old_mode != nullptr) { - *old_mode = VirtualMemory::Mode::NoAccess; - } - - return true; -#else - return VirtualMemory::Protect(start, size, mode, old_mode); -#endif -} - static void CopyVirtualRangeName(char* dst, const char* name) { EXIT_IF(dst == nullptr); @@ -239,49 +164,10 @@ static bool VirtualRangesOverlap(uint64_t left_start, uint64_t left_size, uint64 return left_start < right_end && right_start < left_end; } -static bool CommitFixedHostRange(uint64_t start, uint64_t size, VirtualMemory::Mode mode) { - constexpr uint64_t PAGE_SIZE = 0x4000; - -#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS - // Prefer one syscall, then fall back to the per-page path. - if (size > PAGE_SIZE && VirtualMemory::AllocFixed(start, size, mode)) { - return true; - } +#if defined(KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS) +static uint32_t g_test_backing_store_unmaps_before_failure = UINT32_MAX; #endif - for (uint64_t addr = start; addr < start + size; addr += PAGE_SIZE) { -#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS - MEMORY_BASIC_INFORMATION info {}; - if (VirtualQuery(reinterpret_cast(addr), &info, sizeof(info)) != 0) { - if (info.State == MEM_COMMIT) { - if (!VirtualMemory::Protect(addr, PAGE_SIZE, mode)) { - return false; - } - continue; - } - if (info.State == MEM_RESERVE) { - if (!VirtualMemory::Commit(addr, PAGE_SIZE, mode)) { - return false; - } - continue; - } - } -#endif - if (VirtualMemory::AllocFixed(addr, PAGE_SIZE, mode)) { - continue; - } - if (VirtualMemory::Commit(addr, PAGE_SIZE, mode)) { - continue; - } - if (VirtualMemory::Protect(addr, PAGE_SIZE, mode)) { - continue; - } - return false; - } - - return true; -} - #include "memoryAddressSpace.inc" enum class VirtualRangeType { @@ -292,6 +178,7 @@ enum class VirtualRangeType { Pooled, Stack, Code, + Runtime, }; static bool IsReservedRangeType(VirtualRangeType type) { @@ -306,25 +193,26 @@ static bool IsCommittedRangeType(VirtualRangeType type) { return !IsReservedRangeType(type); } +static bool IsPrivateCommittedRangeType(VirtualRangeType type) { + return type == VirtualRangeType::Stack || type == VirtualRangeType::Code || + type == VirtualRangeType::Runtime; +} + class VirtualRanges { public: struct Range { - uint64_t start = 0; - uint64_t size = 0; - uint64_t offset = 0; - int protection = 0; - int memory_type = 0; - VirtualRangeType type = VirtualRangeType::Reserved; - bool committed_from_reserved = false; - // Reserved placeholders need OS-specific release/commit paths; plain reserves still use - // Common::VirtualMemory. - bool placeholder_backed = false; - char name[KERNEL_MAXIMUM_NAME_LENGTH]; + uint64_t start = 0; + uint64_t size = 0; + uint64_t offset = 0; + int protection = 0; + int memory_type = 0; + VirtualRangeType type = VirtualRangeType::Reserved; + bool disallow_merge = false; + char name[KERNEL_MAXIMUM_NAME_LENGTH]; }; bool Add(uint64_t start, uint64_t size, uint64_t offset, int protection, int memory_type, - VirtualRangeType type, const char* name, bool committed_from_reserved = false, - bool placeholder_backed = false) { + VirtualRangeType type, const char* name, bool disallow_merge = false) { Common::LockGuard lock(m_mutex); if (start == 0 || size == 0) { @@ -340,14 +228,13 @@ public: } Range r {}; - r.start = start; - r.size = size; - r.offset = offset; - r.protection = protection; - r.memory_type = memory_type; - r.type = type; - r.committed_from_reserved = committed_from_reserved; - r.placeholder_backed = placeholder_backed; + r.start = start; + r.size = size; + r.offset = offset; + r.protection = protection; + r.memory_type = memory_type; + r.type = type; + r.disallow_merge = disallow_merge; CopyVirtualRangeName(r.name, name); const auto index = static_cast(position - m_ranges.begin()); m_ranges.insert(position, r); @@ -374,13 +261,16 @@ public: return FindOverlap(start, size) != nullptr; } - bool HasGpuAccess(uint64_t start, uint64_t size) { + bool QueryOverlap(uint64_t start, uint64_t size, Range* out) { + EXIT_IF(out == nullptr); Common::LockGuard lock(m_mutex); - return std::any_of(m_ranges.begin(), m_ranges.end(), [start, size](const auto& range) { - return VirtualRangesOverlap(start, size, range.start, range.size) && - (range.protection & (PROT_GPU_READ | PROT_GPU_WRITE)) != 0; - }); + const auto* overlap = FindOverlap(start, size); + if (overlap == nullptr) { + return false; + } + *out = *overlap; + return true; } bool ReleaseReserved(uint64_t start, uint64_t size) { @@ -390,7 +280,7 @@ public: auto& r = m_ranges[index]; if (r.start == start && r.size == size && IsReservedRangeType(r.type)) { m_ranges.erase(m_ranges.begin() + static_cast(index)); - return VirtualMemory::Free(start); + return true; } } return true; @@ -595,14 +485,13 @@ private: } static bool SameMergeKey(const Range& left, const Range& right) { - if (left.type == VirtualRangeType::Direct || right.type == VirtualRangeType::Direct) { + if (left.disallow_merge || right.disallow_merge || left.type == VirtualRangeType::Direct || + right.type == VirtualRangeType::Direct) { return false; } return left.type == right.type && left.protection == right.protection && left.memory_type == right.memory_type && - left.committed_from_reserved == right.committed_from_reserved && - left.placeholder_backed == right.placeholder_backed && std::strncmp(left.name, right.name, KERNEL_MAXIMUM_NAME_LENGTH) == 0; } @@ -759,7 +648,6 @@ private: #if defined(KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS) static uint32_t g_test_physical_memory_unmaps_before_failure = UINT32_MAX; -static uint32_t g_test_host_reservation_pages_before_failure = UINT32_MAX; static bool g_test_fail_next_fixed_reserve_range_add = false; #endif @@ -784,11 +672,15 @@ public: EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread()); m_free.emplace(0, Size()); } - virtual ~PhysicalMemory() { KYTY_NOT_IMPLEMENTED; } + virtual ~PhysicalMemory() = default; KYTY_CLASS_NO_COPY(PhysicalMemory); - static uint64_t Size() { return static_cast(13824) * 1024 * 1024; } + static constexpr uint64_t TotalSize() { return static_cast(13824) * 1024 * 1024; } + static uint64_t Size() { + EXIT_IF(g_flexible_memory_size >= TotalSize()); + return TotalSize() - g_flexible_memory_size; + } bool Alloc(uint64_t search_start, uint64_t search_end, size_t len, size_t alignment, uint64_t* phys_addr_out, int memory_type, bool pool_expansion = false); @@ -805,8 +697,11 @@ public: bool Find(uint64_t phys_addr, bool next, PhysicalMemory::AllocatedBlock* out); bool CanMapDirect(uint64_t phys_addr, size_t len); bool ReleasePoolExpansion(uint64_t phys_addr, size_t len); + bool GetAllocatedSpan(uint64_t phys_addr, size_t len, std::vector* blocks); std::vector FindMappings(uint64_t phys_addr, size_t len); - void SetVirtualRangeName(uint64_t vaddr, uint64_t len, const char* name); + void ProtectMapping(uint64_t vaddr, uint64_t size, int prot, VirtualMemory::Mode mode, + GpuAccessMode gpu_mode); + void SetVirtualRangeName(uint64_t vaddr, uint64_t len, const char* name); void SetVirtualRangeMemoryType(uint64_t vaddr, uint64_t len, int memory_type); [[nodiscard]] Common::Mutex& GetMutex() { return m_mutex; } @@ -831,6 +726,7 @@ public: struct AllocatedBlock { uint64_t map_vaddr; uint64_t map_size; + uint64_t backing_offset; uint64_t host_vaddr; uint64_t host_size; int prot; @@ -839,8 +735,11 @@ public: char name[KERNEL_MAXIMUM_NAME_LENGTH]; }; - FlexibleMemory() { EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread()); } - virtual ~FlexibleMemory() { KYTY_NOT_IMPLEMENTED; } + FlexibleMemory() { + EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread()); + m_free.emplace(PhysicalMemory::Size(), Size()); + } + virtual ~FlexibleMemory() = default; KYTY_CLASS_NO_COPY(FlexibleMemory); @@ -853,15 +752,24 @@ public: uint64_t* host_vaddr_to_release = nullptr); bool Find(uint64_t vaddr, uint64_t* base_addr, size_t* len, int* prot, VirtualMemory::Mode* mode, GpuAccessMode* gpu_mode); + bool Snapshot(uint64_t vaddr, uint64_t size, std::vector* blocks); + bool Restore(const std::vector& blocks); + void Protect(uint64_t vaddr, uint64_t size, int prot, VirtualMemory::Mode mode, + GpuAccessMode gpu_mode); void SetVirtualRangeName(uint64_t vaddr, uint64_t len, const char* name); [[nodiscard]] Common::Mutex& GetMutex() { return m_mutex; } [[nodiscard]] const std::vector& GetBlocks() const { return m_allocated; } private: - std::vector m_allocated; - uint64_t m_allocated_total = 0; - Common::Mutex m_mutex; + void ConsumeFreeRange(std::map::iterator range, uint64_t start, + uint64_t size); + void AddFreeRange(uint64_t start, uint64_t size); + + std::vector m_allocated; + std::map m_free; + uint64_t m_allocated_total = 0; + Common::Mutex m_mutex; }; class PooledMemory { @@ -897,27 +805,57 @@ private: Common::Mutex m_mutex; }; -static PhysicalMemory* g_physical_memory = nullptr; -static FlexibleMemory* g_flexible_memory = nullptr; -static PooledMemory* g_pooled_memory = nullptr; -static VirtualRanges* g_virtual_ranges = nullptr; -static DirectMemoryBacking* g_direct_memory_backing = nullptr; -static PlaceholderAddressSpace* g_placeholder_address_space = nullptr; -static callback_func_t g_alloc_callback = nullptr; -static callback_func_t g_free_callback = nullptr; -static std::atomic g_memory_pool_committed = 0; -static void MemoryPoolSubtractCommitted(uint64_t len); +static PhysicalMemory* g_physical_memory = nullptr; +static FlexibleMemory* g_flexible_memory = nullptr; +static PooledMemory* g_pooled_memory = nullptr; +static VirtualRanges* g_virtual_ranges = nullptr; +static GuestAddressSpace* g_guest_address_space = nullptr; +static callback_func_t g_alloc_callback = nullptr; +static callback_func_t g_free_callback = nullptr; +static std::atomic g_memory_pool_committed = 0; +static void MemoryPoolSubtractCommitted(uint64_t len); // Keep host mappings, physical blocks, placeholders, and virtual ranges in step. static std::recursive_mutex g_memory_operation_mutex; +static uint64_t FindGuestFreeRange(uint64_t search_addr, uint64_t size, uint64_t alignment) { + EXIT_IF(g_guest_address_space == nullptr || g_virtual_ranges == nullptr); + + auto find_in = [&](uint64_t begin, uint64_t end) { + auto current = begin; + while (current < end && size <= end - current) { + const auto candidate = + g_guest_address_space->FindFreeAligned(current, end, size, alignment); + if (candidate == 0) { + return uint64_t {0}; + } + VirtualRanges::Range overlap {}; + if (!g_virtual_ranges->QueryOverlap(candidate, size, &overlap)) { + return candidate; + } + const auto overlap_end = overlap.start + overlap.size; + if (overlap_end <= current) { + return uint64_t {0}; + } + current = overlap_end; + } + return uint64_t {0}; + }; + + if (search_addr != 0) { + return find_in(search_addr, HOST_USER_MAX + 1u); + } + auto addr = find_in(HOST_SYSTEM_MANAGED_MIN, HOST_SYSTEM_MANAGED_MAX + 1u); + return addr != 0 ? addr : find_in(HOST_USER_MIN, HOST_USER_MAX + 1u); +} + bool TryWriteBacking(uint64_t vaddr, const void* data, uint64_t size) { - return g_direct_memory_backing != nullptr && - g_direct_memory_backing->TryWriteBacking(vaddr, data, size); + return g_guest_address_space != nullptr && + g_guest_address_space->TryWriteBacking(vaddr, data, size); } bool TryReadBacking(uint64_t vaddr, void* data, uint64_t size) { - return g_direct_memory_backing != nullptr && - g_direct_memory_backing->TryReadBacking(vaddr, data, size); + return g_guest_address_space != nullptr && + g_guest_address_space->TryReadBacking(vaddr, data, size); } uint64_t ClampRangeSize(uint64_t vaddr, uint64_t size) { @@ -978,8 +916,7 @@ static bool IsInPrtAperture(uint64_t address) { Common::LockGuard lock(g_prt_aperture_mutex); for (const auto& aperture: g_prt_apertures) { - if (aperture.size != 0 && address >= aperture.address && - address < aperture.address + aperture.size) { + if (address >= aperture.address && address < aperture.address + aperture.size) { return true; } } @@ -987,108 +924,74 @@ static bool IsInPrtAperture(uint64_t address) { return false; } -static void SelfTestSub64SharedPlaceholderAlias() { +static bool SelfTestSub64SharedPlaceholderAlias() { #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS constexpr uint64_t PageSize = 0x4000; - const auto granularity = g_placeholder_address_space->GetGranularity(); + const auto granularity = g_guest_address_space->GetGranularity(); if (granularity < PageSize * 2u) { LOGF_COLOR( Log::Color::Yellow, "\t direct-memory sub-64K placeholder self-test skipped: granularity too small\n"); - return; + return true; } - const auto base = g_placeholder_address_space->ReserveAligned(0, granularity, granularity); + const auto base = FindGuestFreeRange(0, granularity, granularity); if (base == 0) { - LOGF_COLOR(Log::Color::Yellow, - "\t direct-memory sub-64K placeholder self-test skipped: reserve unavailable\n"); - return; + LOGF_COLOR(Log::Color::Red, + "\t direct-memory sub-64K placeholder self-test: reserve unavailable\n"); + return false; } const auto alias = base + PageSize; bool ok = false; - auto failure_reason = DirectMemoryBacking::FailureReason::None; - const bool consumed = g_placeholder_address_space->Consume(alias, PageSize); - - if (consumed && - g_direct_memory_backing->MapExistingPlaceholderFixed( - alias, PageSize, PageSize, VirtualMemory::Mode::ReadWrite, &failure_reason)) { + auto failure_reason = GuestBackingStore::FailureReason::None; + if (g_guest_address_space->MapBacking(alias, PageSize, PageSize, VirtualMemory::Mode::ReadWrite, + &failure_reason)) { auto* ptr = reinterpret_cast(alias); *ptr = 0x4b59545953553634ull; // "KYTYSU64" ok = (*ptr == 0x4b59545953553634ull); std::memset(ptr, 0, PageSize); - bool placeholder_preserved = false; - ok = g_direct_memory_backing->Unmap(alias, PageSize, true, &placeholder_preserved) && - placeholder_preserved && ok; - if (placeholder_preserved) { - g_placeholder_address_space->AddFree(alias, PageSize); - } - } else if (consumed) { - g_placeholder_address_space->AddFree(alias, PageSize); + ok = g_guest_address_space->UnmapBacking(alias, PageSize) && ok; } - const bool released = g_placeholder_address_space->ReleaseFree(base, granularity); - LOGF_COLOR(ok && released ? Log::Color::Green : Log::Color::Red, - "\t direct-memory sub-64K placeholder self-test: %s%s%s\n", - ok && released ? "ok" : "failed", ok ? "" : ", reason = ", - ok ? "" : DirectMemoryBacking::GetFailureReasonName(failure_reason)); + LOGF_COLOR( + ok ? Log::Color::Green : Log::Color::Red, + "\t direct-memory sub-64K placeholder self-test: %s%s%s\n", ok ? "ok" : "failed", + ok ? "" : ", reason = ", ok ? "" : GuestBackingStore::GetFailureReasonName(failure_reason)); + return ok; +#else + return true; #endif } -static bool RestoreCommittedPlaceholderOrProtect(uint64_t vaddr, uint64_t size) { - if (g_placeholder_address_space->ReleaseCommitted(vaddr, size)) { - return true; - } - VirtualMemory::Protect(vaddr, size, VirtualMemory::Mode::NoAccess); - return false; -} - -static bool ReleaseReservedRange(uint64_t vaddr, uint64_t size) { - VirtualRanges::Range range {}; - if (g_virtual_ranges->Query(vaddr, 0, &range) && range.start == vaddr && range.size == size && - IsReservedRangeType(range.type) && range.placeholder_backed) { - if (!g_virtual_ranges->ConsumeReserved(vaddr, size, range.type)) { - return false; - } - if (g_placeholder_address_space->ReleaseFree(vaddr, size)) { - return true; - } - if (VirtualMemory::Free(vaddr)) { - return true; - } - g_virtual_ranges->Add(vaddr, size, 0, 0, 0, range.type, range.name, false, true); - return false; - } - return g_virtual_ranges->ReleaseReserved(vaddr, size); -} - -static bool ReplaceFixedRangeWithReserved(uint64_t start, uint64_t size, bool* placeholder_backed); +static bool ReplaceFixedRangeWithReserved(uint64_t start, uint64_t size); KYTY_SUBSYSTEM_INIT(Memory) { - g_flexible_memory_size = DEFAULT_FLEXIBLE_MEMORY_SIZE; - g_physical_memory = new PhysicalMemory; - g_flexible_memory = new FlexibleMemory; - g_pooled_memory = new PooledMemory; - g_virtual_ranges = new VirtualRanges; - g_direct_memory_backing = new DirectMemoryBacking(PhysicalMemory::Size()); - g_placeholder_address_space = new PlaceholderAddressSpace; - + g_flexible_memory_size_frozen = true; VirtualMemory::Init(); - EXIT_IF(!g_direct_memory_backing->SelfTest()); - g_placeholder_address_space->SelfTest(); - SelfTestSub64SharedPlaceholderAlias(); + g_guest_address_space = new GuestAddressSpace(PhysicalMemory::TotalSize()); + g_physical_memory = new PhysicalMemory; + g_flexible_memory = new FlexibleMemory; + g_pooled_memory = new PooledMemory; + g_virtual_ranges = new VirtualRanges; + EXIT_IF(!g_guest_address_space->SelfTest()); + EXIT_IF(!SelfTestSub64SharedPlaceholderAlias()); } KYTY_SUBSYSTEM_UNEXPECTED_SHUTDOWN(Memory) {} KYTY_SUBSYSTEM_DESTROY(Memory) { - delete g_placeholder_address_space; - g_placeholder_address_space = nullptr; - delete g_direct_memory_backing; - g_direct_memory_backing = nullptr; delete g_pooled_memory; g_pooled_memory = nullptr; + delete g_flexible_memory; + g_flexible_memory = nullptr; + delete g_physical_memory; + g_physical_memory = nullptr; + delete g_virtual_ranges; + g_virtual_ranges = nullptr; + delete g_guest_address_space; + g_guest_address_space = nullptr; } struct AlignedPos { @@ -1141,6 +1044,9 @@ void RegisterCallbacks(callback_func_t alloc_func, callback_func_t free_func) { } void SetFlexibleMemorySize(uint64_t size) { + constexpr uint64_t GuestPageSize = 0x4000; + EXIT_IF(g_flexible_memory_size_frozen || g_guest_address_space != nullptr); + EXIT_IF(size == 0 || (size & (GuestPageSize - 1u)) != 0 || size >= PhysicalMemory::TotalSize()); g_flexible_memory_size = size; LOGF("\t flexible memory size = 0x%016" PRIx64 " (%" PRIu64 " MiB)\n", size, size / (1024ull * 1024ull)); @@ -1355,18 +1261,37 @@ bool PhysicalMemory::Map(uint64_t vaddr, uint64_t phys_addr, size_t len, int pro VirtualMemory::Mode mode, GpuAccessMode gpu_mode) { Common::LockGuard lock(m_mutex); - auto next = m_physical.upper_bound(phys_addr); - if (next == m_physical.begin()) { - return false; - } - const auto& block = std::prev(next)->second; - if (block.pool_expansion || phys_addr < block.start_addr || - phys_addr >= block.start_addr + block.size || - len > block.start_addr + block.size - phys_addr) { + if (len == 0 || UINT64_MAX - phys_addr < len) { return false; } - AllocatedBlock mapping = block; + auto current = phys_addr; + auto next = m_physical.upper_bound(current); + if (next == m_physical.begin()) { + return false; + } + auto first = std::prev(next); + while (current < phys_addr + len) { + auto block = m_physical.upper_bound(current); + if (block == m_physical.begin()) { + return false; + } + --block; + const auto block_end = block->second.start_addr + block->second.size; + if (block->second.pool_expansion || current < block->second.start_addr || + current >= block_end) { + return false; + } + current = std::min(phys_addr + len, block_end); + if (current < phys_addr + len) { + const auto following = std::next(block); + if (following == m_physical.end() || following->second.start_addr != current) { + return false; + } + } + } + + AllocatedBlock mapping = first->second; mapping.start_addr = phys_addr; mapping.size = len; mapping.map_vaddr = vaddr; @@ -1384,14 +1309,32 @@ bool PhysicalMemory::Map(uint64_t vaddr, uint64_t phys_addr, size_t len, int pro bool PhysicalMemory::CanMapDirect(uint64_t phys_addr, size_t len) { Common::LockGuard lock(m_mutex); - auto next = m_physical.upper_bound(phys_addr); - if (next == m_physical.begin()) { + if (len == 0 || UINT64_MAX - phys_addr < len) { return false; } - const auto& block = std::prev(next)->second; - return !block.pool_expansion && phys_addr >= block.start_addr && - phys_addr < block.start_addr + block.size && - len <= block.start_addr + block.size - phys_addr; + + const auto end = phys_addr + len; + auto current = phys_addr; + while (current < end) { + auto block = m_physical.upper_bound(current); + if (block == m_physical.begin()) { + return false; + } + --block; + const auto block_end = block->second.start_addr + block->second.size; + if (block->second.pool_expansion || current < block->second.start_addr || + current >= block_end) { + return false; + } + current = std::min(end, block_end); + if (current < end) { + const auto following = std::next(block); + if (following == m_physical.end() || following->second.start_addr != current) { + return false; + } + } + } + return true; } bool PhysicalMemory::ReleasePoolExpansion(uint64_t phys_addr, size_t len) { @@ -1405,6 +1348,47 @@ bool PhysicalMemory::ReleasePoolExpansion(uint64_t phys_addr, size_t len) { return true; } +bool PhysicalMemory::GetAllocatedSpan(uint64_t phys_addr, size_t len, + std::vector* blocks) { + EXIT_IF(blocks == nullptr); + blocks->clear(); + if (len == 0 || UINT64_MAX - phys_addr < len) { + return false; + } + + Common::LockGuard lock(m_mutex); + const auto end = phys_addr + len; + auto current = phys_addr; + while (current < end) { + auto block = m_physical.upper_bound(current); + if (block == m_physical.begin()) { + blocks->clear(); + return false; + } + --block; + const auto block_end = block->second.start_addr + block->second.size; + if (block->second.pool_expansion || current < block->second.start_addr || + current >= block_end) { + blocks->clear(); + return false; + } + + auto part = block->second; + part.start_addr = current; + part.size = std::min(end, block_end) - current; + blocks->push_back(part); + current += part.size; + if (current < end) { + const auto following = std::next(block); + if (following == m_physical.end() || following->second.start_addr != current) { + blocks->clear(); + return false; + } + } + } + return true; +} + bool PhysicalMemory::Unmap(uint64_t vaddr, uint64_t size, GpuAccessMode* gpu_mode, uint64_t* host_vaddr_to_release) { #if defined(KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS) @@ -1487,6 +1471,51 @@ bool PhysicalMemory::Unmap(uint64_t vaddr, uint64_t size, GpuAccessMode* gpu_mod return false; } +void PhysicalMemory::ProtectMapping(uint64_t vaddr, uint64_t size, int prot, + VirtualMemory::Mode mode, GpuAccessMode gpu_mode) { + Common::LockGuard lock(m_mutex); + if (size == 0 || UINT64_MAX - vaddr < size) { + return; + } + + const auto end = vaddr + size; + std::vector updated; + updated.reserve(m_mappings.size() + 2); + for (const auto& block: m_mappings) { + const auto block_end = block.map_vaddr + block.map_size; + if (!VirtualRangesOverlap(vaddr, size, block.map_vaddr, block.map_size)) { + updated.push_back(block); + continue; + } + const auto overlap_start = std::max(vaddr, block.map_vaddr); + const auto overlap_end = std::min(end, block_end); + if (block.map_vaddr < overlap_start) { + auto left = block; + left.size = overlap_start - block.map_vaddr; + left.map_size = left.size; + updated.push_back(left); + } + auto middle = block; + middle.start_addr = block.start_addr + overlap_start - block.map_vaddr; + middle.size = overlap_end - overlap_start; + middle.map_vaddr = overlap_start; + middle.map_size = middle.size; + middle.prot = prot; + middle.mode = mode; + middle.gpu_mode = gpu_mode; + updated.push_back(middle); + if (overlap_end < block_end) { + auto right = block; + right.start_addr = block.start_addr + overlap_end - block.map_vaddr; + right.size = block_end - overlap_end; + right.map_vaddr = overlap_end; + right.map_size = right.size; + updated.push_back(right); + } + } + m_mappings = std::move(updated); +} + bool PhysicalMemory::Find(uint64_t phys_addr, bool next, AllocatedBlock* out) { EXIT_IF(out == nullptr); @@ -1587,17 +1616,61 @@ bool FlexibleMemory::Map(uint64_t vaddr, size_t len, int prot, VirtualMemory::Mo return false; } - AllocatedBlock b {}; - b.map_vaddr = vaddr; - b.map_size = len; - b.host_vaddr = vaddr; - b.host_size = len; - b.prot = prot; - b.mode = mode; - b.gpu_mode = gpu_mode; - CopyVirtualRangeName(b.name, name); + std::vector blocks; + auto current = vaddr; + auto remaining = static_cast(len); + for (const auto& [backing_offset, free_size]: m_free) { + if (remaining == 0) { + break; + } + const auto chunk = std::min(remaining, free_size); + AllocatedBlock block {}; + block.map_vaddr = current; + block.map_size = chunk; + block.backing_offset = backing_offset; + block.host_vaddr = vaddr; + block.host_size = len; + block.prot = prot; + block.mode = mode; + block.gpu_mode = gpu_mode; + CopyVirtualRangeName(block.name, name); + blocks.push_back(block); + current += chunk; + remaining -= chunk; + } + if (remaining != 0) { + return false; + } - m_allocated.push_back(b); + std::vector mapped; + for (const auto& block: blocks) { + if (!g_guest_address_space->ZeroBacking(block.backing_offset, block.map_size)) { + for (auto it = mapped.rbegin(); it != mapped.rend(); ++it) { + EXIT_IF(!g_guest_address_space->UnmapBacking(it->map_vaddr, it->map_size)); + } + return false; + } + if (!g_guest_address_space->MapBacking(block.map_vaddr, block.map_size, + block.backing_offset, block.mode)) { + for (auto it = mapped.rbegin(); it != mapped.rend(); ++it) { + EXIT_IF(!g_guest_address_space->UnmapBacking(it->map_vaddr, it->map_size)); + } + return false; + } + mapped.push_back(block); + } + + for (const auto& block: blocks) { + auto next = m_free.upper_bound(block.backing_offset); + EXIT_IF(next == m_free.begin()); + auto range = std::prev(next); + EXIT_IF(block.backing_offset < range->first || + block.map_size > range->first + range->second - block.backing_offset); + ConsumeFreeRange(range, block.backing_offset, block.map_size); + m_allocated.push_back(block); + } + std::sort(m_allocated.begin(), m_allocated.end(), + [](const auto& left, const auto& right) { return left.map_vaddr < right.map_vaddr; }); m_allocated_total += len; return true; @@ -1612,66 +1685,216 @@ bool FlexibleMemory::Unmap(uint64_t vaddr, uint64_t size, GpuAccessMode* gpu_mod if (host_vaddr_to_release != nullptr) { *host_vaddr_to_release = 0; } - - auto set_host_release_if_last = [this, host_vaddr_to_release](uint64_t host_vaddr, - uint64_t host_size) { - if (host_vaddr_to_release == nullptr || host_vaddr == 0 || host_size == 0) { - return; - } - const bool still_mapped = std::any_of( - m_allocated.begin(), m_allocated.end(), [host_vaddr, host_size](const auto& block) { - return block.host_vaddr == host_vaddr && block.host_size == host_size; - }); - if (!still_mapped) { - *host_vaddr_to_release = host_vaddr; - } - }; - - size_t index = 0; - for (auto& b: m_allocated) { - if (b.map_vaddr == vaddr && b.map_size == size) { - *gpu_mode = b.gpu_mode; - const auto host_vaddr = b.host_vaddr; - const auto host_size = b.host_size; - - m_allocated.erase(m_allocated.begin() + static_cast(index)); - m_allocated_total -= size; - set_host_release_if_last(host_vaddr, host_size); - return true; - } - if (vaddr > b.map_vaddr && vaddr + size < b.map_vaddr + b.map_size) { - *gpu_mode = b.gpu_mode; - - AllocatedBlock right = b; - right.map_size = b.map_vaddr + b.map_size - (vaddr + size); - right.map_vaddr = vaddr + size; - - b.map_size = vaddr - b.map_vaddr; - m_allocated.push_back(right); - m_allocated_total -= size; - return true; - } - if (vaddr == b.map_vaddr && size < b.map_size) { - *gpu_mode = b.gpu_mode; - - b.map_vaddr += size; - b.map_size -= size; - m_allocated_total -= size; - return true; - } - if (vaddr > b.map_vaddr && vaddr + size == b.map_vaddr + b.map_size) { - *gpu_mode = b.gpu_mode; - - b.map_size = vaddr - b.map_vaddr; - m_allocated_total -= size; - return true; - } - index++; + if (size == 0 || UINT64_MAX - vaddr < size) { + return false; } + const auto end = vaddr + size; + auto current = vaddr; + bool found = false; + for (const auto& block: m_allocated) { + if (block.map_vaddr + block.map_size <= current) { + continue; + } + if (block.map_vaddr > current || block.map_vaddr >= end) { + break; + } + if (!found) { + *gpu_mode = block.gpu_mode; + found = true; + } + current = std::min(end, block.map_vaddr + block.map_size); + if (current == end) { + break; + } + } + if (!found || current != end || !g_guest_address_space->UnmapBacking(vaddr, size)) { + return false; + } + + std::vector remaining; + remaining.reserve(m_allocated.size() + 1); + uint64_t removed = 0; + for (const auto& block: m_allocated) { + const auto block_end = block.map_vaddr + block.map_size; + if (!VirtualRangesOverlap(vaddr, size, block.map_vaddr, block.map_size)) { + remaining.push_back(block); + continue; + } + const auto overlap_start = std::max(vaddr, block.map_vaddr); + const auto overlap_end = std::min(end, block_end); + const auto overlap_size = overlap_end - overlap_start; + AddFreeRange(block.backing_offset + overlap_start - block.map_vaddr, overlap_size); + removed += overlap_size; + + if (block.map_vaddr < overlap_start) { + auto left = block; + left.map_size = overlap_start - block.map_vaddr; + remaining.push_back(left); + } + if (overlap_end < block_end) { + auto right = block; + right.map_vaddr = overlap_end; + right.map_size = block_end - overlap_end; + right.backing_offset = block.backing_offset + overlap_end - block.map_vaddr; + remaining.push_back(right); + } + } + EXIT_IF(removed != size || removed > m_allocated_total); + m_allocated = std::move(remaining); + m_allocated_total -= removed; + return true; +} + +void FlexibleMemory::ConsumeFreeRange(std::map::iterator range, uint64_t start, + uint64_t size) { + const auto range_start = range->first; + const auto range_end = range->first + range->second; + m_free.erase(range); + if (range_start < start) { + m_free.emplace(range_start, start - range_start); + } + if (start + size < range_end) { + m_free.emplace(start + size, range_end - start - size); + } +} + +void FlexibleMemory::AddFreeRange(uint64_t start, uint64_t size) { + auto end = start + size; + auto next = m_free.lower_bound(start); + if (next != m_free.begin()) { + auto previous = std::prev(next); + if (previous->first + previous->second >= start) { + start = previous->first; + end = std::max(end, previous->first + previous->second); + next = m_free.erase(previous); + } + } + while (next != m_free.end() && next->first <= end) { + end = std::max(end, next->first + next->second); + next = m_free.erase(next); + } + m_free.emplace(start, end - start); +} + +bool FlexibleMemory::Snapshot(uint64_t vaddr, uint64_t size, std::vector* blocks) { + EXIT_IF(blocks == nullptr); + Common::LockGuard lock(m_mutex); + blocks->clear(); + if (size == 0 || UINT64_MAX - vaddr < size) { + return false; + } + + const auto end = vaddr + size; + auto current = vaddr; + for (const auto& block: m_allocated) { + const auto block_end = block.map_vaddr + block.map_size; + if (block_end <= current) { + continue; + } + if (block.map_vaddr > current || block.map_vaddr >= end) { + break; + } + const auto part_end = std::min(end, block_end); + auto part = block; + part.map_vaddr = current; + part.map_size = part_end - current; + part.backing_offset += current - block.map_vaddr; + blocks->push_back(part); + current = part_end; + if (current == end) { + return true; + } + } + blocks->clear(); return false; } +bool FlexibleMemory::Restore(const std::vector& blocks) { + Common::LockGuard lock(m_mutex); + if (blocks.empty()) { + return false; + } + + for (const auto& block: blocks) { + auto next = m_free.upper_bound(block.backing_offset); + if (next == m_free.begin()) { + return false; + } + const auto range = std::prev(next); + if (block.backing_offset < range->first || + block.map_size > range->first + range->second - block.backing_offset) { + return false; + } + } + + std::vector mapped; + for (const auto& block: blocks) { + if (!g_guest_address_space->MapBacking(block.map_vaddr, block.map_size, + block.backing_offset, block.mode)) { + for (auto it = mapped.rbegin(); it != mapped.rend(); ++it) { + EXIT_IF(!g_guest_address_space->UnmapBacking(it->map_vaddr, it->map_size)); + } + return false; + } + mapped.push_back(block); + } + + for (const auto& block: blocks) { + auto next = m_free.upper_bound(block.backing_offset); + EXIT_IF(next == m_free.begin()); + auto range = std::prev(next); + ConsumeFreeRange(range, block.backing_offset, block.map_size); + m_allocated.push_back(block); + m_allocated_total += block.map_size; + } + std::sort(m_allocated.begin(), m_allocated.end(), + [](const auto& left, const auto& right) { return left.map_vaddr < right.map_vaddr; }); + return true; +} + +void FlexibleMemory::Protect(uint64_t vaddr, uint64_t size, int prot, VirtualMemory::Mode mode, + GpuAccessMode gpu_mode) { + Common::LockGuard lock(m_mutex); + if (size == 0 || UINT64_MAX - vaddr < size) { + return; + } + + const auto end = vaddr + size; + std::vector updated; + updated.reserve(m_allocated.size() + 2); + for (const auto& block: m_allocated) { + const auto block_end = block.map_vaddr + block.map_size; + if (!VirtualRangesOverlap(vaddr, size, block.map_vaddr, block.map_size)) { + updated.push_back(block); + continue; + } + const auto overlap_start = std::max(vaddr, block.map_vaddr); + const auto overlap_end = std::min(end, block_end); + if (block.map_vaddr < overlap_start) { + auto left = block; + left.map_size = overlap_start - block.map_vaddr; + updated.push_back(left); + } + auto middle = block; + middle.map_vaddr = overlap_start; + middle.map_size = overlap_end - overlap_start; + middle.backing_offset = block.backing_offset + overlap_start - block.map_vaddr; + middle.prot = prot; + middle.mode = mode; + middle.gpu_mode = gpu_mode; + updated.push_back(middle); + if (overlap_end < block_end) { + auto right = block; + right.map_vaddr = overlap_end; + right.map_size = block_end - overlap_end; + right.backing_offset = block.backing_offset + overlap_end - block.map_vaddr; + updated.push_back(right); + } + } + m_allocated = std::move(updated); +} + bool FlexibleMemory::Find(uint64_t vaddr, uint64_t* base_addr, size_t* len, int* prot, VirtualMemory::Mode* mode, GpuAccessMode* gpu_mode) { Common::LockGuard lock(m_mutex); @@ -1920,42 +2143,21 @@ std::vector PooledMemory::GetMappings() { static bool UnmapPooledBackingTransactional(const std::vector& mappings, VirtualMemory::Mode mode) { - struct RemovedMapping { - PooledMemory::Mapping mapping; - bool placeholder_preserved; - }; - - std::vector removed; + std::vector removed; for (const auto& mapping: mappings) { - bool placeholder_preserved = false; - if (!g_direct_memory_backing->Unmap(mapping.vaddr, mapping.size, true, - &placeholder_preserved)) { + if (!g_guest_address_space->UnmapBacking(mapping.vaddr, mapping.size)) { for (auto it = removed.rbegin(); it != removed.rend(); ++it) { - auto failure_reason = DirectMemoryBacking::FailureReason::None; - const bool restored = it->placeholder_preserved - ? g_direct_memory_backing->MapExistingPlaceholderFixed( - it->mapping.vaddr, it->mapping.size, - it->mapping.phys_addr, mode, &failure_reason) - : g_direct_memory_backing->MapFixed( - it->mapping.vaddr, it->mapping.size, - it->mapping.phys_addr, mode, &failure_reason); + auto failure_reason = GuestBackingStore::FailureReason::None; + const bool restored = g_guest_address_space->MapBacking( + it->vaddr, it->size, it->phys_addr, mode, &failure_reason); if (!restored) { EXIT("pooled-memory unmap rollback failed: %s\n", - DirectMemoryBacking::GetFailureReasonName(failure_reason)); + GuestBackingStore::GetFailureReasonName(failure_reason)); } } return false; } - removed.push_back({mapping, placeholder_preserved}); - } - - for (const auto& entry: removed) { - if (entry.placeholder_preserved) { - g_placeholder_address_space->AddFree(entry.mapping.vaddr, entry.mapping.size); - } else if (!g_placeholder_address_space->ReserveFixed(entry.mapping.vaddr, - entry.mapping.size)) { - EXIT("failed to reserve pooled-memory placeholder\n"); - } + removed.push_back(mapping); } return true; } @@ -1968,28 +2170,25 @@ int32_t KYTY_SYSV_ABI KernelMapNamedFlexibleMemory(void** addr_in_out, size_t le EXIT_NOT_IMPLEMENTED(addr_in_out == nullptr); - constexpr size_t PAGE_SIZE = 0x4000; - constexpr size_t MAXIMUM_NAME_SIZE = 32; - constexpr uint64_t DEFAULT_PS5_BASE = 0x200000000; - constexpr int GUEST_MAP_FIXED = 0x10; - constexpr int GUEST_MAP_SHARED = 0x01; - constexpr int GUEST_MAP_PRIVATE = 0x02; - constexpr int GUEST_MAP_NO_OVERWRITE = 0x80; - constexpr int GUEST_MAP_VOID = 0x100; - constexpr int GUEST_MAP_STACK = 0x400; - constexpr int GUEST_MAP_NO_SYNC = 0x800; - constexpr int GUEST_MAP_ANON = 0x1000; - constexpr int GUEST_MAP_UNKNOWN_8000 = 0x8000; - constexpr int GUEST_MAP_NO_CORE = 0x20000; - constexpr int GUEST_MAP_NO_COALESCE = 0x400000; - constexpr int SUPPORTED_MAP_BITS = GUEST_MAP_SHARED | GUEST_MAP_PRIVATE | GUEST_MAP_FIXED | - GUEST_MAP_NO_OVERWRITE | GUEST_MAP_VOID | GUEST_MAP_STACK | - GUEST_MAP_NO_SYNC | GUEST_MAP_ANON | GUEST_MAP_UNKNOWN_8000 | - GUEST_MAP_NO_CORE | GUEST_MAP_NO_COALESCE; + constexpr size_t PAGE_SIZE = 0x4000; + constexpr size_t MAXIMUM_NAME_SIZE = 32; + constexpr uint64_t DEFAULT_PS5_BASE = 0x200000000; + constexpr uint32_t GUEST_MAP_FIXED = 0x10; + constexpr uint32_t GUEST_MAP_NO_OVERWRITE = 0x80; + constexpr uint32_t GUEST_MAP_DMEM_COMPAT = 0x400; + constexpr uint32_t GUEST_MAP_UNKNOWN_8000 = 0x8000; + constexpr uint32_t GUEST_MAP_NO_COALESCE = 0x400000; + constexpr uint32_t GUEST_MAP_ALIGNMENT_MASK = 0xff000000; + constexpr uint32_t SUPPORTED_MAP_BITS = GUEST_MAP_FIXED | GUEST_MAP_NO_OVERWRITE | + GUEST_MAP_DMEM_COMPAT | GUEST_MAP_UNKNOWN_8000 | + GUEST_MAP_NO_COALESCE | GUEST_MAP_ALIGNMENT_MASK; if (len == 0 || (len & (PAGE_SIZE - 1)) != 0) { return KERNEL_ERROR_EINVAL; } + if (len > g_flexible_memory->Available()) { + return KERNEL_ERROR_ENOMEM; + } if (name == nullptr) { return KERNEL_ERROR_EFAULT; @@ -1999,11 +2198,15 @@ int32_t KYTY_SYSV_ABI KernelMapNamedFlexibleMemory(void** addr_in_out, size_t le return KERNEL_ERROR_ENAMETOOLONG; } - if ((flags & ~SUPPORTED_MAP_BITS) != 0) { - LOGF_COLOR(Log::Color::Red, "\t unsupported flags = 0x%08" PRIx32 "\n", - static_cast(flags & ~SUPPORTED_MAP_BITS)); + const auto map_flags = static_cast(flags); + const auto alignment_shift = (map_flags & GUEST_MAP_ALIGNMENT_MASK) >> 24u; + if ((map_flags & ~SUPPORTED_MAP_BITS) != 0 || + (alignment_shift != 0 && (alignment_shift < 14 || alignment_shift > 31))) { + LOGF_COLOR(Log::Color::Red, "\t unsupported flags = 0x%08" PRIx32 "\n", map_flags); return KERNEL_ERROR_EINVAL; } + const uint64_t map_alignment = + alignment_shift != 0 ? uint64_t {1} << alignment_shift : PAGE_SIZE; VirtualMemory::Mode mode = VirtualMemory::Mode::NoAccess; GpuAccessMode gpu_mode = GpuAccessMode::NoAccess; @@ -2012,72 +2215,70 @@ int32_t KYTY_SYSV_ABI KernelMapNamedFlexibleMemory(void** addr_in_out, size_t le EXIT("unknown prot: %d\n", prot); } - auto in_addr = reinterpret_cast(*addr_in_out); - uint64_t out_addr = 0; - bool committed_from_reserved = false; - bool consumed_reserved = false; + auto in_addr = reinterpret_cast(*addr_in_out); + uint64_t out_addr = 0; + bool consumed_reservation = false; VirtualRanges::Range consumed_range {}; if ((flags & GUEST_MAP_FIXED) != 0) { - if (in_addr == 0 || (in_addr & (PAGE_SIZE - 1)) != 0) { + if (in_addr == 0 || (in_addr & (PAGE_SIZE - 1)) != 0 || + (in_addr & (map_alignment - 1u)) != 0) { return KERNEL_ERROR_EINVAL; } if ((flags & GUEST_MAP_NO_OVERWRITE) != 0 && g_virtual_ranges->HasOverlap(in_addr, len)) { return KERNEL_ERROR_ENOMEM; } - if (g_virtual_ranges->Query(in_addr, 0, &consumed_range) && - consumed_range.type == VirtualRangeType::Reserved && - g_virtual_ranges->ConsumeReserved(in_addr, len)) { - consumed_reserved = true; - if (g_placeholder_address_space->Commit(in_addr, len, mode) || - CommitFixedHostRange(in_addr, len, mode)) { - out_addr = in_addr; - committed_from_reserved = true; + std::vector reserved_ranges; + if (g_virtual_ranges->QuerySpan(in_addr, len, &reserved_ranges) && + std::all_of(reserved_ranges.begin(), reserved_ranges.end(), [](const auto& range) { + return range.type == VirtualRangeType::Reserved; + })) { + UnmapGpuRange(in_addr, len); + consumed_range = reserved_ranges.front(); + if (g_virtual_ranges->ConsumeReservedSpan(in_addr, len)) { + consumed_reservation = true; + out_addr = in_addr; } - } else if (!ReleaseReservedRange(in_addr, len)) { - return KERNEL_ERROR_ENOMEM; - } else if (VirtualMemory::AllocFixed(in_addr, len, mode)) { - out_addr = in_addr; + } + if (!consumed_reservation && ReplaceFixedRangeWithReserved(in_addr, len) && + g_virtual_ranges->ConsumeReservedSpan(in_addr, len, &consumed_range)) { + consumed_reservation = true; + out_addr = in_addr; } } else { const auto search_addr = (in_addr != 0 ? in_addr : DEFAULT_PS5_BASE); - out_addr = VirtualMemory::AllocAligned(search_addr, len, mode, PAGE_SIZE); + out_addr = FindGuestFreeRange(search_addr, len, map_alignment); + if (out_addr != 0) { + UnmapGpuRange(out_addr, len); + } } *addr_in_out = reinterpret_cast(out_addr); if (out_addr == 0) { - if (consumed_reserved) { + if (consumed_reservation) { g_virtual_ranges->Add(in_addr, len, 0, 0, 0, VirtualRangeType::Reserved, - consumed_range.name, false, consumed_range.placeholder_backed); + consumed_range.name); } return KERNEL_ERROR_ENOMEM; } if (!g_flexible_memory->Map(out_addr, len, prot, mode, gpu_mode, name)) { LOGF_COLOR(Log::Color::Red, "\t [Fail]\n"); - if (committed_from_reserved) { - const bool placeholder_backed = RestoreCommittedPlaceholderOrProtect(out_addr, len); - g_virtual_ranges->Add(out_addr, len, 0, 0, 0, VirtualRangeType::Reserved, name, false, - placeholder_backed); - } else { - VirtualMemory::Free(out_addr); + if (consumed_reservation) { + EXIT_IF(!g_virtual_ranges->Add(out_addr, len, 0, 0, 0, VirtualRangeType::Reserved, + consumed_range.name)); } return KERNEL_ERROR_ENOMEM; } - const auto range_type = - ((flags & GUEST_MAP_STACK) != 0 ? VirtualRangeType::Stack : VirtualRangeType::Flexible); - if (!g_virtual_ranges->Add(out_addr, len, 0, prot, 0, range_type, name, - committed_from_reserved)) { + if (!g_virtual_ranges->Add(out_addr, len, 0, prot, 0, VirtualRangeType::Flexible, name, + (map_flags & GUEST_MAP_NO_COALESCE) != 0)) { GpuAccessMode rollback_gpu_mode = GpuAccessMode::NoAccess; - g_flexible_memory->Unmap(out_addr, len, &rollback_gpu_mode); - if (committed_from_reserved) { - const bool placeholder_backed = RestoreCommittedPlaceholderOrProtect(out_addr, len); - g_virtual_ranges->Add(out_addr, len, 0, 0, 0, VirtualRangeType::Reserved, name, false, - placeholder_backed); - } else { - VirtualMemory::Free(out_addr); + EXIT_IF(!g_flexible_memory->Unmap(out_addr, len, &rollback_gpu_mode)); + if (consumed_reservation) { + EXIT_IF(!g_virtual_ranges->Add(out_addr, len, 0, 0, 0, VirtualRangeType::Reserved, + consumed_range.name)); } return KERNEL_ERROR_EBUSY; } @@ -2092,7 +2293,7 @@ int32_t KYTY_SYSV_ABI KernelMapNamedFlexibleMemory(void** addr_in_out, size_t le in_addr, out_addr, len, Common::EnumName(mode).c_str(), static_cast(flags), name, Common::EnumName(gpu_mode).c_str()); - MapGpuRange(out_addr, len, gpu_mode); + MapGpuRange(out_addr, len); if (g_alloc_callback != nullptr) { g_alloc_callback(out_addr, len); @@ -2108,6 +2309,8 @@ int KYTY_SYSV_ABI KernelMapFlexibleMemory(void** addr_in_out, size_t len, int pr int KYTY_SYSV_ABI KernelSetPrtAperture(int index, void* addr, size_t len) { PRINT_NAME(); + std::lock_guard memory_operation_lock(g_memory_operation_mutex); + const auto address = reinterpret_cast(addr); LOGF("\t index = %d\n" @@ -2119,34 +2322,27 @@ int KYTY_SYSV_ABI KernelSetPrtAperture(int index, void* addr, size_t len) { return KERNEL_ERROR_EINVAL; } - if (len == 0) { - Common::LockGuard lock(g_prt_aperture_mutex); - const auto old = g_prt_apertures[static_cast(index)]; - if (old.size != 0) { - UnmapGpuRange(old.address, old.size, GpuAccessMode::ReadWrite); - } - g_prt_apertures[static_cast(index)] = {}; - LOGF_COLOR(Log::Color::Green, "\t[Ok]\n"); - return OK; - } - - if (address == 0 || (address & (PRT_PAGE_SIZE - 1u)) != 0 || - (len & (PRT_PAGE_SIZE - 1u)) != 0) { - return KERNEL_ERROR_EINVAL; - } - - if (address < PRT_APERTURE_START || len > PRT_APERTURE_END - address) { + if (len != 0 && (address == 0 || (address & (PRT_PAGE_SIZE - 1u)) != 0 || + (len & (PRT_PAGE_SIZE - 1u)) != 0 || address < PRT_APERTURE_START || + len > PRT_APERTURE_END - address)) { return KERNEL_ERROR_EINVAL; } + PrtAperture old {}; { Common::LockGuard lock(g_prt_aperture_mutex); - const auto old = g_prt_apertures[static_cast(index)]; - if (old.size != 0) { - UnmapGpuRange(old.address, old.size, GpuAccessMode::ReadWrite); - } - MapGpuRange(address, len, GpuAccessMode::ReadWrite); - g_prt_apertures[static_cast(index)] = {address, static_cast(len)}; + old = g_prt_apertures[static_cast(index)]; + } + if (old.size != 0) { + UnmapGpuRange(old.address, old.size); + } + { + Common::LockGuard lock(g_prt_aperture_mutex); + g_prt_apertures[static_cast(index)] = + len == 0 ? PrtAperture {} : PrtAperture {address, static_cast(len)}; + } + if (len != 0) { + MapGpuRange(address, len); } LOGF_COLOR(Log::Color::Green, "\t[Ok]\n"); @@ -2215,6 +2411,88 @@ int KYTY_SYSV_ABI KernelSetVirtualRangeName(const void* addr, uint64_t len, cons return OK; } +static bool FreeGuestMemoryOwner(uint64_t vaddr, uint64_t size) { + return g_guest_address_space->ReleaseCommitted(vaddr, size) && + g_virtual_ranges->Remove(vaddr, size); +} + +static int UnmapMemoryRange(uint64_t vaddr, size_t len) { + if (len == 0 || UINT64_MAX - vaddr < len) { + return KERNEL_ERROR_EINVAL; + } + + VirtualRanges::Range range {}; + if (!g_virtual_ranges->Query(vaddr, 0, &range)) { + return KERNEL_ERROR_EACCES; + } + const auto chunk_len = std::min(len, range.size - (vaddr - range.start)); + if (chunk_len < len) { + const int ret = UnmapMemoryRange(vaddr, chunk_len); + return ret == OK ? UnmapMemoryRange(vaddr + chunk_len, len - chunk_len) : ret; + } + if (IsReservedRangeType(range.type)) { + if (!g_guest_address_space->ReleaseFree(vaddr, len)) { + return KERNEL_ERROR_EACCES; + } + g_virtual_ranges->Remove(vaddr, len); + return OK; + } + if (range.type == VirtualRangeType::Code || range.type == VirtualRangeType::Runtime) { + return FreeGuestMemoryOwner(vaddr, len) ? OK : KERNEL_ERROR_EACCES; + } + + GpuAccessMode gpu_mode = GpuAccessMode::NoAccess; + bool owner_unmapped = false; + if (range.type == VirtualRangeType::Pooled) { + std::vector mappings; + if (g_pooled_memory->Query(vaddr, len, &mappings)) { + VirtualMemory::Mode mode = VirtualMemory::Mode::NoAccess; + GpuAccessMode decoded_gpu = GpuAccessMode::NoAccess; + owner_unmapped = DecodeMemoryProtection(range.protection, &mode, &decoded_gpu) && + UnmapPooledBackingTransactional(mappings, mode); + if (owner_unmapped && !g_pooled_memory->Release(vaddr, len, &gpu_mode)) { + EXIT("failed to release unmapped pooled-memory range\n"); + } + } + if (!owner_unmapped) { + return KERNEL_ERROR_EACCES; + } + } else if (range.type == VirtualRangeType::Direct) { + VirtualMemory::Mode direct_mode = VirtualMemory::Mode::NoAccess; + GpuAccessMode direct_gpu = GpuAccessMode::NoAccess; + const auto direct_offset = range.offset + vaddr - range.start; + if (DecodeMemoryProtection(range.protection, &direct_mode, &direct_gpu) && + g_guest_address_space->BackingContains(vaddr, len) && + g_guest_address_space->UnmapBacking(vaddr, len)) { + uint64_t ignored_host = 0; + if (g_physical_memory->Unmap(vaddr, len, &gpu_mode, &ignored_host)) { + owner_unmapped = true; + } else { + EXIT_IF(!g_guest_address_space->MapBacking(vaddr, len, direct_offset, direct_mode)); + } + } + } else if (range.type == VirtualRangeType::Stack) { + owner_unmapped = g_guest_address_space->ReleaseCommitted(vaddr, len); + } else { + uint64_t ignored_host = 0; + owner_unmapped = g_flexible_memory->Unmap(vaddr, len, &gpu_mode, &ignored_host); + } + if (!owner_unmapped) { + return KERNEL_ERROR_EACCES; + } + + g_virtual_ranges->Remove(vaddr, len); + + if (g_free_callback != nullptr && IsCommittedRangeType(range.type)) { + g_free_callback(vaddr, len); + } + if (range.type == VirtualRangeType::Pooled) { + MemoryPoolSubtractCommitted(len); + } + + return OK; +} + int KYTY_SYSV_ABI KernelMunmap(uint64_t vaddr, size_t len) { PRINT_NAME(); @@ -2227,134 +2505,12 @@ int KYTY_SYSV_ABI KernelMunmap(uint64_t vaddr, size_t len) { if (len == 0 || UINT64_MAX - vaddr < len) { return KERNEL_ERROR_EINVAL; } - - VirtualRanges::Range range {}; - if (!g_virtual_ranges->Query(vaddr, 0, &range)) { + std::vector ranges; + if (!g_virtual_ranges->QuerySpan(vaddr, len, &ranges)) { return KERNEL_ERROR_EACCES; } - const auto chunk_len = std::min(len, range.size - (vaddr - range.start)); - if (chunk_len < len) { - const int ret = KernelMunmap(vaddr, chunk_len); - return ret == OK ? KernelMunmap(vaddr + chunk_len, len - chunk_len) : ret; - } - if (IsReservedRangeType(range.type)) { - const bool released = range.placeholder_backed - ? g_placeholder_address_space->ReleaseFree(vaddr, len) - : VirtualMemory::FreeRange(vaddr, len); - if (!released) { - return KERNEL_ERROR_EACCES; - } - g_virtual_ranges->Remove(vaddr, len); - return OK; - } - UnmapGpuRange(vaddr, len, GetGpuAccessMode(range.protection)); - - size_t backend_len = 0; - uint64_t backend_base = 0; - GpuAccessMode gpu_mode = GpuAccessMode::NoAccess; - const bool pooled_backend = range.type == VirtualRangeType::Pooled; - bool backend_found = false; - bool exact_host_range = backend_found && backend_base == vaddr && backend_len == len; - bool backend_unmapped = false; - bool shared_unmapped = false; - bool placeholder_restored = false; - bool flexible_backend = false; - uint64_t physical_host_to_release = 0; - uint64_t flexible_host_to_release = 0; - const bool can_restore_placeholder = - range.committed_from_reserved && range.start == vaddr && range.size == len; - - if (pooled_backend) { - std::vector mappings; - backend_found = g_pooled_memory->Query(vaddr, len, &mappings); - exact_host_range = backend_found; - if (backend_found) { - VirtualMemory::Mode mode = VirtualMemory::Mode::NoAccess; - GpuAccessMode decoded_gpu = GpuAccessMode::NoAccess; - shared_unmapped = DecodeMemoryProtection(range.protection, &mode, &decoded_gpu) && - UnmapPooledBackingTransactional(mappings, mode); - if (shared_unmapped && !g_pooled_memory->Release(vaddr, len, &gpu_mode)) { - EXIT("failed to release unmapped pooled-memory range\n"); - } - backend_unmapped = shared_unmapped; - placeholder_restored = shared_unmapped; - } - if (!backend_unmapped) { - EXIT("pooled-memory unmap failed: addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", vaddr, - len); - return KERNEL_ERROR_EACCES; - } - } else if ((backend_found = g_physical_memory->Find(vaddr, &backend_base, &backend_len, nullptr, - nullptr, &gpu_mode))) { - exact_host_range = backend_base == vaddr && backend_len == len; - backend_unmapped = - g_physical_memory->Unmap(vaddr, len, &gpu_mode, &physical_host_to_release); - if (backend_unmapped) { - const bool direct_contains = g_direct_memory_backing->Contains(vaddr, len); - if (direct_contains) { - shared_unmapped = g_direct_memory_backing->Unmap( - vaddr, len, can_restore_placeholder, &placeholder_restored); - } - if (placeholder_restored) { - g_placeholder_address_space->AddFree(vaddr, len); - } - } - } else { - flexible_backend = true; - backend_found = g_flexible_memory->Find(vaddr, &backend_base, &backend_len, nullptr, - nullptr, &gpu_mode); - exact_host_range = backend_found && backend_base == vaddr && backend_len == len; - backend_unmapped = (backend_found ? g_flexible_memory->Unmap(vaddr, len, &gpu_mode, - &flexible_host_to_release) - : false); - if (backend_unmapped && can_restore_placeholder) { - placeholder_restored = g_placeholder_address_space->ReleaseCommitted(vaddr, len); - } - } - if (!backend_unmapped) { - EXIT("memory backend unmap failed: addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", vaddr, - len); - } - - g_virtual_ranges->Remove(vaddr, len); - - const bool host_range_released_or_shared = - flexible_backend ? flexible_host_to_release != 0 - : (shared_unmapped || physical_host_to_release != 0); - const bool can_release_host_range = exact_host_range && backend_unmapped && - !range.committed_from_reserved && - host_range_released_or_shared; - if (can_release_host_range && (vaddr != 0 || len != 0)) { - if (!shared_unmapped) { - VirtualMemory::Free(flexible_backend ? flexible_host_to_release - : physical_host_to_release); - } - } else if (IsCommittedRangeType(range.type)) { - constexpr uint64_t PAGE_SIZE = 0x4000; - auto aligned_addr = vaddr & ~(PAGE_SIZE - 1); - auto aligned_len = (len + (vaddr - aligned_addr) + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1); - if (placeholder_restored) { - // Exact placeholder-backed unmaps already returned this range to the placeholder - // manager. - } else if (!shared_unmapped) { - VirtualMemory::Protect(aligned_addr, aligned_len, VirtualMemory::Mode::NoAccess); - } else if (range.committed_from_reserved) { - VirtualMemory::ReserveFixed(aligned_addr, aligned_len); - } - if (range.committed_from_reserved) { - g_virtual_ranges->Add(vaddr, len, 0, 0, 0, VirtualRangeType::Reserved, range.name, - false, placeholder_restored); - } - } - - if (g_free_callback != nullptr && IsCommittedRangeType(range.type)) { - g_free_callback(vaddr, len); - } - if (pooled_backend) { - MemoryPoolSubtractCommitted(len); - } - - return OK; + UnmapGpuRange(vaddr, len); + return UnmapMemoryRange(vaddr, len); } size_t KYTY_SYSV_ABI KernelGetDirectMemorySize() { @@ -2536,7 +2692,10 @@ int KYTY_SYSV_ABI KernelAllocateDirectMemory(int64_t search_start, int64_t searc "\t memory_type = %d\n", search_start, search_end, len, alignment, memory_type); - if (search_start < 0 || search_end <= search_start || len == 0 || phys_addr_out == nullptr) { + constexpr uint64_t PAGE_SIZE = 0x4000; + if (search_start < 0 || search_end <= search_start || len == 0 || + (len & (PAGE_SIZE - 1u)) != 0 || (alignment != 0 && (alignment & (PAGE_SIZE - 1u)) != 0) || + phys_addr_out == nullptr) { return KERNEL_ERROR_EINVAL; } @@ -2564,35 +2723,13 @@ int KYTY_SYSV_ABI KernelAllocateMainDirectMemory(size_t len, size_t alignment, i "\t memory_type = %d\n", len, alignment, memory_type); - if (len == 0 || phys_addr_out == nullptr) { - return KERNEL_ERROR_EINVAL; - } - - uint64_t addr = 0; - if (!g_physical_memory->Alloc(0, PhysicalMemory::Size(), len, alignment, &addr, memory_type)) { - LOGF_COLOR(Log::Color::Red, "\t[Fail]\n"); - return KERNEL_ERROR_EAGAIN; - } - - *phys_addr_out = static_cast(addr); - - LOGF_COLOR(Log::Color::Green, "\tphys_addr = %016" PRIx64 "\n\t[Ok]\n", addr); - - return OK; + return KernelAllocateDirectMemory(0, static_cast(PhysicalMemory::Size()), len, + alignment, memory_type, phys_addr_out); } -int KYTY_SYSV_ABI KernelReleaseDirectMemory(int64_t start, size_t len) { - PRINT_NAME(); - +static int ReleaseDirectMemoryInternal(int64_t start, size_t len) { std::lock_guard memory_operation_lock(g_memory_operation_mutex); - LOGF("\t start = 0x%016" PRIx64 "\n" - "\t len = 0x%016" PRIx64 "\n", - start, len); - - if (start < 0 || len == 0) { - return KERNEL_ERROR_EINVAL; - } if (g_pooled_memory->ReleaseExpansion(static_cast(start), len)) { if (!g_physical_memory->ReleasePoolExpansion(static_cast(start), len)) { EXIT("failed to release physical pool expansion\n"); @@ -2600,111 +2737,110 @@ int KYTY_SYSV_ABI KernelReleaseDirectMemory(int64_t start, size_t len) { return OK; } - uint64_t vaddr = 0; - uint64_t size = 0; - GpuAccessMode gpu_mode = GpuAccessMode::NoAccess; - - PhysicalMemory::AllocatedBlock block {}; - bool found = g_physical_memory->Find(start, false, &block); - bool exact_host_range = - found && block.start_addr == static_cast(start) && block.size == len; - VirtualRanges::Range range {}; - const bool range_found = (found ? g_virtual_ranges->Query(block.map_vaddr, 0, &range) : false); - bool shared_unmapped = false; - bool placeholder_restored = false; - const auto mapped_aliases = g_physical_memory->FindMappings(start, len); - for (const auto& alias: mapped_aliases) { - UnmapGpuRange(alias.map_vaddr, alias.map_size, alias.gpu_mode); - } - - bool result = g_physical_memory->Release(start, len, &vaddr, &size, &gpu_mode); - - if (!result) { - if (!mapped_aliases.empty()) { - EXIT("physical-memory release failed after GPU alias unmap: addr=0x%016" PRIx64 - " size=0x%016" PRIx64 " aliases=%zu\n", - start, len, mapped_aliases.size()); - } + std::vector allocated_span; + if (!g_physical_memory->GetAllocatedSpan(static_cast(start), len, &allocated_span)) { return KERNEL_ERROR_EACCES; } + const auto mapped_aliases = g_physical_memory->FindMappings(start, len); for (const auto& alias: mapped_aliases) { - GpuAccessMode alias_gpu_mode = GpuAccessMode::NoAccess; - uint64_t alias_host_to_release = 0; - if (!g_physical_memory->Unmap(alias.map_vaddr, alias.map_size, &alias_gpu_mode, - &alias_host_to_release)) { - EXIT("physical-memory alias unmap failed: addr=0x%016" PRIx64 " size=0x%016" PRIx64 - "\n", - alias.map_vaddr, alias.map_size); + VirtualRanges::Range range {}; + if (!g_guest_address_space->BackingContains(alias.map_vaddr, alias.map_size) || + !g_virtual_ranges->Query(alias.map_vaddr, 0, &range) || + range.type != VirtualRangeType::Direct || + alias.map_size > range.start + range.size - alias.map_vaddr) { + EXIT("direct-memory alias escaped guest address-space ownership\n"); } + } - VirtualRanges::Range alias_range {}; - const bool alias_range_found = g_virtual_ranges->Query(alias.map_vaddr, 0, &alias_range); - g_virtual_ranges->Remove(alias.map_vaddr, alias.map_size); + for (const auto& alias: mapped_aliases) { + UnmapGpuRange(alias.map_vaddr, alias.map_size); + } - bool alias_shared_unmapped = false; - bool alias_placeholder_restored = false; - if (g_direct_memory_backing->Contains(alias.map_vaddr, alias.map_size)) { - const bool can_restore_placeholder = - alias_range_found && alias_range.committed_from_reserved && - alias_range.start == alias.map_vaddr && alias_range.size == alias.map_size; - alias_shared_unmapped = g_direct_memory_backing->Unmap(alias.map_vaddr, alias.map_size, - can_restore_placeholder, - &alias_placeholder_restored); - if (alias_placeholder_restored) { - g_placeholder_address_space->AddFree(alias.map_vaddr, alias.map_size); + auto restore_gpu_aliases = [&mapped_aliases]() { + for (const auto& alias: mapped_aliases) { + MapGpuRange(alias.map_vaddr, alias.map_size); + } + }; + auto restore_owner_aliases = [](const std::vector& aliases) { + for (auto it = aliases.rbegin(); it != aliases.rend(); ++it) { + EXIT_IF(!g_guest_address_space->MapBacking(it->map_vaddr, it->map_size, it->start_addr, + it->mode)); + } + }; + + std::vector owner_unmapped; + for (const auto& alias: mapped_aliases) { + if (!g_guest_address_space->UnmapBacking(alias.map_vaddr, alias.map_size)) { + restore_owner_aliases(owner_unmapped); + restore_gpu_aliases(); + return KERNEL_ERROR_EACCES; + } + owner_unmapped.push_back(alias); + } + + std::vector metadata_unmapped; + for (const auto& alias: mapped_aliases) { + GpuAccessMode alias_gpu_mode = GpuAccessMode::NoAccess; + if (!g_physical_memory->Unmap(alias.map_vaddr, alias.map_size, &alias_gpu_mode)) { + for (const auto& removed: metadata_unmapped) { + EXIT_IF(!g_physical_memory->Map(removed.map_vaddr, removed.start_addr, + removed.map_size, removed.prot, removed.mode, + removed.gpu_mode)); } + restore_owner_aliases(owner_unmapped); + restore_gpu_aliases(); + return KERNEL_ERROR_EACCES; } - - if (alias_range_found && alias_range.committed_from_reserved) { - g_virtual_ranges->Add(alias.map_vaddr, alias.map_size, 0, 0, 0, - VirtualRangeType::Reserved, alias_range.name, false, - alias_placeholder_restored); - } else if (!alias_shared_unmapped && alias_host_to_release != 0) { - VirtualMemory::Free(alias_host_to_release); - } else if (!alias_shared_unmapped) { - VirtualMemory::Protect(alias.map_vaddr, alias.map_size, VirtualMemory::Mode::NoAccess); - } + metadata_unmapped.push_back(alias); } - if (vaddr != 0 || size != 0) { - g_virtual_ranges->Remove(vaddr, size); - const bool can_restore_placeholder = range_found && range.committed_from_reserved && - range.start == vaddr && range.size == size; - shared_unmapped = g_direct_memory_backing->Unmap(vaddr, size, can_restore_placeholder, - &placeholder_restored); - if (placeholder_restored) { - g_placeholder_address_space->AddFree(vaddr, size); - } + for (const auto& alias: mapped_aliases) { + EXIT_IF(!g_virtual_ranges->Remove(alias.map_vaddr, alias.map_size)); } - if (exact_host_range && (!range_found || !range.committed_from_reserved) && - (vaddr != 0 || size != 0)) { - if (!shared_unmapped) { - VirtualMemory::Free(vaddr); - } - } else if (vaddr != 0 || size != 0) { - if (placeholder_restored) { - // Exact placeholder-backed unmaps already returned this range to the placeholder - // manager. - } else if (!shared_unmapped) { - VirtualMemory::Protect(vaddr, size, VirtualMemory::Mode::NoAccess); - } else if (range_found && range.committed_from_reserved) { - VirtualMemory::ReserveFixed(vaddr, size); - } - if (range_found && range.committed_from_reserved) { - g_virtual_ranges->Add(vaddr, size, 0, 0, 0, VirtualRangeType::Reserved, range.name, - false, placeholder_restored); - } + for (const auto& block: allocated_span) { + uint64_t unused_vaddr = 0; + uint64_t unused_size = 0; + GpuAccessMode unused_gpu = GpuAccessMode::NoAccess; + EXIT_IF(!g_physical_memory->Release(block.start_addr, block.size, &unused_vaddr, + &unused_size, &unused_gpu)); } if (g_free_callback != nullptr) { - g_free_callback(vaddr, len); + for (const auto& alias: mapped_aliases) { + g_free_callback(alias.map_vaddr, alias.map_size); + } } return OK; } +static int ValidateDirectReleaseRange(int64_t start, size_t len) { + constexpr uint64_t PAGE_SIZE = 0x4000; + return start < 0 || (static_cast(start) & (PAGE_SIZE - 1u)) != 0 || + (len & (PAGE_SIZE - 1u)) != 0 + ? KERNEL_ERROR_EINVAL + : OK; +} + +int KYTY_SYSV_ABI KernelReleaseDirectMemory(int64_t start, size_t len) { + PRINT_NAME(); + + LOGF("\t start = 0x%016" PRIx64 "\n" + "\t len = 0x%016" PRIx64 "\n", + start, len); + + const int validation = ValidateDirectReleaseRange(start, len); + if (validation != OK) { + return validation; + } + if (len != 0) { + (void)ReleaseDirectMemoryInternal(start, len); + } + return OK; +} + int KYTY_SYSV_ABI KernelCheckedReleaseDirectMemory(int64_t start, size_t len) { PRINT_NAME(); @@ -2712,20 +2848,13 @@ int KYTY_SYSV_ABI KernelCheckedReleaseDirectMemory(int64_t start, size_t len) { "\t len = 0x%016" PRIx64 "\n", start, len); - if (start < 0) { - return KERNEL_ERROR_EINVAL; + const int validation = ValidateDirectReleaseRange(start, len); + if (validation != OK || len == 0) { + return validation; } - constexpr size_t PAGE_SIZE = 0x4000; - if ((static_cast(start) & (PAGE_SIZE - 1)) != 0 || (len & (PAGE_SIZE - 1)) != 0) { - return KERNEL_ERROR_EINVAL; - } - - if (len == 0) { - return OK; - } - - return KernelReleaseDirectMemory(start, len); + const int result = ReleaseDirectMemoryInternal(start, len); + return result == KERNEL_ERROR_EACCES ? KERNEL_ERROR_ENOENT : result; } int KYTY_SYSV_ABI KernelMapDirectMemory(void** addr, size_t len, int prot, int flags, @@ -2734,9 +2863,22 @@ int KYTY_SYSV_ABI KernelMapDirectMemory(void** addr, size_t len, int prot, int f std::lock_guard memory_operation_lock(g_memory_operation_mutex); - EXIT_NOT_IMPLEMENTED(addr == nullptr); - constexpr int GUEST_MAP_FIXED = 0x10; - constexpr int GUEST_MAP_NO_OVERWRITE = 0x80; + if (addr == nullptr) { + return KERNEL_ERROR_EFAULT; + } + constexpr uint64_t PAGE_SIZE = 0x4000; + constexpr int GUEST_MAP_FIXED = 0x10; + constexpr int GUEST_MAP_NO_OVERWRITE = 0x80; + + if (len == 0 || (len & (PAGE_SIZE - 1u)) != 0 || direct_memory_start < 0 || + (static_cast(direct_memory_start) & (PAGE_SIZE - 1u)) != 0 || + (alignment != 0 && (alignment & (alignment - 1u)) != 0 && + (alignment & (PAGE_SIZE - 1u)) != 0)) { + return KERNEL_ERROR_EINVAL; + } + if ((prot & PROT_CPU_EXEC) != 0) { + return KERNEL_ERROR_EACCES; + } bool fixed = ((flags & GUEST_MAP_FIXED) != 0); bool no_overwrite = ((flags & GUEST_MAP_NO_OVERWRITE) != 0); @@ -2745,136 +2887,87 @@ int KYTY_SYSV_ABI KernelMapDirectMemory(void** addr, size_t len, int prot, int f GpuAccessMode gpu_mode = GpuAccessMode::NoAccess; if (!DecodeMemoryProtection(prot, &mode, &gpu_mode)) { - EXIT("unknown prot: %d\n", prot); + return KERNEL_ERROR_EINVAL; } - if (direct_memory_start < 0 || len == 0 || - !g_physical_memory->CanMapDirect(static_cast(direct_memory_start), len)) { + if (!g_physical_memory->CanMapDirect(static_cast(direct_memory_start), len)) { return KERNEL_ERROR_ENOMEM; } - auto in_addr = reinterpret_cast(*addr); - uint64_t out_addr = 0; - bool committed_from_reserved = false; - bool shared_backing = false; - bool consumed_reserved = false; + auto in_addr = reinterpret_cast(*addr); + uint64_t out_addr = 0; + bool shared_backing = false; + bool consumed_reservation = false; VirtualRanges::Range consumed_range {}; - auto shared_failure = DirectMemoryBacking::FailureReason::None; + auto shared_failure = GuestBackingStore::FailureReason::None; // Direct mappings must remain views of the single backing object. Anonymous fallbacks break // aliasing and lose direct-memory contents when a range is unmapped and mapped again. auto map_shared_fixed = [&](uint64_t target_addr) -> bool { - const bool placeholder_ready = g_placeholder_address_space->Consume(target_addr, len); - if (placeholder_ready) { - if (g_direct_memory_backing->MapExistingPlaceholderFixed( - target_addr, len, direct_memory_start, mode, &shared_failure)) { - return true; - } - g_placeholder_address_space->AddFree(target_addr, len); - return false; - } - - return g_direct_memory_backing->MapFixed(target_addr, len, direct_memory_start, mode, + return g_guest_address_space->MapBacking(target_addr, len, direct_memory_start, mode, &shared_failure); }; auto map_consumed_reserved_fixed = [&]() { - consumed_reserved = true; if (map_shared_fixed(in_addr)) { - out_addr = in_addr; - committed_from_reserved = true; - shared_backing = true; + out_addr = in_addr; + shared_backing = true; } }; if (fixed) { - EXIT_NOT_IMPLEMENTED(in_addr == 0); - EXIT_NOT_IMPLEMENTED(alignment != 0 && (in_addr & (alignment - 1)) != 0); + if (in_addr == 0 || (in_addr & (PAGE_SIZE - 1u)) != 0 || + (alignment != 0 && in_addr % alignment != 0)) { + return KERNEL_ERROR_EINVAL; + } if (no_overwrite && g_virtual_ranges->HasOverlap(in_addr, len)) { return KERNEL_ERROR_ENOMEM; } - VirtualRanges::Range same_range {}; - if (g_virtual_ranges->Query(in_addr, 0, &same_range) && - same_range.type == VirtualRangeType::Direct && - len <= same_range.start + same_range.size - in_addr && - same_range.offset + (in_addr - same_range.start) == - static_cast(direct_memory_start) && - same_range.protection == prot) { - *addr = reinterpret_cast(in_addr); - LOGF_COLOR(Log::Color::Green, - "\t in_addr = 0x%016" PRIx64 "\n" - "\t out_addr = 0x%016" PRIx64 "\n" - "\t dmem = 0x%016" PRIx64 "\n" - "\t size = 0x%016" PRIx64 "\n" - "\t mode = %s\n" - "\t flags = 0x%08" PRIx32 "\n" - "\t align = 0x%016" PRIx64 "\n" - "\t gpu_mode = %s\n" - "\t shared = %s\n" - "\t reason = already-mapped\n" - "\t [Ok]\n", - in_addr, in_addr, static_cast(direct_memory_start), len, - Common::EnumName(mode).c_str(), static_cast(flags), alignment, - Common::EnumName(gpu_mode).c_str(), - g_direct_memory_backing->Contains(in_addr, len) ? "yes" : "no"); - return OK; - } - - if (g_virtual_ranges->ConsumeReservedSpan(in_addr, len, &consumed_range)) { - map_consumed_reserved_fixed(); - } else { - bool placeholder_backed = false; - if (ReplaceFixedRangeWithReserved(in_addr, len, &placeholder_backed) && - g_virtual_ranges->ConsumeReservedSpan(in_addr, len, &consumed_range)) { + std::vector reserved_ranges; + if (g_virtual_ranges->QuerySpan(in_addr, len, &reserved_ranges) && + std::all_of(reserved_ranges.begin(), reserved_ranges.end(), [](const auto& range) { + return range.type == VirtualRangeType::Reserved; + })) { + UnmapGpuRange(in_addr, len); + consumed_range = reserved_ranges.front(); + if (g_virtual_ranges->ConsumeReservedSpan(in_addr, len)) { + consumed_reservation = true; map_consumed_reserved_fixed(); - } else { - VirtualRanges::Range existing_range {}; - if (g_virtual_ranges->Query(in_addr, 0, &existing_range)) { - GpuAccessMode old_gpu_mode = GpuAccessMode::NoAccess; - g_physical_memory->Unmap(in_addr, len, &old_gpu_mode); - g_flexible_memory->Unmap(in_addr, len, &old_gpu_mode); - g_virtual_ranges->Remove(in_addr, len); - const bool old_shared = g_direct_memory_backing->Contains(in_addr, len); - - bool old_placeholder_restored = false; - g_direct_memory_backing->Unmap(in_addr, len, - existing_range.committed_from_reserved, - &old_placeholder_restored); - if (old_placeholder_restored) { - g_placeholder_address_space->AddFree(in_addr, len); - } - - if (old_shared && g_direct_memory_backing->Contains(in_addr, len)) { - out_addr = 0; - } else if (map_shared_fixed(in_addr)) { - out_addr = in_addr; - committed_from_reserved = existing_range.committed_from_reserved || - existing_range.start != in_addr || - existing_range.size != len; - shared_backing = true; - } - } else if (!ReleaseReservedRange(in_addr, len)) { - return KERNEL_ERROR_ENOMEM; - } else if (map_shared_fixed(in_addr)) { + } + } + if (!consumed_reservation && ReplaceFixedRangeWithReserved(in_addr, len) && + g_virtual_ranges->ConsumeReservedSpan(in_addr, len, &consumed_range)) { + consumed_reservation = true; + map_consumed_reserved_fixed(); + } + if (!consumed_reservation) { + return KERNEL_ERROR_ENOMEM; + } + } else { + constexpr size_t DEFAULT_ALIGNMENT = 0x4000; + alignment = (alignment != 0 ? alignment : DEFAULT_ALIGNMENT); + std::vector reserved_ranges; + if (in_addr != 0 && g_virtual_ranges->QuerySpan(in_addr, len, &reserved_ranges) && + std::all_of(reserved_ranges.begin(), reserved_ranges.end(), [](const auto& range) { + return range.type == VirtualRangeType::Reserved; + })) { + UnmapGpuRange(in_addr, len); + consumed_range = reserved_ranges.front(); + if (g_virtual_ranges->ConsumeReservedSpan(in_addr, len)) { + consumed_reservation = true; + if (map_shared_fixed(in_addr)) { out_addr = in_addr; shared_backing = true; } } } - } else { - constexpr size_t DEFAULT_ALIGNMENT = 0x4000; - alignment = (alignment != 0 ? alignment : DEFAULT_ALIGNMENT); - if (in_addr != 0 && g_virtual_ranges->Query(in_addr, 0, &consumed_range) && - consumed_range.type == VirtualRangeType::Reserved && - g_virtual_ranges->ConsumeReserved(in_addr, len)) { - consumed_reserved = true; - if (map_shared_fixed(in_addr)) { - out_addr = in_addr; - committed_from_reserved = true; - shared_backing = true; + if (!consumed_reservation) { + out_addr = FindGuestFreeRange(in_addr, len, alignment); + if (out_addr != 0) { + UnmapGpuRange(out_addr, len); + shared_backing = map_shared_fixed(out_addr); + if (!shared_backing) { + out_addr = 0; + } } - } else { - out_addr = g_direct_memory_backing->MapAligned(in_addr, len, direct_memory_start, mode, - alignment, &shared_failure); - shared_backing = out_addr != 0; } } @@ -2882,7 +2975,7 @@ int KYTY_SYSV_ABI KernelMapDirectMemory(void** addr, size_t len, int prot, int f const char* shared_reason = "n/a"; if (!shared_backing) { - shared_reason = DirectMemoryBacking::GetFailureReasonName(shared_failure); + shared_reason = GuestBackingStore::GetFailureReasonName(shared_failure); } LOGF("\t in_addr = 0x%016" PRIx64 "\n" @@ -2900,43 +2993,38 @@ int KYTY_SYSV_ABI KernelMapDirectMemory(void** addr, size_t len, int prot, int f Common::EnumName(gpu_mode).c_str(), shared_backing ? "yes" : "no", shared_reason); if (out_addr == 0) { - if (consumed_reserved) { + if (consumed_reservation) { g_virtual_ranges->Add(in_addr, len, 0, 0, 0, VirtualRangeType::Reserved, - consumed_range.name, false, consumed_range.placeholder_backed); + consumed_range.name); } return KERNEL_ERROR_ENOMEM; } if (!g_physical_memory->Map(out_addr, direct_memory_start, len, prot, mode, gpu_mode)) { LOGF_COLOR(Log::Color::Red, "\t [Fail]\n"); - bool placeholder_restored = false; - g_direct_memory_backing->Unmap(out_addr, len, committed_from_reserved, - &placeholder_restored); - if (placeholder_restored) { - g_placeholder_address_space->AddFree(out_addr, len); + EXIT_IF(!g_guest_address_space->UnmapBacking(out_addr, len)); + if (consumed_reservation) { + EXIT_IF(!g_virtual_ranges->Add(in_addr, len, 0, 0, 0, VirtualRangeType::Reserved, + consumed_range.name)); } - - KYTY_NOT_IMPLEMENTED; - return KERNEL_ERROR_EBUSY; } PhysicalMemory::AllocatedBlock mapped_block {}; g_physical_memory->Find(direct_memory_start, false, &mapped_block); if (!g_virtual_ranges->Add(out_addr, len, direct_memory_start, prot, mapped_block.memory_type, - VirtualRangeType::Direct, "", committed_from_reserved)) { + VirtualRangeType::Direct, "")) { GpuAccessMode rollback_gpu_mode = GpuAccessMode::NoAccess; - g_physical_memory->Unmap(out_addr, len, &rollback_gpu_mode); - bool placeholder_restored = false; - g_direct_memory_backing->Unmap(out_addr, len, committed_from_reserved, - &placeholder_restored); - if (placeholder_restored) { - g_placeholder_address_space->AddFree(out_addr, len); + EXIT_IF(!g_physical_memory->Unmap(out_addr, len, &rollback_gpu_mode)); + EXIT_IF(!g_guest_address_space->UnmapBacking(out_addr, len)); + if (consumed_reservation) { + EXIT_IF(!g_virtual_ranges->Add(in_addr, len, 0, 0, 0, VirtualRangeType::Reserved, + consumed_range.name)); } return KERNEL_ERROR_EBUSY; } - MapGpuRange(out_addr, len, gpu_mode); + MapGpuRange(out_addr, len); if (g_alloc_callback != nullptr) { g_alloc_callback(out_addr, len); @@ -3022,93 +3110,15 @@ int KYTY_SYSV_ABI KernelQueryMemoryProtection(void* addr, void** start, void** e return OK; } -static bool ReserveFixedHostRange(uint64_t start, uint64_t size) { - constexpr uint64_t PAGE_SIZE = 0x4000; - -#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS - SYSTEM_INFO system_info {}; - GetSystemInfo(&system_info); - const uint64_t granularity = system_info.dwAllocationGranularity; - if ((start & (granularity - 1u)) == 0 && (size & (granularity - 1u)) == 0) { - MEMORY_BASIC_INFORMATION info {}; - if (VirtualQuery(reinterpret_cast(start), &info, sizeof(info)) != 0 && - info.State == MEM_FREE && info.RegionSize >= size && - VirtualMemory::ReserveFixed(start, size)) { - return true; - } - } -#endif - -#if defined(KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS) - for (uint64_t addr = start; addr < start + size; addr += PAGE_SIZE) { - if (g_test_host_reservation_pages_before_failure == 0) { - g_test_host_reservation_pages_before_failure = UINT32_MAX; - return false; - } - if (g_test_host_reservation_pages_before_failure != UINT32_MAX) { - g_test_host_reservation_pages_before_failure--; - } - } - g_test_host_reservation_pages_before_failure = UINT32_MAX; -#endif - -#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS - // Prefer one syscall, then fall back to the per-page path. - if (size > PAGE_SIZE && VirtualMemory::ReserveFixed(start, size)) { - return true; - } -#endif - - bool host_mutated = false; - for (uint64_t addr = start; addr < start + size; addr += PAGE_SIZE) { -#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS - MEMORY_BASIC_INFORMATION info {}; - if (VirtualQuery(reinterpret_cast(addr), &info, sizeof(info)) != 0) { - if (info.State == MEM_COMMIT) { - if (!VirtualMemory::Decommit(addr, PAGE_SIZE)) { - if (host_mutated) { - EXIT("reserve-fixed partial host decommit cannot be rolled back safely\n"); - } - LOGF_COLOR(Log::Color::Red, - "\t reserve-fixed replace: decommit failed at 0x%016" PRIx64 "\n", - addr); - return false; - } - host_mutated = true; - continue; - } - if (info.State == MEM_RESERVE) { - continue; - } - } -#endif - if (!VirtualMemory::ReserveFixed(addr, PAGE_SIZE)) { - if (host_mutated) { - EXIT("reserve-fixed partial host reservation cannot be rolled back safely\n"); - } - LOGF_COLOR(Log::Color::Red, - "\t reserve-fixed replace: reserve failed at 0x%016" PRIx64 "\n", addr); - return false; - } - host_mutated = true; - } - - return true; -} - -static bool ReplaceFixedRangeWithReserved(uint64_t start, uint64_t size, bool* placeholder_backed) { - EXIT_IF(placeholder_backed == nullptr); - - *placeholder_backed = false; +static bool ReplaceFixedRangeWithReserved(uint64_t start, uint64_t size) { struct ReplacedChunk { - VirtualRanges::Range range {}; - VirtualMemory::Mode mode = VirtualMemory::Mode::NoAccess; - GpuAccessMode gpu_mode = GpuAccessMode::NoAccess; - bool shared_backing = false; - bool placeholder_restored = false; - bool host_unmapped = false; - bool backend_unmapped = false; + VirtualRanges::Range range {}; + VirtualMemory::Mode mode = VirtualMemory::Mode::NoAccess; + GpuAccessMode gpu_mode = GpuAccessMode::NoAccess; + std::vector flexible_blocks; + bool host_unmapped = false; + bool backend_unmapped = false; }; std::vector chunks; @@ -3139,8 +3149,19 @@ static bool ReplaceFixedRangeWithReserved(uint64_t start, uint64_t size, bool* p replaced.range.offset += current - range.start; } DecodeMemoryProtection(replaced.range.protection, &replaced.mode, &replaced.gpu_mode); - replaced.shared_backing = (range.type == VirtualRangeType::Direct && - g_direct_memory_backing->Contains(current, chunk)); + if (range.type == VirtualRangeType::Direct && + !g_guest_address_space->BackingContains(current, chunk)) { + return false; + } + if (range.type == VirtualRangeType::Flexible && + !g_flexible_memory->Snapshot(current, chunk, &replaced.flexible_blocks)) { + return false; + } + if (range.type == VirtualRangeType::Pooled) { + EXIT("reserve-fixed replacement of pooled memory is unsupported: addr=0x%016" PRIx64 + " size=0x%016" PRIx64 "\n", + replaced.range.start, replaced.range.size); + } chunks.push_back(replaced); current += chunk; @@ -3154,68 +3175,43 @@ static bool ReplaceFixedRangeWithReserved(uint64_t start, uint64_t size, bool* p bool backend_restored = true; if (chunk.range.type == VirtualRangeType::Direct) { - if (chunk.host_unmapped && chunk.placeholder_restored) { - const bool consumed = - g_placeholder_address_space->Consume(chunk.range.start, chunk.range.size); - host_restored = - consumed && - g_direct_memory_backing->MapExistingPlaceholderFixed( - chunk.range.start, chunk.range.size, chunk.range.offset, chunk.mode); - if (consumed && !host_restored) { - g_placeholder_address_space->AddFree(chunk.range.start, chunk.range.size); - } - } else if (chunk.host_unmapped || !chunk.shared_backing) { - host_restored = - (chunk.shared_backing ? g_direct_memory_backing->MapFixed( - chunk.range.start, chunk.range.size, - chunk.range.offset, chunk.mode) - : CommitFixedHostRange(chunk.range.start, - chunk.range.size, chunk.mode)); + if (chunk.host_unmapped) { + host_restored = g_guest_address_space->MapBacking( + chunk.range.start, chunk.range.size, chunk.range.offset, chunk.mode); } if (chunk.backend_unmapped) { backend_restored = g_physical_memory->Map( chunk.range.start, chunk.range.offset, chunk.range.size, chunk.range.protection, chunk.mode, chunk.gpu_mode); } - } else if (chunk.range.type == VirtualRangeType::Flexible || - chunk.range.type == VirtualRangeType::Stack || - chunk.range.type == VirtualRangeType::Pooled) { + } else if (chunk.range.type == VirtualRangeType::Flexible && chunk.backend_unmapped) { + host_restored = backend_restored = + g_flexible_memory->Restore(chunk.flexible_blocks); + } else if (IsPrivateCommittedRangeType(chunk.range.type) && chunk.host_unmapped) { host_restored = - CommitFixedHostRange(chunk.range.start, chunk.range.size, chunk.mode); - if (chunk.backend_unmapped) { - backend_restored = g_flexible_memory->Map(chunk.range.start, chunk.range.size, - chunk.range.protection, chunk.mode, - chunk.gpu_mode, chunk.range.name); - } + g_guest_address_space->Commit(chunk.range.start, chunk.range.size, chunk.mode); } + ok = host_restored && backend_restored && ok; + } + for (const auto& chunk: chunks) { const bool range_restored = g_virtual_ranges->Add( chunk.range.start, chunk.range.size, chunk.range.offset, chunk.range.protection, chunk.range.memory_type, chunk.range.type, chunk.range.name, - chunk.range.committed_from_reserved, chunk.range.placeholder_backed); - ok = host_restored && backend_restored && range_restored && ok; + chunk.range.disallow_merge); + ok = range_restored && ok; + } + if (ok) { + for (const auto& chunk: chunks) { + if (IsCommittedRangeType(chunk.range.type)) { + MapGpuRange(chunk.range.start, chunk.range.size); + } + } } return ok; }; - for (const auto& chunk: chunks) { - if (chunk.range.type == VirtualRangeType::Pooled) { - EXIT("reserve-fixed replacement of pooled memory is unsupported: addr=0x%016" PRIx64 - " size=0x%016" PRIx64 "\n", - chunk.range.start, chunk.range.size); - } - } - const bool gpu_unmapped = std::any_of(chunks.begin(), chunks.end(), [](const auto& chunk) { - return IsCommittedRangeType(chunk.range.type) && - chunk.gpu_mode != GpuAccessMode::NoAccess && - IsGpuAddressRange(chunk.range.start, chunk.range.size); - }); - for (const auto& chunk: chunks) { - if (IsCommittedRangeType(chunk.range.type)) { - UnmapGpuRange(chunk.range.start, chunk.range.size, chunk.gpu_mode); - } - } - + UnmapGpuRange(start, size); g_virtual_ranges->Remove(start, size); for (auto& chunk: chunks) { @@ -3223,38 +3219,36 @@ static bool ReplaceFixedRangeWithReserved(uint64_t start, uint64_t size, bool* p bool unmapped = true; if (chunk.range.type == VirtualRangeType::Direct) { - if (chunk.shared_backing) { - bool chunk_placeholder_restored = false; - if (!g_direct_memory_backing->Unmap(chunk.range.start, chunk.range.size, true, - &chunk_placeholder_restored)) { - unmapped = false; - } else { - chunk.host_unmapped = true; - if (chunk_placeholder_restored) { - g_placeholder_address_space->AddFree(chunk.range.start, chunk.range.size); - chunk.placeholder_restored = true; - } - } + if (!g_guest_address_space->UnmapBacking(chunk.range.start, chunk.range.size)) { + unmapped = false; + } else { + chunk.host_unmapped = true; } if (unmapped) { unmapped = g_physical_memory->Unmap(chunk.range.start, chunk.range.size, &gpu_mode); chunk.gpu_mode = gpu_mode; chunk.backend_unmapped = unmapped; } - } else if (chunk.range.type == VirtualRangeType::Flexible || - chunk.range.type == VirtualRangeType::Stack || - chunk.range.type == VirtualRangeType::Pooled) { + } else if (chunk.range.type == VirtualRangeType::Flexible) { unmapped = g_flexible_memory->Unmap(chunk.range.start, chunk.range.size, &gpu_mode); - chunk.gpu_mode = gpu_mode; + chunk.host_unmapped = unmapped; chunk.backend_unmapped = unmapped; + chunk.gpu_mode = gpu_mode; + } else if (IsPrivateCommittedRangeType(chunk.range.type)) { + unmapped = g_guest_address_space->ReleaseCommitted(chunk.range.start, chunk.range.size); + chunk.host_unmapped = unmapped; + chunk.backend_unmapped = unmapped; + } else if (IsReservedRangeType(chunk.range.type)) { + unmapped = g_guest_address_space->ReleaseFree(chunk.range.start, chunk.range.size); + } else { + unmapped = false; + } + if (chunk.range.type == VirtualRangeType::Direct) { + chunk.gpu_mode = gpu_mode; + chunk.backend_unmapped = chunk.backend_unmapped && unmapped; } if (!unmapped) { - if (gpu_unmapped) { - EXIT("reserve-fixed backend unmap failed after GPU unmap: addr=0x%016" PRIx64 - " size=0x%016" PRIx64 "\n", - chunk.range.start, chunk.range.size); - } LOGF_COLOR(Log::Color::Red, "\t reserve-fixed replace: backend unmap failed at 0x%016" PRIx64 ", size=0x%016" PRIx64 ", type=%s\n", @@ -3267,14 +3261,7 @@ static bool ReplaceFixedRangeWithReserved(uint64_t start, uint64_t size, bool* p } } - if (g_placeholder_address_space->ReserveFixed(start, size)) { - *placeholder_backed = true; - } else if (!ReserveFixedHostRange(start, size)) { - if (gpu_unmapped) { - EXIT("reserve-fixed host reservation failed after GPU unmap: addr=0x%016" PRIx64 - " size=0x%016" PRIx64 "\n", - start, size); - } + if (!g_guest_address_space->ReserveFixed(start, size)) { if (!restore_chunks()) { EXIT("reserve-fixed host-reservation rollback failed\n"); } @@ -3288,18 +3275,10 @@ static bool ReplaceFixedRangeWithReserved(uint64_t start, uint64_t size, bool* p } else #endif { - range_added = g_virtual_ranges->Add(start, size, 0, 0, 0, VirtualRangeType::Reserved, - "anon", false, *placeholder_backed); + range_added = + g_virtual_ranges->Add(start, size, 0, 0, 0, VirtualRangeType::Reserved, "anon"); } if (!range_added) { - if (gpu_unmapped) { - EXIT("reserve-fixed range registration failed after GPU unmap: addr=0x%016" PRIx64 - " size=0x%016" PRIx64 "\n", - start, size); - } - if (!*placeholder_backed) { - VirtualMemory::Free(start); - } LOGF_COLOR(Log::Color::Red, "\t reserve-fixed replace: range add failed at 0x%016" PRIx64 ", size=0x%016" PRIx64 "\n", @@ -3307,20 +3286,17 @@ static bool ReplaceFixedRangeWithReserved(uint64_t start, uint64_t size, bool* p if (!restore_chunks()) { EXIT("reserve-fixed range-registration rollback failed\n"); } - if (*placeholder_backed) { - auto free_start = start; - for (const auto& chunk: chunks) { - if (free_start < chunk.range.start && - !g_placeholder_address_space->ReleaseFree(free_start, - chunk.range.start - free_start)) { - EXIT("reserve-fixed range-registration gap cleanup failed\n"); - } - free_start = chunk.range.start + chunk.range.size; - } - if (free_start < start + size && - !g_placeholder_address_space->ReleaseFree(free_start, start + size - free_start)) { - EXIT("reserve-fixed range-registration tail cleanup failed\n"); + auto free_start = start; + for (const auto& chunk: chunks) { + if (free_start < chunk.range.start && + !g_guest_address_space->ReleaseFree(free_start, chunk.range.start - free_start)) { + EXIT("reserve-fixed range-registration gap cleanup failed\n"); } + free_start = chunk.range.start + chunk.range.size; + } + if (free_start < start + size && + !g_guest_address_space->ReleaseFree(free_start, start + size - free_start)) { + EXIT("reserve-fixed range-registration tail cleanup failed\n"); } return false; } @@ -3354,13 +3330,13 @@ int KYTY_SYSV_ABI KernelReserveVirtualRange(void** addr, size_t len, int flags, if (addr == nullptr || len == 0 || (len & (PAGE_SIZE - 1)) != 0) { return KERNEL_ERROR_EINVAL; } - if (alignment != 0 && (alignment & (alignment - 1)) != 0) { + if (alignment != 0 && (alignment & (alignment - 1)) != 0 && + (alignment & (PAGE_SIZE - 1)) != 0) { return KERNEL_ERROR_EINVAL; } uint64_t out_addr = 0; bool range_already_added = false; - bool placeholder_backed = false; if ((flags & GUEST_MAP_FIXED) != 0) { if (in_addr == 0 || (in_addr & (PAGE_SIZE - 1)) != 0) { return KERNEL_ERROR_EINVAL; @@ -3368,17 +3344,15 @@ int KYTY_SYSV_ABI KernelReserveVirtualRange(void** addr, size_t len, int flags, if ((flags & GUEST_MAP_NO_OVERWRITE) != 0 && g_virtual_ranges->HasOverlap(in_addr, len)) { return KERNEL_ERROR_ENOMEM; } - if (ReplaceFixedRangeWithReserved(in_addr, len, &placeholder_backed)) { + if (ReplaceFixedRangeWithReserved(in_addr, len)) { out_addr = in_addr; range_already_added = true; } } else { - alignment = (alignment != 0 ? alignment : PAGE_SIZE); - out_addr = g_placeholder_address_space->ReserveAligned(in_addr, len, alignment); - placeholder_backed = (out_addr != 0); - if (out_addr == 0) { - out_addr = VirtualMemory::ReserveAligned(in_addr, len, alignment); - placeholder_backed = false; + alignment = (alignment != 0 ? alignment : PAGE_SIZE); + out_addr = FindGuestFreeRange(in_addr, len, alignment); + if (out_addr != 0) { + UnmapGpuRange(out_addr, len); } } @@ -3387,19 +3361,13 @@ int KYTY_SYSV_ABI KernelReserveVirtualRange(void** addr, size_t len, int flags, } if (!range_already_added && - !g_virtual_ranges->Add(out_addr, len, 0, 0, 0, VirtualRangeType::Reserved, "anon", false, - placeholder_backed)) { - if (!placeholder_backed || !g_placeholder_address_space->ReleaseFree(out_addr, len)) { - VirtualMemory::Free(out_addr); - } + !g_virtual_ranges->Add(out_addr, len, 0, 0, 0, VirtualRangeType::Reserved, "anon")) { return KERNEL_ERROR_EBUSY; } *addr = reinterpret_cast(out_addr); - LOGF("\t out_addr = 0x%016" PRIx64 "\n" - "\t placeholder = %s\n", - out_addr, placeholder_backed ? "yes" : "no"); + LOGF("\t out_addr = 0x%016" PRIx64 "\n", out_addr); return OK; } @@ -3413,8 +3381,8 @@ void TestFailPhysicalMemoryUnmapAfter(uint32_t successful_unmaps) { g_test_physical_memory_unmaps_before_failure = successful_unmaps; } -void TestFailHostReservationAfter(uint32_t successful_pages) { - g_test_host_reservation_pages_before_failure = successful_pages; +void TestFailGuestBackingStoreUnmapAfter(uint32_t successful_unmaps) { + g_test_backing_store_unmaps_before_failure = successful_unmaps; } void TestFailNextFixedReserveRangeRegistration() { @@ -3422,7 +3390,26 @@ void TestFailNextFixedReserveRangeRegistration() { } bool TestPlaceholderRangeIsFree(uint64_t vaddr, uint64_t size) { - return g_placeholder_address_space->TestContainsFree(vaddr, size); + return g_guest_address_space->TestContainsFree(vaddr, size); +} + +bool TestGuestAddressRangeIsOwned(uint64_t vaddr, uint64_t size) { + return g_guest_address_space->Owns(vaddr, size); +} + +bool TestGuestBackingOutsideAddressSpace() { + return !g_guest_address_space->OverlapsOwned(g_guest_address_space->GetBackingBase(), + g_guest_address_space->GetBackingSize()); +} + +uint64_t TestGuestBackingSize() { + return g_guest_address_space->GetBackingSize(); +} + +bool TestGuestFreeRangeBounds() { + return GuestFreeRangeContains(0x10000, 0x20000, 0x18000, 0x4000) && + !GuestFreeRangeContains(0x10000, 0x20000, 0x40000, 0x4000) && + !GuestFreeRangeContains(UINT64_MAX - 0x1000, 0x2000, UINT64_MAX - 0x800, 0x400); } #endif @@ -3532,7 +3519,9 @@ int KYTY_SYSV_ABI KernelAvailableFlexibleMemorySize(size_t* size) { std::lock_guard memory_operation_lock(g_memory_operation_mutex); - EXIT_NOT_IMPLEMENTED(size == nullptr); + if (size == nullptr) { + return KERNEL_ERROR_EINVAL; + } *size = g_flexible_memory->Available(); @@ -3541,7 +3530,7 @@ int KYTY_SYSV_ABI KernelAvailableFlexibleMemorySize(size_t* size) { return OK; } -int KYTY_SYSV_ABI KernelConfiguredFlexibleMemorySize(uint64_t* size) { +int KYTY_SYSV_ABI KernelConfiguredFlexibleMemorySize(size_t* size) { PRINT_NAME(); if (size == nullptr) { @@ -3563,55 +3552,123 @@ static int ProgramProtection(VirtualMemory::Mode mode) { return protection; } -static std::vector RequireProgramMemory(uint64_t vaddr, uint64_t size) { +static std::vector RequireGuestRuntimeMemory(uint64_t vaddr, uint64_t size) { std::vector ranges; if (g_virtual_ranges == nullptr || !g_virtual_ranges->QuerySpan(vaddr, size, &ranges) || - std::any_of(ranges.begin(), ranges.end(), - [](const auto& range) { return range.type != VirtualRangeType::Code; })) { - EXIT("program-memory range is not fully mapped: addr=0x%016" PRIx64 " size=0x%016" PRIx64 + std::any_of(ranges.begin(), ranges.end(), [](const auto& range) { + return range.type != VirtualRangeType::Code && range.type != VirtualRangeType::Runtime; + })) { + EXIT("guest runtime range is not fully mapped: addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", vaddr, size); } return ranges; } -void RegisterProgramMemory(uint64_t vaddr, uint64_t size, VirtualMemory::Mode mode, - const char* name) { +static uint64_t AllocateGuestRuntimeMemory(uint64_t search_addr, uint64_t size, + VirtualMemory::Mode mode, const char* name, + VirtualRangeType type, bool fixed) { std::lock_guard memory_operation_lock(g_memory_operation_mutex); - if (g_virtual_ranges == nullptr || vaddr == 0 || size == 0 || size > UINT64_MAX - vaddr || - name == nullptr || - !g_virtual_ranges->Add(vaddr, size, 0, ProgramProtection(mode), 0, VirtualRangeType::Code, - name)) { - EXIT("failed to register program memory: addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", - vaddr, size); + constexpr uint64_t GuestPageSize = 0x4000; + if (size == 0 || size > UINT64_MAX - (GuestPageSize - 1u) || name == nullptr || + (fixed && (search_addr == 0 || (search_addr & (GuestPageSize - 1u)) != 0))) { + return 0; } + const auto mapped_size = (size + GuestPageSize - 1u) & ~(GuestPageSize - 1u); + const auto vaddr = + fixed ? search_addr : FindGuestFreeRange(search_addr, mapped_size, GuestPageSize); + if (vaddr == 0 || g_virtual_ranges->HasOverlap(vaddr, mapped_size)) { + return 0; + } + UnmapGpuRange(vaddr, mapped_size); + if (!g_guest_address_space->Commit(vaddr, mapped_size, mode)) { + return 0; + } + if (!g_virtual_ranges->Add(vaddr, mapped_size, 0, ProgramProtection(mode), 0, type, name)) { + EXIT_IF(!g_guest_address_space->ReleaseCommitted(vaddr, mapped_size)); + return 0; + } + MapGpuRange(vaddr, mapped_size); + return vaddr; } -void UpdateProgramMemoryProtection(uint64_t vaddr, uint64_t size, VirtualMemory::Mode mode) { +uint64_t AllocateProgramMemory(uint64_t search_addr, uint64_t size, VirtualMemory::Mode mode, + const char* name) { + return AllocateGuestRuntimeMemory(search_addr, size, mode, name, VirtualRangeType::Code, false); +} + +void SetProgramMemoryProtection(uint64_t vaddr, uint64_t size, VirtualMemory::Mode mode) { std::lock_guard memory_operation_lock(g_memory_operation_mutex); - RequireProgramMemory(vaddr, size); + + const auto ranges = RequireGuestRuntimeMemory(vaddr, size); + if (std::any_of(ranges.begin(), ranges.end(), + [](const auto& range) { return range.type != VirtualRangeType::Code; })) { + EXIT("program-memory range is not fully mapped: addr=0x%016" PRIx64 " size=0x%016" PRIx64 + "\n", + vaddr, size); + } + + const auto host_mode = VirtualMemory::IsExecute(mode) ? VirtualMemory::Mode::ExecuteReadWrite + : VirtualMemory::Mode::ReadWrite; + EXIT_IF(!g_guest_address_space->Protect(vaddr, size, host_mode)); g_virtual_ranges->Protect(vaddr, size, ProgramProtection(mode)); } -void UnregisterProgramMemory(uint64_t vaddr, uint64_t size) { +uint64_t AllocateRuntimeMemory(uint64_t search_addr, uint64_t size, VirtualMemory::Mode mode, + const char* name, bool fixed) { + return AllocateGuestRuntimeMemory(search_addr, size, mode, name, VirtualRangeType::Runtime, + fixed); +} + +uint64_t AllocateGuestStackMemory(uint64_t search_addr, uint64_t size, VirtualMemory::Mode mode, + const char* name) { + const auto vaddr = + AllocateGuestRuntimeMemory(search_addr, size, mode, name, VirtualRangeType::Stack, false); + if (vaddr != 0 && g_alloc_callback != nullptr) { + g_alloc_callback(vaddr, size); + } + return vaddr; +} + +bool ProtectGuestMemory(uint64_t vaddr, uint64_t size, VirtualMemory::Mode mode, + VirtualMemory::Mode* old_mode) { + std::lock_guard memory_operation_lock(g_memory_operation_mutex); + constexpr uint64_t GuestPageSize = 0x4000; + if (vaddr == 0 || size == 0 || size > UINT64_MAX - (vaddr & (GuestPageSize - 1u))) { + return false; + } + const auto aligned_addr = vaddr & ~(GuestPageSize - 1u); + const auto aligned_size = + (size + (vaddr - aligned_addr) + GuestPageSize - 1u) & ~(GuestPageSize - 1u); + const auto ranges = RequireGuestRuntimeMemory(aligned_addr, aligned_size); + if (old_mode != nullptr) { + *old_mode = static_cast( + ranges.front().protection & (PROT_CPU_READ | PROT_CPU_WRITE | PROT_CPU_EXEC)); + } + if (!g_guest_address_space->Protect(aligned_addr, aligned_size, mode)) { + return false; + } + g_virtual_ranges->Protect(aligned_addr, aligned_size, ProgramProtection(mode)); + return true; +} + +bool ProtectGuestHostMemory(uint64_t vaddr, uint64_t size, VirtualMemory::Mode mode) { + return g_guest_address_space != nullptr && + g_guest_address_space->ProtectTransient(vaddr, size, mode); +} + +bool FreeGuestMemory(uint64_t vaddr, uint64_t size) { std::lock_guard memory_operation_lock(g_memory_operation_mutex); - for (const auto& range: RequireProgramMemory(vaddr, size)) { - const auto gpu_mode = GetGpuAccessMode(range.protection); - if (gpu_mode != GpuAccessMode::NoAccess) { - if (!GetGpuResources().IsMapped(range.start, range.size)) { - EXIT("program GPU range is not tracked: addr=0x%016" PRIx64 " size=0x%016" PRIx64 - "\n", - range.start, range.size); - } - UnmapGpuRange(range.start, range.size, gpu_mode); - } - } - if (!g_virtual_ranges->Remove(vaddr, size)) { - EXIT("failed to unregister program memory: addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n", - vaddr, size); + constexpr uint64_t GuestPageSize = 0x4000; + if (vaddr == 0 || size == 0 || size > UINT64_MAX - (GuestPageSize - 1u)) { + return false; } + const auto mapped_size = (size + GuestPageSize - 1u) & ~(GuestPageSize - 1u); + (void)RequireGuestRuntimeMemory(vaddr, mapped_size); + UnmapGpuRange(vaddr, mapped_size); + return FreeGuestMemoryOwner(vaddr, mapped_size); } int KYTY_SYSV_ABI KernelMprotect(const void* addr, size_t len, int prot) { @@ -3652,30 +3709,24 @@ int KYTY_SYSV_ABI KernelMprotect(const void* addr, size_t len, int prot) { "\n", aligned_addr, aligned_len); } - for (const auto& old_range: old_ranges) { - const auto old_gpu_mode = GetGpuAccessMode(old_range.protection); - if (old_gpu_mode == GpuAccessMode::NoAccess) { - continue; - } - if (!GetGpuResources().IsMapped(old_range.start, old_range.size)) { - EXIT("GPU protection transition requires tracked memory: addr=0x%016" PRIx64 - " size=0x%016" PRIx64 " type=%s\n", - old_range.start, old_range.size, Common::EnumName(old_range.type).c_str()); - } - UnmapGpuRange(old_range.start, old_range.size, old_gpu_mode); - } - - VirtualMemory::Mode old_mode {}; - bool ok = ProtectCommittedHostMemory(aligned_addr, aligned_len, mode, &old_mode); + const auto old_mode = static_cast( + old_ranges.front().protection & (PROT_CPU_READ | PROT_CPU_WRITE | PROT_CPU_EXEC)); + bool ok = g_guest_address_space->Protect(aligned_addr, aligned_len, mode); if (!ok) { EXIT("host memory-protection update failed: addr=0x%016" PRIx64 " size=0x%016" PRIx64 " prot=0x%08x\n", aligned_addr, aligned_len, prot); } + for (const auto& old_range: old_ranges) { + if (old_range.type == VirtualRangeType::Direct) { + g_physical_memory->ProtectMapping(old_range.start, old_range.size, prot, mode, + gpu_mode); + } else if (old_range.type == VirtualRangeType::Flexible) { + g_flexible_memory->Protect(old_range.start, old_range.size, prot, mode, gpu_mode); + } + } g_virtual_ranges->Protect(aligned_addr, aligned_len, prot); - g_direct_memory_backing->UpdateProtection(aligned_addr, aligned_len, mode); - MapGpuRange(aligned_addr, aligned_len, gpu_mode); LOGF("\t prot: %s -> %s\n", Common::EnumName(old_mode).c_str(), Common::EnumName(mode).c_str()); @@ -3878,11 +3929,10 @@ int KYTY_SYSV_ABI KernelMemoryPoolReserve(void* addr_in, size_t len, size_t alig } g_virtual_ranges->Remove(out_vaddr, len); if (!g_virtual_ranges->Add(out_vaddr, len, 0, 0, 0, VirtualRangeType::PoolReserved, - reserved_range.name, false, reserved_range.placeholder_backed)) { + reserved_range.name)) { g_virtual_ranges->Add(out_vaddr, len, reserved_range.offset, reserved_range.protection, reserved_range.memory_type, reserved_range.type, - reserved_range.name, reserved_range.committed_from_reserved, - reserved_range.placeholder_backed); + reserved_range.name); return KERNEL_ERROR_EBUSY; } *addr_out = out_addr; @@ -3933,8 +3983,7 @@ int KYTY_SYSV_ABI KernelMemoryPoolCommit(void* addr, size_t len, int type, int p std::vector mappings; if (!g_pooled_memory->Allocate(vaddr, len, gpu_mode, &mappings)) { - g_virtual_ranges->Add(vaddr, len, 0, 0, 0, VirtualRangeType::PoolReserved, old_range.name, - false, old_range.placeholder_backed); + g_virtual_ranges->Add(vaddr, len, 0, 0, 0, VirtualRangeType::PoolReserved, old_range.name); return KERNEL_ERROR_ENOMEM; } @@ -3947,39 +3996,29 @@ int KYTY_SYSV_ABI KernelMemoryPoolCommit(void* addr, size_t len, int type, int p if (!g_pooled_memory->Release(vaddr, len, &rollback_gpu_mode)) { EXIT("failed to release pooled-memory rollback allocation\n"); } - g_virtual_ranges->Add(vaddr, len, 0, 0, 0, VirtualRangeType::PoolReserved, old_range.name, - false, old_range.placeholder_backed); + g_virtual_ranges->Add(vaddr, len, 0, 0, 0, VirtualRangeType::PoolReserved, old_range.name); }; for (const auto& mapping: mappings) { - auto failure_reason = DirectMemoryBacking::FailureReason::None; - const bool placeholder_ready = - g_placeholder_address_space->Consume(mapping.vaddr, mapping.size); - const bool ok = - placeholder_ready - ? g_direct_memory_backing->MapExistingPlaceholderFixed( - mapping.vaddr, mapping.size, mapping.phys_addr, mode, &failure_reason) - : g_direct_memory_backing->MapFixed(mapping.vaddr, mapping.size, mapping.phys_addr, - mode, &failure_reason); + auto failure_reason = GuestBackingStore::FailureReason::None; + const bool ok = g_guest_address_space->MapBacking(mapping.vaddr, mapping.size, + mapping.phys_addr, mode, &failure_reason); if (!ok) { - if (placeholder_ready) { - g_placeholder_address_space->AddFree(mapping.vaddr, mapping.size); - } LOGF_COLOR(Log::Color::Red, "\t pool backing map failed: %s\n", - DirectMemoryBacking::GetFailureReasonName(failure_reason)); + GuestBackingStore::GetFailureReasonName(failure_reason)); rollback(); return KERNEL_ERROR_ENOMEM; } mapped.push_back(mapping); } - if (!g_virtual_ranges->Add(vaddr, len, 0, prot, type, VirtualRangeType::Pooled, old_range.name, - true, old_range.placeholder_backed)) { + if (!g_virtual_ranges->Add(vaddr, len, 0, prot, type, VirtualRangeType::Pooled, + old_range.name)) { rollback(); return KERNEL_ERROR_EBUSY; } - MapGpuRange(vaddr, len, gpu_mode); + MapGpuRange(vaddr, len); if (g_alloc_callback != nullptr) { g_alloc_callback(vaddr, len); @@ -3990,6 +4029,55 @@ int KYTY_SYSV_ABI KernelMemoryPoolCommit(void* addr, size_t len, int type, int p return OK; } +static int DecommitMemoryPoolRange(uint64_t vaddr, size_t len) { + VirtualRanges::Range old_range {}; + if (!g_virtual_ranges->Query(vaddr, 0, &old_range)) { + return KERNEL_ERROR_EACCES; + } + const auto chunk_len = std::min(len, old_range.size - (vaddr - old_range.start)); + if (old_range.type == VirtualRangeType::PoolReserved) { + return chunk_len < len ? DecommitMemoryPoolRange(vaddr + chunk_len, len - chunk_len) : OK; + } + if (old_range.type != VirtualRangeType::Pooled) { + return KERNEL_ERROR_EACCES; + } + if (chunk_len < len) { + const int ret = DecommitMemoryPoolRange(vaddr, chunk_len); + return ret == OK ? DecommitMemoryPoolRange(vaddr + chunk_len, len - chunk_len) : ret; + } + + std::vector mappings; + if (!g_pooled_memory->Query(vaddr, len, &mappings)) { + return KERNEL_ERROR_EACCES; + } + + VirtualMemory::Mode mode = VirtualMemory::Mode::NoAccess; + GpuAccessMode decoded_gpu = GpuAccessMode::NoAccess; + if (!DecodeMemoryProtection(old_range.protection, &mode, &decoded_gpu)) { + return KERNEL_ERROR_EACCES; + } + if (!UnmapPooledBackingTransactional(mappings, mode)) { + EXIT("pooled-memory backing transaction failed after GPU unmap: addr=0x%016" PRIx64 + " size=0x%016" PRIx64 "\n", + vaddr, len); + } + + GpuAccessMode gpu_mode = GpuAccessMode::NoAccess; + if (!g_pooled_memory->Release(vaddr, len, &gpu_mode)) { + EXIT("failed to release decommitted pooled-memory range\n"); + } + + g_virtual_ranges->Remove(vaddr, len); + g_virtual_ranges->Add(vaddr, len, 0, 0, 0, VirtualRangeType::PoolReserved, old_range.name); + + if (g_free_callback != nullptr) { + g_free_callback(vaddr, len); + } + + MemoryPoolSubtractCommitted(len); + return OK; +} + int KYTY_SYSV_ABI KernelMemoryPoolDecommit(void* addr, size_t len, int flags) { PRINT_NAME(); @@ -4027,61 +4115,8 @@ int KYTY_SYSV_ABI KernelMemoryPoolDecommit(void* addr, size_t len, int flags) { scan = next; } - VirtualRanges::Range old_range {}; - if (!g_virtual_ranges->Query(vaddr, 0, &old_range)) { - return KERNEL_ERROR_EACCES; - } - const auto chunk_len = std::min(len, old_range.size - (vaddr - old_range.start)); - if (old_range.type == VirtualRangeType::PoolReserved) { - return chunk_len < len - ? KernelMemoryPoolDecommit(reinterpret_cast(vaddr + chunk_len), - len - chunk_len, flags) - : OK; - } - if (old_range.type != VirtualRangeType::Pooled) { - return KERNEL_ERROR_EACCES; - } - if (chunk_len < len) { - const int ret = KernelMemoryPoolDecommit(addr, chunk_len, flags); - return ret == OK ? KernelMemoryPoolDecommit(reinterpret_cast(vaddr + chunk_len), - len - chunk_len, flags) - : ret; - } - - std::vector mappings; - if (!g_pooled_memory->Query(vaddr, len, &mappings)) { - return KERNEL_ERROR_EACCES; - } - - VirtualMemory::Mode mode = VirtualMemory::Mode::NoAccess; - GpuAccessMode decoded_gpu = GpuAccessMode::NoAccess; - if (!DecodeMemoryProtection(old_range.protection, &mode, &decoded_gpu)) { - return KERNEL_ERROR_EACCES; - } - UnmapGpuRange(vaddr, len, decoded_gpu); - if (!UnmapPooledBackingTransactional(mappings, mode)) { - EXIT("pooled-memory backing transaction failed after GPU unmap: addr=0x%016" PRIx64 - " size=0x%016" PRIx64 "\n", - vaddr, len); - } - - GpuAccessMode gpu_mode = GpuAccessMode::NoAccess; - if (!g_pooled_memory->Release(vaddr, len, &gpu_mode)) { - EXIT("failed to release decommitted pooled-memory range\n"); - } - - g_virtual_ranges->Remove(vaddr, len); - old_range.placeholder_backed = true; - g_virtual_ranges->Add(vaddr, len, 0, 0, 0, VirtualRangeType::PoolReserved, old_range.name, - false, old_range.placeholder_backed); - - if (g_free_callback != nullptr) { - g_free_callback(vaddr, len); - } - - MemoryPoolSubtractCommitted(len); - - return OK; + UnmapGpuRange(vaddr, len); + return DecommitMemoryPoolRange(vaddr, len); } int KYTY_SYSV_ABI KernelMemoryPoolBatch(const KernelMemoryPoolBatchEntry* entries, int num_entries, diff --git a/src/kernel/memory.h b/src/kernel/memory.h index 7c71402..775192e 100644 --- a/src/kernel/memory.h +++ b/src/kernel/memory.h @@ -145,7 +145,7 @@ int KYTY_SYSV_ABI KernelIsStack(void* addr, void** start, void** end); int KYTY_SYSV_ABI KernelReserveVirtualRange(void** addr, size_t len, int flags, size_t alignment); bool KernelHandleReservedRangeAccessViolation(uint64_t vaddr); int KYTY_SYSV_ABI KernelAvailableFlexibleMemorySize(size_t* size); -int KYTY_SYSV_ABI KernelConfiguredFlexibleMemorySize(uint64_t* size); +int KYTY_SYSV_ABI KernelConfiguredFlexibleMemorySize(size_t* size); int KYTY_SYSV_ABI KernelMprotect(const void* addr, size_t len, int prot); int KYTY_SYSV_ABI KernelMtypeprotect(const void* addr, size_t len, int type, int prot); int KYTY_SYSV_ABI KernelBatchMap(KernelBatchMapEntry* entries, int num_entries, @@ -163,17 +163,30 @@ int KYTY_SYSV_ABI KernelMemoryPoolBatch(const KernelMemoryPoolBatchEntry* entrie int KYTY_SYSV_ABI KernelMemoryPoolGetBlockStats(KernelMemoryPoolBlockStats* output, size_t output_size); -void RegisterProgramMemory(uint64_t vaddr, uint64_t size, Common::VirtualMemory::Mode mode, - const char* name); -void UpdateProgramMemoryProtection(uint64_t vaddr, uint64_t size, Common::VirtualMemory::Mode mode); -void UnregisterProgramMemory(uint64_t vaddr, uint64_t size); +uint64_t AllocateProgramMemory(uint64_t search_addr, uint64_t size, + Common::VirtualMemory::Mode mode, const char* name); +void SetProgramMemoryProtection(uint64_t vaddr, uint64_t size, Common::VirtualMemory::Mode mode); +uint64_t AllocateRuntimeMemory(uint64_t search_addr, uint64_t size, + Common::VirtualMemory::Mode mode, const char* name, + bool fixed = false); +uint64_t AllocateGuestStackMemory(uint64_t search_addr, uint64_t size, + Common::VirtualMemory::Mode mode, const char* name); +bool ProtectGuestMemory(uint64_t vaddr, uint64_t size, Common::VirtualMemory::Mode mode, + Common::VirtualMemory::Mode* old_mode = nullptr); +// Transient PageManager watch state; does not change the guest mapping's semantic protection. +bool ProtectGuestHostMemory(uint64_t vaddr, uint64_t size, Common::VirtualMemory::Mode mode); +bool FreeGuestMemory(uint64_t vaddr, uint64_t size); #if defined(KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS) -void TestFailNextPhysicalMemoryUnmap(); -void TestFailPhysicalMemoryUnmapAfter(uint32_t successful_unmaps); -void TestFailHostReservationAfter(uint32_t successful_pages); -void TestFailNextFixedReserveRangeRegistration(); -bool TestPlaceholderRangeIsFree(uint64_t vaddr, uint64_t size); +void TestFailNextPhysicalMemoryUnmap(); +void TestFailPhysicalMemoryUnmapAfter(uint32_t successful_unmaps); +void TestFailGuestBackingStoreUnmapAfter(uint32_t successful_unmaps); +void TestFailNextFixedReserveRangeRegistration(); +bool TestPlaceholderRangeIsFree(uint64_t vaddr, uint64_t size); +bool TestGuestAddressRangeIsOwned(uint64_t vaddr, uint64_t size); +bool TestGuestBackingOutsideAddressSpace(); +uint64_t TestGuestBackingSize(); +bool TestGuestFreeRangeBounds(); #endif } // namespace Libs::LibKernel::Memory diff --git a/src/kernel/memoryAddressSpace.inc b/src/kernel/memoryAddressSpace.inc index 3340202..9fde561 100644 --- a/src/kernel/memoryAddressSpace.inc +++ b/src/kernel/memoryAddressSpace.inc @@ -1,40 +1,42 @@ // Internal memory address-space helpers included by memory.cpp. // Kept in the same translation unit because they depend on memory.cpp-local helpers. -#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS -static constexpr uint64_t HOST_SYSTEM_MANAGED_MIN = 0x0000040000ull; -static constexpr uint64_t HOST_SYSTEM_MANAGED_MAX = 0x07fffeffffull; -static constexpr uint64_t HOST_USER_MIN = 0x1000000000ull; -static constexpr uint64_t HOST_USER_MAX = 0xfbffffffffull; +static constexpr uint64_t HOST_SYSTEM_MANAGED_MIN = 0x0000040000ull; +static constexpr uint64_t HOST_SYSTEM_MANAGED_MAX = 0x07ffffbfffull; +static constexpr uint64_t HOST_SYSTEM_RESERVED_MIN = HOST_SYSTEM_MANAGED_MAX + 1u; +#if defined(__APPLE__) && (defined(__x86_64__) || defined(_M_X64)) +static constexpr uint64_t HOST_SYSTEM_RESERVED_MAX = 0x0fbfffffffull; +static constexpr uint64_t HOST_USER_MIN = 0x7000000000ull; +#else +static constexpr uint64_t HOST_SYSTEM_RESERVED_MAX = 0x0fffffffffull; +static constexpr uint64_t HOST_USER_MIN = 0x1000000000ull; #endif +static constexpr uint64_t HOST_USER_MAX = 0xfbffffffffull; -class DirectMemoryBacking { +static bool GuestFreeRangeContains(uint64_t base, uint64_t range_size, uint64_t candidate, + uint64_t size) { + if (UINT64_MAX - base < range_size || candidate < base) { + return false; + } + const auto range_end = base + range_size; + return candidate <= range_end && size <= range_end - candidate; +} + +class GuestBackingStore { public: enum class FailureReason { None, BackingUnavailable, PhysicalRangeOutOfBounds, - HostAddressNotGranularityAligned, - SizeNotGranularityAligned, - ReserveAlignedFailed, FixedMapViewFailed, - PlaceholderReserveFailed, PlaceholderMapFailed, ProtectFailed, - ReservedContainerHasCommittedPages, - ReservedContainerReleaseFailed, - ReservedContainerRestoreFailed, - ReservedContainerSplitNotGranularityAligned, WrongHostAddress, - UnmapRollbackFailed, + AddressRangeUnavailable, }; - explicit DirectMemoryBacking(uint64_t size): m_size(size) { + explicit GuestBackingStore(uint64_t size): m_size(size) { #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS - SYSTEM_INFO info {}; - GetSystemInfo(&info); - m_granularity = info.dwAllocationGranularity; - m_handle = CreateFileMappingA(INVALID_HANDLE_VALUE, nullptr, PAGE_EXECUTE_READWRITE, static_cast(m_size >> 32u), static_cast(m_size & 0xffffffffu), nullptr); @@ -90,8 +92,12 @@ public: #endif } - ~DirectMemoryBacking() { + ~GuestBackingStore() { #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + for (const auto& [_, entry]: m_maps) { + UnmapViewPreservePlaceholder(reinterpret_cast(entry.vaddr), entry.size); + } + m_maps.clear(); if (m_backing_base != nullptr) { UnmapViewOfFile(m_backing_base); } @@ -99,6 +105,11 @@ public: CloseHandle(m_handle); } #elif KYTY_PLATFORM == KYTY_PLATFORM_LINUX + for (const auto& [_, entry]: m_maps) { + EXIT_IF( + !UnmapViewPreservePlaceholder(reinterpret_cast(entry.vaddr), entry.size)); + } + m_maps.clear(); if (m_backing_base != nullptr) { munmap(m_backing_base, m_size); } @@ -108,31 +119,18 @@ public: #endif } - KYTY_CLASS_NO_COPY(DirectMemoryBacking); + KYTY_CLASS_NO_COPY(GuestBackingStore); static const char* GetFailureReasonName(FailureReason reason) { switch (reason) { case FailureReason::None: return "n/a"; case FailureReason::BackingUnavailable: return "backing-unavailable"; case FailureReason::PhysicalRangeOutOfBounds: return "physical-range-out-of-bounds"; - case FailureReason::HostAddressNotGranularityAligned: - return "host-address-not-64k-aligned"; - case FailureReason::SizeNotGranularityAligned: return "size-not-64k-aligned"; - case FailureReason::ReserveAlignedFailed: return "reserve-aligned-failed"; case FailureReason::FixedMapViewFailed: return "fixed-map-view-failed"; - case FailureReason::PlaceholderReserveFailed: return "placeholder-reserve-failed"; case FailureReason::PlaceholderMapFailed: return "placeholder-map-failed"; case FailureReason::ProtectFailed: return "protect-failed"; - case FailureReason::ReservedContainerHasCommittedPages: - return "reserved-container-has-committed-pages"; - case FailureReason::ReservedContainerReleaseFailed: - return "reserved-container-release-failed"; - case FailureReason::ReservedContainerRestoreFailed: - return "reserved-container-restore-failed"; - case FailureReason::ReservedContainerSplitNotGranularityAligned: - return "reserved-container-split-not-64k-aligned"; case FailureReason::WrongHostAddress: return "wrong-host-address"; - case FailureReason::UnmapRollbackFailed: return "unmap-rollback-failed"; + case FailureReason::AddressRangeUnavailable: return "address-range-unavailable"; default: return "unknown"; } } @@ -147,6 +145,20 @@ public: #endif } + [[nodiscard]] uint64_t GetBackingBase() const { + return reinterpret_cast(m_backing_base); + } + + [[nodiscard]] uint64_t GetBackingSize() const { return m_size; } + + bool ZeroRange(uint64_t backing_offset, uint64_t size) { + if (!IsAvailable() || !RangeOk(backing_offset, size)) { + return false; + } + std::memset(m_backing_base + backing_offset, 0, size); + return true; + } + bool TryWriteBacking(uint64_t vaddr, const void* data, uint64_t size) { if (data == nullptr) { return false; @@ -169,123 +181,6 @@ public: }); } - uint64_t MapAligned(uint64_t search_addr, uint64_t size, uint64_t backing_offset, - VirtualMemory::Mode mode, uint64_t alignment, - FailureReason* failure_reason = nullptr) { - if (!IsAvailable() || !RangeOk(backing_offset, size)) { - SetFailure(failure_reason, !IsAvailable() ? FailureReason::BackingUnavailable - : FailureReason::PhysicalRangeOutOfBounds); - return 0; - } - - if (alignment == 0) { - alignment = 0x4000; - } -#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS - alignment = std::max(alignment, m_granularity); - return MapNewPlaceholderAligned(search_addr, size, backing_offset, mode, alignment, - failure_reason); -#else - auto reserved = VirtualMemory::ReserveAligned(search_addr, size, alignment); - if (reserved == 0) { - SetFailure(failure_reason, FailureReason::ReserveAlignedFailed); - return 0; - } -#if !defined(__APPLE__) - // On macOS the PROT_NONE reservation is left in place: MapView's MAP_FIXED - // replaces it atomically, so the range never becomes a hole that dyld, - // Rosetta, or Metal background threads could claim (which the subsequent - // MAP_FIXED would then silently destroy). - if (!ReleaseReservedProbe(reserved, failure_reason)) { - return 0; - } -#endif - if (MapFixed(reserved, size, backing_offset, mode, failure_reason)) { - return reserved; - } - - return 0; -#endif - } - - bool MapFixed(uint64_t vaddr, uint64_t size, uint64_t backing_offset, VirtualMemory::Mode mode, - FailureReason* failure_reason = nullptr) { -#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS - if (ShouldChunkMap(size) && IsGranularityAligned(vaddr) && - IsGranularityAligned(backing_offset)) { - return MapFixedChunked(vaddr, size, backing_offset, mode, failure_reason); - } -#endif - - if (!IsAvailable() || !RangeOk(backing_offset, size)) { - SetFailure(failure_reason, !IsAvailable() ? FailureReason::BackingUnavailable - : FailureReason::PhysicalRangeOutOfBounds); - return false; - } - - void* mapped = MapView(vaddr, size, backing_offset, mode, failure_reason); -#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS - if (mapped == nullptr && ReleaseExactHostAllocation(vaddr, size)) { - mapped = MapView(vaddr, size, backing_offset, mode, failure_reason); - } - if (mapped == nullptr) { - ReleasedReservedHostAllocation released {}; - if (ReleaseReservedHostAllocationForMap(vaddr, size, &released, failure_reason)) { - mapped = MapView(vaddr, size, backing_offset, mode, failure_reason); - if (mapped == nullptr || reinterpret_cast(mapped) != vaddr) { - if (mapped != nullptr) { - UnmapView(mapped, size); - mapped = nullptr; - } - RestoreReleasedReservedHostAllocation(released); - } else if (!ReserveReleasedReservedHostRemainders(released)) { - UnmapView(mapped, size); - mapped = nullptr; - RestoreReleasedReservedHostAllocation(released); - SetFailure(failure_reason, FailureReason::ReservedContainerRestoreFailed); - } - } - } - if (mapped == nullptr) { - std::vector released_ranges; - if (ReleaseReservedHostAllocationsForMap(vaddr, size, &released_ranges, - failure_reason)) { - mapped = MapView(vaddr, size, backing_offset, mode, failure_reason); - if (mapped == nullptr || reinterpret_cast(mapped) != vaddr) { - if (mapped != nullptr) { - UnmapView(mapped, size); - mapped = nullptr; - } - RestoreReleasedReservedHostAllocations(released_ranges); - } else if (!ReserveReleasedReservedHostRemainders(released_ranges)) { - UnmapView(mapped, size); - mapped = nullptr; - RestoreReleasedReservedHostAllocations(released_ranges); - SetFailure(failure_reason, FailureReason::ReservedContainerRestoreFailed); - } - } - } -#endif - if (mapped == nullptr || reinterpret_cast(mapped) != vaddr) { - if (mapped != nullptr) { - if (!UnmapView(mapped, size)) { - SetFailure(failure_reason, FailureReason::UnmapRollbackFailed); - } - } - if (mapped != nullptr) { - SetFailure(failure_reason, FailureReason::WrongHostAddress); - } - return false; - } - - { - std::lock_guard lock(m_mutex); - EXIT_IF(!m_maps.emplace(vaddr, MapEntry {vaddr, size, backing_offset, mode}).second); - } - SetFailure(failure_reason, FailureReason::None); - return true; - } - bool MapExistingPlaceholderFixed(uint64_t vaddr, uint64_t size, uint64_t backing_offset, VirtualMemory::Mode mode, FailureReason* failure_reason = nullptr) { @@ -299,7 +194,7 @@ public: void* mapped = ReplacePlaceholderView(vaddr, size, backing_offset, mode, failure_reason); if (mapped == nullptr || reinterpret_cast(mapped) != vaddr) { if (mapped != nullptr) { - UnmapView(mapped, size); + UnmapViewPreservePlaceholder(mapped, size); SetFailure(failure_reason, FailureReason::WrongHostAddress); } return false; @@ -312,12 +207,20 @@ public: SetFailure(failure_reason, FailureReason::None); return true; #else - return MapFixed(vaddr, size, backing_offset, mode, failure_reason); + void* mapped = MapView(vaddr, size, backing_offset, mode, failure_reason); + if (mapped == nullptr || reinterpret_cast(mapped) != vaddr) { + return false; + } + { + std::lock_guard lock(m_mutex); + EXIT_IF(!m_maps.emplace(vaddr, MapEntry {vaddr, size, backing_offset, mode}).second); + } + SetFailure(failure_reason, FailureReason::None); + return true; #endif } - bool Unmap(uint64_t vaddr, uint64_t size, bool preserve_placeholder = false, - bool* placeholder_preserved = nullptr) { + bool Unmap(uint64_t vaddr, uint64_t size, bool* placeholder_preserved = nullptr) { if (size == 0 || UINT64_MAX - vaddr < size) { return false; } @@ -326,37 +229,58 @@ public: *placeholder_preserved = false; } - if (!Contains(vaddr, size)) { - return false; - } - - const auto end = vaddr + size; - auto current = vaddr; - while (current < end) { - MapEntry entry {}; - { - std::lock_guard lock(m_mutex); + const auto end = vaddr + size; + std::vector targets; + { + std::lock_guard lock(m_mutex); + auto current = vaddr; + while (current < end) { + MapEntry entry {}; if (!FindContainingUnlocked(current, 1, &entry)) { return false; } + const auto part_size = std::min(end, entry.vaddr + entry.size) - current; + targets.push_back( + {current, part_size, entry.backing_offset + current - entry.vaddr, entry.mode}); + current += part_size; } - - const auto entry_end = entry.vaddr + entry.size; - const auto part_size = std::min(end, entry_end) - current; - const bool preserve_part_placeholder = - preserve_placeholder && current == vaddr && part_size == size; - if (!UnmapOne(current, part_size, preserve_part_placeholder, placeholder_preserved)) { - return false; - } - - current += part_size; } + std::vector removed; + removed.reserve(targets.size()); + for (const auto& entry: targets) { + bool part_placeholder_preserved = false; + if (!UnmapOne(entry.vaddr, entry.size, &part_placeholder_preserved)) { + for (auto it = removed.rbegin(); it != removed.rend(); ++it) { + EXIT_IF(!MapExistingPlaceholderFixed(it->vaddr, it->size, it->backing_offset, + it->mode)); + } + if (placeholder_preserved != nullptr) { + *placeholder_preserved = false; + } + return false; + } + EXIT_IF(!part_placeholder_preserved); + removed.push_back(entry); + } + + if (placeholder_preserved != nullptr) { + *placeholder_preserved = true; + } return true; } - bool UnmapOne(uint64_t vaddr, uint64_t size, bool preserve_placeholder = false, - bool* placeholder_preserved = nullptr) { + bool UnmapOne(uint64_t vaddr, uint64_t size, bool* placeholder_preserved = nullptr) { +#if defined(KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS) + if (g_test_backing_store_unmaps_before_failure == 0) { + g_test_backing_store_unmaps_before_failure = UINT32_MAX; + return false; + } + if (g_test_backing_store_unmaps_before_failure != UINT32_MAX) { + g_test_backing_store_unmaps_before_failure--; + } +#endif + MapEntry old {}; { std::lock_guard lock(m_mutex); @@ -377,20 +301,15 @@ public: const bool exact_unmap = (vaddr == old.vaddr && size == old.size); if (exact_unmap) { const bool unmapped = - preserve_placeholder - ? UnmapViewPreservePlaceholder(reinterpret_cast(old.vaddr), old.size) - : UnmapView(reinterpret_cast(old.vaddr), old.size); + UnmapViewPreservePlaceholder(reinterpret_cast(old.vaddr), old.size); if (!unmapped) { std::lock_guard lock(m_mutex); m_maps.emplace(old.vaddr, old); return false; } -#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS - if (preserve_placeholder && placeholder_preserved != nullptr) { + if (placeholder_preserved != nullptr) { *placeholder_preserved = true; } -#endif - // Only Windows preserves placeholders. return true; } @@ -425,47 +344,57 @@ public: } return true; } - } else -#endif - { - if (!UnmapView(reinterpret_cast(old.vaddr), old.size)) { - std::lock_guard lock(m_mutex); - m_maps.emplace(old.vaddr, old); - return false; + + if (left_size != 0 && Contains(old.vaddr, left_size)) { + EXIT_IF(!UnmapOne(old.vaddr, left_size)); } + if (right_size != 0 && Contains(right_addr, right_size)) { + EXIT_IF(!UnmapOne(right_addr, right_size)); + } + EXIT_IF(VirtualFree(reinterpret_cast(old.vaddr), old.size, + MEM_RELEASE | MEM_COALESCE_PLACEHOLDERS) == 0); + EXIT_IF( + !MapExistingPlaceholderFixed(old.vaddr, old.size, old.backing_offset, old.mode)); + return false; + } + std::lock_guard lock(m_mutex); + m_maps.emplace(old.vaddr, old); + return false; +#else + if (!UnmapViewPreservePlaceholder(reinterpret_cast(old.vaddr), old.size)) { + std::lock_guard lock(m_mutex); + m_maps.emplace(old.vaddr, old); + return false; } bool ok = true; if (left_size != 0) { - ok = MapFixed(old.vaddr, left_size, old.backing_offset, old.mode) && ok; + ok = MapExistingPlaceholderFixed(old.vaddr, left_size, old.backing_offset, old.mode) && + ok; } if (right_size != 0) { - ok = MapFixed(right_addr, right_size, old.backing_offset + (right_addr - old.vaddr), - old.mode) && + ok = MapExistingPlaceholderFixed(right_addr, right_size, + old.backing_offset + (right_addr - old.vaddr), + old.mode) && ok; } if (!ok) { - if (left_size != 0) { - UnmapOne(old.vaddr, left_size); + if (left_size != 0 && Contains(old.vaddr, left_size)) { + EXIT_IF(!UnmapOne(old.vaddr, left_size)); } - if (right_size != 0) { - UnmapOne(right_addr, right_size); + if (right_size != 0 && Contains(right_addr, right_size)) { + EXIT_IF(!UnmapOne(right_addr, right_size)); } - MapFixed(old.vaddr, old.size, old.backing_offset, old.mode); + EXIT_IF( + !MapExistingPlaceholderFixed(old.vaddr, old.size, old.backing_offset, old.mode)); return false; } - return ok; - } - - void UpdateProtection(uint64_t vaddr, uint64_t size, VirtualMemory::Mode mode) { - std::lock_guard lock(m_mutex); - for (auto& map_entry: m_maps) { - auto& entry = map_entry.second; - if (VirtualRangesOverlap(vaddr, size, entry.vaddr, entry.size)) { - entry.mode = mode; - } + if (placeholder_preserved != nullptr) { + *placeholder_preserved = true; } + return ok; +#endif } bool Contains(uint64_t vaddr, uint64_t size) { @@ -492,45 +421,6 @@ public: return true; } - bool SelfTest() { - const uint64_t test_size = GetMappableTestSize(); - if (test_size == 0 || !RangeOk(0, test_size)) { - LOGF_COLOR(Log::Color::Yellow, "\t direct-memory backing self-test skipped\n"); - return false; - } - - const auto first = MapAligned(0, test_size, 0, VirtualMemory::Mode::ReadWrite, test_size); - if (first == 0) { - LOGF_COLOR(Log::Color::Yellow, - "\t direct-memory backing self-test skipped: first alias unavailable\n"); - return false; - } - - const auto second = MapAligned(0, test_size, 0, VirtualMemory::Mode::ReadWrite, test_size); - if (second == 0) { - Unmap(first, test_size); - LOGF_COLOR(Log::Color::Yellow, - "\t direct-memory backing self-test skipped: second alias unavailable\n"); - return false; - } - - auto* second_ptr = reinterpret_cast(second); - - constexpr uint64_t test_value = 0x4b595459414c4941ull; // "KYTYALIA" - EXIT_IF(!VirtualMemory::Protect(first, test_size, VirtualMemory::Mode::NoAccess)); - const bool ok = - TryWriteBacking(first, &test_value, sizeof(test_value)) && *second_ptr == test_value; - const uint64_t clear = 0; - EXIT_IF(!TryWriteBacking(first, &clear, sizeof(clear))); - - Unmap(second, test_size); - Unmap(first, test_size); - - LOGF_COLOR(ok ? Log::Color::Green : Log::Color::Red, - "\t direct-memory backing self-test: %s\n", ok ? "shared aliases ok" : "failed"); - return ok; - } - private: template bool TryTransferBacking(uint64_t vaddr, uint64_t size, Transfer transfer) { @@ -602,138 +492,7 @@ private: } } -#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS - static bool ReleaseReservedProbe(uint64_t vaddr, FailureReason* failure_reason) { - if (VirtualMemory::Free(vaddr)) { - return true; - } - SetFailure(failure_reason, FailureReason::ReservedContainerReleaseFailed); - return false; - } -#endif - #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS - [[nodiscard]] uint64_t GetMappableTestSize() const { return m_granularity; } - static constexpr uint64_t PageSize() { return 0x4000; } - - [[nodiscard]] static bool IsPageAligned(uint64_t value) { - return (value & (PageSize() - 1u)) == 0; - } - - [[nodiscard]] bool IsGranularityAligned(uint64_t value) const { - return (value & (m_granularity - 1u)) == 0; - } - - [[nodiscard]] uint64_t GetMapChunkSize() const { - return std::max(m_granularity, 0x200000ull); - } - - [[nodiscard]] bool ShouldChunkMap(uint64_t size) const { return size > GetMapChunkSize(); } - - uint64_t MapNewPlaceholderAligned(uint64_t search_addr, uint64_t size, uint64_t backing_offset, - VirtualMemory::Mode mode, uint64_t alignment, - FailureReason* failure_reason) { - MEM_ADDRESS_REQUIREMENTS requirements {}; - MEM_EXTENDED_PARAMETER parameter {}; - const auto aligned_search = (search_addr + alignment - 1u) & ~(alignment - 1u); - requirements.LowestStartingAddress = - reinterpret_cast(search_addr != 0 ? aligned_search : HOST_SYSTEM_MANAGED_MIN); - requirements.HighestEndingAddress = - reinterpret_cast(search_addr != 0 ? HOST_USER_MAX : HOST_SYSTEM_MANAGED_MAX); - requirements.Alignment = alignment; - parameter.Type = MemExtendedParameterAddressRequirements; - parameter.Pointer = &requirements; - - auto reserve = [&]() { - return VirtualAlloc2(GetCurrentProcess(), nullptr, size, - MEM_RESERVE | MEM_RESERVE_PLACEHOLDER, PAGE_NOACCESS, ¶meter, - 1); - }; - auto* placeholder = reserve(); - if (placeholder == nullptr && search_addr == 0) { - requirements.LowestStartingAddress = reinterpret_cast(HOST_USER_MIN); - requirements.HighestEndingAddress = reinterpret_cast(HOST_USER_MAX); - placeholder = reserve(); - } - if (placeholder == nullptr) { - SetFailure(failure_reason, FailureReason::PlaceholderReserveFailed); - return 0; - } - - const auto vaddr = reinterpret_cast(placeholder); - std::vector entries; - uint64_t mapped = 0; - while (mapped < size) { - const auto current = vaddr + mapped; - const auto remaining = size - mapped; - const auto part_size = std::min(GetMapChunkSize(), remaining); - const bool split = part_size != remaining; - if (split && !SplitPlaceholder(current, part_size)) { - for (const auto& entry: entries) { - UnmapView(reinterpret_cast(entry.vaddr), entry.size); - } - VirtualFree(reinterpret_cast(current), 0, MEM_RELEASE); - SetFailure(failure_reason, FailureReason::PlaceholderReserveFailed); - return 0; - } - if (ReplacePlaceholderView(current, part_size, backing_offset + mapped, mode, - failure_reason) == nullptr) { - for (const auto& entry: entries) { - UnmapView(reinterpret_cast(entry.vaddr), entry.size); - } - VirtualFree(reinterpret_cast(current), 0, MEM_RELEASE); - if (split) { - VirtualFree(reinterpret_cast(current + part_size), 0, MEM_RELEASE); - } - return 0; - } - entries.push_back(MapEntry {current, part_size, backing_offset + mapped, mode}); - mapped += part_size; - } - - { - std::lock_guard lock(m_mutex); - for (const auto& entry: entries) { - EXIT_IF(!m_maps.emplace(entry.vaddr, entry).second); - } - } - SetFailure(failure_reason, FailureReason::None); - return vaddr; - } - - bool MapFixedChunked(uint64_t vaddr, uint64_t size, uint64_t backing_offset, - VirtualMemory::Mode mode, FailureReason* failure_reason) { - if (!IsAvailable() || !RangeOk(backing_offset, size)) { - SetFailure(failure_reason, !IsAvailable() ? FailureReason::BackingUnavailable - : FailureReason::PhysicalRangeOutOfBounds); - return false; - } - if (!IsGranularityAligned(vaddr) || !IsGranularityAligned(backing_offset)) { - SetFailure(failure_reason, FailureReason::HostAddressNotGranularityAligned); - return false; - } - - const auto chunk_size = GetMapChunkSize(); - uint64_t mapped = 0; - while (mapped < size) { - const auto part_size = std::min(chunk_size, size - mapped); - if (!MapFixed(vaddr + mapped, part_size, backing_offset + mapped, mode, - failure_reason)) { - uint64_t rollback = 0; - while (rollback < mapped) { - const auto rollback_size = std::min(chunk_size, mapped - rollback); - UnmapOne(vaddr + rollback, rollback_size); - rollback += rollback_size; - } - return false; - } - mapped += part_size; - } - - SetFailure(failure_reason, FailureReason::None); - return true; - } - static DWORD GetProtect(VirtualMemory::Mode mode) { switch (mode) { case VirtualMemory::Mode::Read: return PAGE_READONLY; @@ -761,51 +520,6 @@ private: MEM_RELEASE | MEM_PRESERVE_PLACEHOLDER) != 0; } - static bool ReleaseExactHostAllocation(uint64_t vaddr, uint64_t size) { - MEMORY_BASIC_INFORMATION info {}; - if (VirtualQuery(reinterpret_cast(vaddr), &info, sizeof(info)) == 0) { - return false; - } - - if ((info.State != MEM_RESERVE && info.State != MEM_COMMIT) || - reinterpret_cast(info.BaseAddress) != vaddr || - reinterpret_cast(info.AllocationBase) != vaddr || info.RegionSize != size) { - return false; - } - - return VirtualFree(reinterpret_cast(vaddr), 0, MEM_RELEASE) != 0; - } - - struct ReleasedReservedHostAllocation { - uint64_t base = 0; - uint64_t size = 0; - uint64_t map_start = 0; - uint64_t map_size = 0; - }; - - static bool ReserveHostRange(uint64_t vaddr, uint64_t size) { - if (size == 0) { - return true; - } - - auto* ptr = VirtualAlloc(reinterpret_cast(vaddr), size, MEM_RESERVE, PAGE_NOACCESS); - return ptr != nullptr && reinterpret_cast(ptr) == vaddr; - } - - static bool - RestoreReleasedReservedHostAllocation(const ReleasedReservedHostAllocation& released) { - return ReserveHostRange(released.base, released.size); - } - - static bool RestoreReleasedReservedHostAllocations( - const std::vector& released_ranges) { - bool ok = true; - for (const auto& released: released_ranges) { - ok = RestoreReleasedReservedHostAllocation(released) && ok; - } - return ok; - } - void* ReplacePlaceholderView(uint64_t vaddr, uint64_t size, uint64_t backing_offset, VirtualMemory::Mode mode, FailureReason* failure_reason) { const DWORD map_protect = @@ -815,233 +529,25 @@ private: backing_offset, size, MEM_REPLACE_PLACEHOLDER, map_protect, nullptr, 0); if (ptr == nullptr || reinterpret_cast(ptr) != vaddr) { if (ptr != nullptr) { - UnmapViewOfFile(ptr); + UnmapViewOfFile2(GetCurrentProcess(), ptr, MEM_PRESERVE_PLACEHOLDER); } SetFailure(failure_reason, FailureReason::PlaceholderMapFailed); return nullptr; } if (!ApplyProtection(ptr, size, mode)) { - UnmapViewOfFile(ptr); + UnmapViewOfFile2(GetCurrentProcess(), ptr, MEM_PRESERVE_PLACEHOLDER); SetFailure(failure_reason, FailureReason::ProtectFailed); return nullptr; } return ptr; } - static bool - ReserveReleasedReservedHostRemainders(const ReleasedReservedHostAllocation& released) { - const auto map_end = released.map_start + released.map_size; - const auto released_end = released.base + released.size; - const auto left_size = released.map_start - released.base; - const auto right_size = released_end - map_end; - - if (!ReserveHostRange(released.base, left_size)) { - return false; - } - if (!ReserveHostRange(map_end, right_size)) { - if (left_size != 0) { - VirtualFree(reinterpret_cast(released.base), 0, MEM_RELEASE); - } - return false; - } - return true; - } - - static bool ReserveReleasedReservedHostRemainders( - const std::vector& released_ranges) { - bool ok = true; - for (const auto& released: released_ranges) { - ok = ReserveReleasedReservedHostRemainders(released) && ok; - } - return ok; - } - - bool QueryReservedHostAllocation(uint64_t vaddr, ReleasedReservedHostAllocation* out, - FailureReason* failure_reason) const { - EXIT_IF(out == nullptr); - - MEMORY_BASIC_INFORMATION info {}; - if (VirtualQuery(reinterpret_cast(vaddr), &info, sizeof(info)) == 0) { - SetFailure(failure_reason, FailureReason::PlaceholderReserveFailed); - return false; - } - - const auto allocation_base = reinterpret_cast(info.AllocationBase); - const auto query_base = reinterpret_cast(info.BaseAddress); - if (info.State != MEM_RESERVE || query_base > vaddr || allocation_base > vaddr) { - SetFailure(failure_reason, FailureReason::PlaceholderReserveFailed); - return false; - } - - uint64_t allocation_end = query_base + info.RegionSize; - uint64_t scan = allocation_end; - while (true) { - MEMORY_BASIC_INFORMATION next {}; - if (VirtualQuery(reinterpret_cast(scan), &next, sizeof(next)) == 0 || - reinterpret_cast(next.AllocationBase) != allocation_base || - next.State == MEM_FREE) { - break; - } - if (next.State != MEM_RESERVE) { - SetFailure(failure_reason, FailureReason::ReservedContainerHasCommittedPages); - return false; - } - allocation_end = reinterpret_cast(next.BaseAddress) + next.RegionSize; - scan = allocation_end; - } - - scan = allocation_base; - while (scan < allocation_end) { - MEMORY_BASIC_INFORMATION part {}; - if (VirtualQuery(reinterpret_cast(scan), &part, sizeof(part)) == 0 || - reinterpret_cast(part.AllocationBase) != allocation_base || - part.State != MEM_RESERVE) { - SetFailure(failure_reason, FailureReason::ReservedContainerHasCommittedPages); - return false; - } - scan = reinterpret_cast(part.BaseAddress) + part.RegionSize; - } - - out->base = allocation_base; - out->size = allocation_end - allocation_base; - out->map_start = 0; - out->map_size = 0; - return true; - } - - bool ReleaseReservedHostAllocationForMap(uint64_t vaddr, uint64_t size, - ReleasedReservedHostAllocation* out, - FailureReason* failure_reason) const { - if (!QueryReservedHostAllocation(vaddr, out, failure_reason)) { - return false; - } - - const auto allocation_end = out->base + out->size; - if (vaddr < out->base || vaddr + size > allocation_end) { - SetFailure(failure_reason, FailureReason::PlaceholderReserveFailed); - return false; - } - if (out->base == vaddr && out->size == size) { - return false; - } - if (!IsGranularityAligned(size)) { - SetFailure(failure_reason, FailureReason::ReservedContainerSplitNotGranularityAligned); - return false; - } - if (VirtualFree(reinterpret_cast(out->base), 0, MEM_RELEASE) == 0) { - SetFailure(failure_reason, FailureReason::ReservedContainerReleaseFailed); - return false; - } - - out->map_start = vaddr; - out->map_size = size; - return true; - } - - bool ReleaseReservedHostAllocationsForMap(uint64_t vaddr, uint64_t size, - std::vector* out, - FailureReason* failure_reason) const { - EXIT_IF(out == nullptr); - - out->clear(); - if (size == 0 || !IsPageAligned(vaddr) || !IsPageAligned(size)) { - SetFailure(failure_reason, FailureReason::ReservedContainerSplitNotGranularityAligned); - return false; - } - - const auto target_end = vaddr + size; - auto current = vaddr; - while (current < target_end) { - ReleasedReservedHostAllocation released {}; - if (!QueryReservedHostAllocation(current, &released, failure_reason)) { - out->clear(); - return false; - } - - const auto map_start = std::max(vaddr, released.base); - const auto map_end = std::min(target_end, released.base + released.size); - if (map_start >= map_end || !IsPageAligned(map_start) || - !IsPageAligned(map_end - map_start)) { - SetFailure(failure_reason, - FailureReason::ReservedContainerSplitNotGranularityAligned); - out->clear(); - return false; - } - - released.map_start = map_start; - released.map_size = map_end - map_start; - out->push_back(released); - current = map_end; - } - - for (size_t i = 0; i < out->size(); i++) { - const auto& released = (*out)[i]; - if (VirtualFree(reinterpret_cast(released.base), 0, MEM_RELEASE) == 0) { - for (size_t restore_index = 0; restore_index < i; restore_index++) { - RestoreReleasedReservedHostAllocation((*out)[restore_index]); - } - SetFailure(failure_reason, FailureReason::ReservedContainerReleaseFailed); - out->clear(); - return false; - } - } - - return !out->empty(); - } - - void* MapPlaceholderView(uint64_t vaddr, uint64_t size, uint64_t backing_offset, - VirtualMemory::Mode mode, FailureReason* failure_reason) { - void* placeholder = - VirtualAlloc2(GetCurrentProcess(), reinterpret_cast(vaddr), size, - MEM_RESERVE | MEM_RESERVE_PLACEHOLDER, PAGE_NOACCESS, nullptr, 0); - if (placeholder == nullptr || reinterpret_cast(placeholder) != vaddr) { - if (placeholder != nullptr) { - VirtualFree(placeholder, 0, MEM_RELEASE); - } - SetFailure(failure_reason, FailureReason::PlaceholderReserveFailed); - return nullptr; - } - - auto* ptr = ReplacePlaceholderView(vaddr, size, backing_offset, mode, failure_reason); - if (ptr == nullptr) { - VirtualFree(reinterpret_cast(vaddr), 0, MEM_RELEASE); - return nullptr; - } - return ptr; - } - - void* MapView(uint64_t vaddr, uint64_t size, uint64_t backing_offset, VirtualMemory::Mode mode, - FailureReason* failure_reason) { - constexpr DWORD access = FILE_MAP_READ | FILE_MAP_WRITE | FILE_MAP_EXECUTE; - void* ptr = MapViewOfFileEx(m_handle, access, static_cast(backing_offset >> 32u), - static_cast(backing_offset & 0xffffffffu), size, - reinterpret_cast(vaddr)); - if (ptr == nullptr) { - SetFailure(failure_reason, FailureReason::FixedMapViewFailed); - ptr = ReplacePlaceholderView(vaddr, size, backing_offset, mode, failure_reason); - if (ptr != nullptr) { - return ptr; - } - return MapPlaceholderView(vaddr, size, backing_offset, mode, failure_reason); - } - - if (!ApplyProtection(ptr, size, mode)) { - UnmapViewOfFile(ptr); - SetFailure(failure_reason, FailureReason::ProtectFailed); - return nullptr; - } - return ptr; - } - - static bool UnmapView(void* vaddr, uint64_t /*size*/) { return UnmapViewOfFile(vaddr) != 0; } - static bool UnmapViewPreservePlaceholder(void* vaddr, uint64_t /*size*/) { return UnmapViewOfFile2(GetCurrentProcess(), vaddr, MEM_PRESERVE_PLACEHOLDER) != 0; } - HANDLE m_handle = nullptr; - uint64_t m_granularity = 0x10000; + HANDLE m_handle = nullptr; #elif KYTY_PLATFORM == KYTY_PLATFORM_LINUX [[nodiscard]] static uint64_t GetMappableTestSize() { return 0x4000; } @@ -1070,21 +576,11 @@ private: return ptr; } - static bool UnmapView(void* vaddr, uint64_t size) { -#if defined(__APPLE__) - // Re-reserve instead of freeing: the guest may MAP_FIXED-remap this range - // later, and that must never destroy a host mapping (dyld, Rosetta, Metal) - // that moved into the hole in the meantime. - return mmap(vaddr, size, PROT_NONE, MAP_FIXED | MAP_PRIVATE | MAP_ANON | MAP_NORESERVE, - -1, 0) != MAP_FAILED; -#else - return munmap(vaddr, size) == 0; -#endif - } - - // POSIX has no placeholder equivalent. + // Replace the file view atomically instead of exposing an unmapped hole in the + // process-lifetime guest reservation. static bool UnmapViewPreservePlaceholder(void* vaddr, uint64_t size) { - return munmap(vaddr, size) == 0; + void* ptr = mmap(vaddr, size, PROT_NONE, MAP_PRIVATE | MAP_ANON | MAP_FIXED, -1, 0); + return ptr != MAP_FAILED && ptr == vaddr; } int m_fd = -1; @@ -1096,224 +592,303 @@ private: std::map m_maps; }; -class PlaceholderAddressSpace { +class GuestAddressSpace { public: - PlaceholderAddressSpace() { + explicit GuestAddressSpace(uint64_t backing_size) { #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS SYSTEM_INFO info {}; GetSystemInfo(&info); m_granularity = info.dwAllocationGranularity; #endif + ReserveGuestRegions(); + m_backing = new GuestBackingStore(backing_size); + EXIT_IF(!m_backing->IsAvailable()); } - KYTY_CLASS_NO_COPY(PlaceholderAddressSpace); + ~GuestAddressSpace() { + delete m_backing; + m_backing = nullptr; +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + for (const auto& [start, size]: m_owned) { + const auto end = start + size; + auto current = start; + while (current < end) { + MEMORY_BASIC_INFORMATION info {}; + EXIT_IF(VirtualQuery(reinterpret_cast(current), &info, sizeof(info)) == + 0); + const auto region_start = reinterpret_cast(info.BaseAddress); + const auto region_end = region_start + static_cast(info.RegionSize); + if (info.State == MEM_COMMIT) { + EXIT_IF(VirtualFree(reinterpret_cast(region_start), info.RegionSize, + MEM_RELEASE | MEM_PRESERVE_PLACEHOLDER) == 0); + continue; + } + if (info.State == MEM_RESERVE) { + EXIT_IF(VirtualFree(info.AllocationBase, 0, MEM_RELEASE) == 0); + } + current = region_end; + } + } +#else + for (const auto& [start, size]: m_owned) { + munmap(reinterpret_cast(start), size); + } +#endif + } + + KYTY_CLASS_NO_COPY(GuestAddressSpace); [[nodiscard]] uint64_t GetGranularity() const { return m_granularity; } + [[nodiscard]] uint64_t GetBackingBase() const { return m_backing->GetBackingBase(); } + [[nodiscard]] uint64_t GetBackingSize() const { return m_backing->GetBackingSize(); } - uint64_t ReserveAligned(uint64_t search_addr, uint64_t size, uint64_t alignment) { -#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS - if (size == 0) { - return 0; - } - alignment = std::max(alignment != 0 ? alignment : 0x4000, m_granularity); - - MEM_ADDRESS_REQUIREMENTS req {}; - MEM_EXTENDED_PARAMETER param {}; - req.LowestStartingAddress = reinterpret_cast( - AlignUp(search_addr != 0 ? search_addr : HOST_SYSTEM_MANAGED_MIN, alignment)); - req.HighestEndingAddress = - reinterpret_cast(search_addr != 0 ? HOST_USER_MAX : HOST_SYSTEM_MANAGED_MAX); - req.Alignment = alignment; - param.Type = MemExtendedParameterAddressRequirements; - param.Pointer = &req; - - auto* ptr = VirtualAlloc2(GetCurrentProcess(), nullptr, size, - MEM_RESERVE | MEM_RESERVE_PLACEHOLDER, PAGE_NOACCESS, ¶m, 1); - if (ptr == nullptr && search_addr == 0) { - req.LowestStartingAddress = reinterpret_cast(HOST_USER_MIN); - req.HighestEndingAddress = reinterpret_cast(HOST_USER_MAX); - ptr = VirtualAlloc2(GetCurrentProcess(), nullptr, size, - MEM_RESERVE | MEM_RESERVE_PLACEHOLDER, PAGE_NOACCESS, ¶m, 1); - } - if (ptr == nullptr) { - return 0; - } - - const auto addr = reinterpret_cast(ptr); - AddFree(addr, size); - return addr; -#else - auto addr = VirtualMemory::ReserveAligned(search_addr, size, alignment); - return addr; -#endif + bool ZeroBacking(uint64_t backing_offset, uint64_t size) { + return m_backing->ZeroRange(backing_offset, size); } - bool ReserveFixed(uint64_t vaddr, uint64_t size) { -#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS - if (vaddr == 0 || size == 0 || !IsPageAligned(vaddr) || !IsPageAligned(size)) { + [[nodiscard]] bool Owns(uint64_t vaddr, uint64_t size) { + if (size == 0 || UINT64_MAX - vaddr < size) { return false; } - { - std::lock_guard lock(m_mutex); - if (FindContaining(vaddr, size) != m_free.end()) { - return true; - } + std::lock_guard lock(m_mutex); + return std::any_of(m_owned.begin(), m_owned.end(), [vaddr, size](const auto& range) { + return vaddr >= range.first && vaddr + size <= range.first + range.second; + }); + } + + [[nodiscard]] bool OverlapsOwned(uint64_t vaddr, uint64_t size) { + if (size == 0 || UINT64_MAX - vaddr < size) { + return false; } - auto* ptr = VirtualAlloc2(GetCurrentProcess(), reinterpret_cast(vaddr), size, - MEM_RESERVE | MEM_RESERVE_PLACEHOLDER, PAGE_NOACCESS, nullptr, 0); - if (ptr == nullptr || reinterpret_cast(ptr) != vaddr) { - if (ptr != nullptr) { - VirtualFree(ptr, 0, MEM_RELEASE); + std::lock_guard lock(m_mutex); + return std::any_of(m_owned.begin(), m_owned.end(), [vaddr, size](const auto& range) { + return VirtualRangesOverlap(vaddr, size, range.first, range.second); + }); + } + + bool TryWriteBacking(uint64_t vaddr, const void* data, uint64_t size) { + return m_backing->TryWriteBacking(vaddr, data, size); + } + + bool TryReadBacking(uint64_t vaddr, void* data, uint64_t size) { + return m_backing->TryReadBacking(vaddr, data, size); + } + + bool BackingContains(uint64_t vaddr, uint64_t size) { return m_backing->Contains(vaddr, size); } + + bool Protect(uint64_t vaddr, uint64_t size, VirtualMemory::Mode mode) { + if (size == 0 || UINT64_MAX - vaddr < size) { + return false; + } + std::lock_guard lock(m_mutex); + return ProtectMappedUnlocked(vaddr, size, mode); + } + + bool ProtectTransient(uint64_t vaddr, uint64_t size, VirtualMemory::Mode mode) { + if (size == 0 || UINT64_MAX - vaddr < size) { + return false; + } + + std::lock_guard lock(m_mutex); + return ProtectMappedUnlocked(vaddr, size, mode); + } + + bool MapBacking(uint64_t vaddr, uint64_t size, uint64_t backing_offset, + VirtualMemory::Mode mode, + GuestBackingStore::FailureReason* failure_reason = nullptr) { + std::lock_guard lock(m_mutex); + if (!ConsumeUnlocked(vaddr, size)) { + if (failure_reason != nullptr) { + *failure_reason = GuestBackingStore::FailureReason::AddressRangeUnavailable; } return false; } - AddFree(vaddr, size); - return true; -#else - return VirtualMemory::ReserveFixed(vaddr, size); -#endif - } - - bool Consume(uint64_t vaddr, uint64_t size) { -#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS - std::lock_guard lock(m_mutex); - auto it = FindContaining(vaddr, size); - if (it == m_free.end()) { - return false; - } - if (!SplitUnlocked(it, vaddr, size)) { - return false; - } - it = m_free.find(vaddr); - if (it == m_free.end() || it->second != size) { - return false; - } - m_free.erase(it); - return true; -#else - return false; -#endif - } - -#if defined(KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS) - bool TestContainsFree(uint64_t vaddr, uint64_t size) { -#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS - std::lock_guard lock(m_mutex); - return FindContaining(vaddr, size) != m_free.end(); -#else - return false; -#endif - } -#endif - - bool ReleaseFree(uint64_t vaddr, uint64_t size) { -#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS - std::lock_guard lock(m_mutex); - auto it = FindContaining(vaddr, size); - if (it == m_free.end() || !SplitUnlocked(it, vaddr, size)) { - return false; - } - it = m_free.find(vaddr); - if (it == m_free.end() || it->second != size) { - return false; - } - m_free.erase(it); - if (VirtualFree(reinterpret_cast(vaddr), 0, MEM_RELEASE) != 0) { + const auto host_mode = VirtualMemory::IsExecute(mode) + ? VirtualMemory::Mode::ExecuteReadWrite + : VirtualMemory::Mode::ReadWrite; + if (m_backing->MapExistingPlaceholderFixed(vaddr, size, backing_offset, host_mode, + failure_reason)) { + EXIT_IF(!AddMappedUnlocked(vaddr, size, MappingType::Backing)); return true; } AddFreeUnlocked(vaddr, size); return false; -#else - return VirtualMemory::FreeRange(vaddr, size); + } + + bool UnmapBacking(uint64_t vaddr, uint64_t size) { + std::lock_guard lock(m_mutex); + if (!ContainsMappedUnlocked(vaddr, size, MappingType::Backing)) { + return false; + } + bool placeholder_preserved = false; + if (!m_backing->Unmap(vaddr, size, &placeholder_preserved)) { + return false; + } + if (!placeholder_preserved) { + return false; + } + EXIT_IF(!RemoveMappedUnlocked(vaddr, size, MappingType::Backing)); + AddFreeUnlocked(vaddr, size); + return true; + } + + uint64_t ReserveAligned(uint64_t search_addr, uint64_t size, uint64_t alignment) { + if (size == 0) { + return 0; + } + alignment = std::max(alignment != 0 ? alignment : PageSize(), PageSize()); + std::lock_guard lock(m_mutex); + if (search_addr == 0) { + auto addr = FindFreeAlignedUnlocked(HOST_SYSTEM_MANAGED_MIN, + HOST_SYSTEM_MANAGED_MAX + 1u, size, alignment); + return addr != 0 ? addr + : FindFreeAlignedUnlocked(HOST_USER_MIN, HOST_USER_MAX + 1u, size, + alignment); + } + return FindFreeAlignedUnlocked(search_addr, HOST_USER_MAX + 1u, size, alignment); + } + + uint64_t FindFreeAligned(uint64_t search_start, uint64_t search_end, uint64_t size, + uint64_t alignment) { + if (size == 0 || alignment == 0) { + return 0; + } + alignment = std::max(alignment, PageSize()); + std::lock_guard lock(m_mutex); + return FindFreeAlignedUnlocked(search_start, search_end, size, alignment); + } + + bool ReserveFixed(uint64_t vaddr, uint64_t size) { + if (vaddr == 0 || size == 0 || !IsPageAligned(vaddr) || !IsPageAligned(size)) { + return false; + } + std::lock_guard lock(m_mutex); + return FindContaining(vaddr, size) != m_free.end(); + } + +#if defined(KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS) + bool TestContainsFree(uint64_t vaddr, uint64_t size) { + std::lock_guard lock(m_mutex); + return FindContaining(vaddr, size) != m_free.end(); + } #endif + + bool ReleaseFree(uint64_t vaddr, uint64_t size) { + std::lock_guard lock(m_mutex); + return FindContaining(vaddr, size) != m_free.end(); } bool Commit(uint64_t vaddr, uint64_t size, VirtualMemory::Mode mode) { -#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS - if (!Consume(vaddr, size)) { + std::lock_guard lock(m_mutex); + if (!ConsumeUnlocked(vaddr, size)) { return false; } + const auto host_mode = VirtualMemory::IsExecute(mode) + ? VirtualMemory::Mode::ExecuteReadWrite + : VirtualMemory::Mode::ReadWrite; +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS auto* ptr = VirtualAlloc2(GetCurrentProcess(), reinterpret_cast(vaddr), size, MEM_RESERVE | MEM_COMMIT | MEM_REPLACE_PLACEHOLDER, - GetProtect(mode), nullptr, 0); + GetProtect(host_mode), nullptr, 0); if (ptr == nullptr || reinterpret_cast(ptr) != vaddr) { if (ptr != nullptr) { VirtualFree(ptr, 0, MEM_RELEASE); } - AddFree(vaddr, size); + AddFreeUnlocked(vaddr, size); return false; } + EXIT_IF(!AddMappedUnlocked(vaddr, size, MappingType::Private)); return true; #else + if (VirtualMemory::Protect(vaddr, size, host_mode)) { + EXIT_IF(!AddMappedUnlocked(vaddr, size, MappingType::Private)); + return true; + } + AddFreeUnlocked(vaddr, size); return false; #endif } bool ReleaseCommitted(uint64_t vaddr, uint64_t size) { -#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS if (vaddr == 0 || size == 0 || !IsPageAligned(vaddr) || !IsPageAligned(size)) { return false; } - if (VirtualFree(reinterpret_cast(vaddr), size, - MEM_RELEASE | MEM_PRESERVE_PLACEHOLDER) == 0) { + std::lock_guard lock(m_mutex); + if (!ContainsMappedUnlocked(vaddr, size, MappingType::Private)) { return false; } - AddFree(vaddr, size); - return true; + const auto end = vaddr + size; + auto current = vaddr; + std::vector> regions; + while (current < end) { + const auto it = FindMappedContainingUnlocked(current); + const auto part_end = std::min(end, it->second.vaddr + it->second.size); + regions.emplace_back(current, part_end - current); + current = part_end; + } +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + for (const auto& [region_vaddr, region_size]: regions) { + EXIT_IF(VirtualFree(reinterpret_cast(region_vaddr), region_size, + MEM_RELEASE | MEM_PRESERVE_PLACEHOLDER) == 0); + } #else - return false; + for (const auto& [region_vaddr, region_size]: regions) { + RestorePlaceholder(region_vaddr, region_size); + } #endif + EXIT_IF(!RemoveMappedUnlocked(vaddr, size, MappingType::Private)); + AddFreeUnlocked(vaddr, size); + return true; } bool SelfTest() { -#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS - const auto addr = ReserveAligned(0, m_granularity, m_granularity); + const auto addr = ReserveAligned(0, PageSize(), PageSize()); if (addr == 0) { - LOGF_COLOR(Log::Color::Yellow, - "\t placeholder address-space self-test skipped: reserve unavailable\n"); + LOGF_COLOR(Log::Color::Red, + "\t guest address-space self-test: 16 KiB range unavailable\n"); return false; } - if (!Commit(addr, m_granularity, VirtualMemory::Mode::ReadWrite)) { - ReleaseFree(addr, m_granularity); - LOGF_COLOR(Log::Color::Red, "\t placeholder address-space self-test: commit failed\n"); + if (!Commit(addr, PageSize(), VirtualMemory::Mode::ReadWrite)) { + LOGF_COLOR(Log::Color::Red, "\t guest address-space self-test: commit failed\n"); return false; } auto* ptr = reinterpret_cast(addr); *ptr = 0x4b59545950484f4cull; // "KYTYPHOL" - if (!ReleaseCommitted(addr, m_granularity)) { - VirtualFree(reinterpret_cast(addr), 0, MEM_RELEASE); + if (!ReleaseCommitted(addr, PageSize())) { LOGF_COLOR(Log::Color::Red, - "\t placeholder address-space self-test: release-committed failed\n"); + "\t guest address-space self-test: release-committed failed\n"); return false; } - const bool ok = ReleaseFree(addr, m_granularity); - LOGF_COLOR(ok ? Log::Color::Green : Log::Color::Red, - "\t placeholder address-space self-test: %s\n", - ok ? "ok" : "release-free failed"); - return ok; -#else - LOGF_COLOR(Log::Color::Yellow, "\t placeholder address-space self-test skipped\n"); - return false; -#endif - } - - void AddFree(uint64_t vaddr, uint64_t size) { -#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS - std::lock_guard lock(m_mutex); - AddFreeUnlocked(vaddr, size); -#endif + LOGF_COLOR(Log::Color::Green, "\t guest address-space self-test: ok\n"); + return true; } private: -#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + enum class MappingType { Backing, Private }; + + struct MappedRegion { + uint64_t vaddr = 0; + uint64_t size = 0; + MappingType type = MappingType::Private; + }; + static constexpr uint64_t PageSize() { return 0x4000; } static uint64_t AlignUp(uint64_t value, uint64_t alignment) { - return (value + alignment - 1u) & ~(alignment - 1u); + if (alignment == 0) { + return 0; + } + const auto remainder = value % alignment; + const auto increment = (remainder != 0 ? alignment - remainder : 0); + return increment <= UINT64_MAX - value ? value + increment : 0; + } + + static uint64_t AlignDown(uint64_t value, uint64_t alignment) { + return value & ~(alignment - 1u); } [[nodiscard]] static bool IsPageAligned(uint64_t value) { @@ -1324,6 +899,105 @@ private: return (value & (m_granularity - 1u)) == 0; } + bool ProtectMappedUnlocked(uint64_t vaddr, uint64_t size, VirtualMemory::Mode mode) { + const auto end = vaddr + size; + auto it = m_mapped.upper_bound(vaddr); + if (it != m_mapped.begin()) { + --it; + } + for (; it != m_mapped.end() && it->second.vaddr < end; ++it) { + const auto& region = it->second; + const auto range_start = std::max(vaddr, region.vaddr); + const auto range_end = std::min(end, region.vaddr + region.size); + if (range_start < range_end && + !VirtualMemory::Protect(range_start, range_end - range_start, mode)) { + return false; + } + } + return true; + } + + bool ConsumeUnlocked(uint64_t vaddr, uint64_t size) { + auto it = FindContaining(vaddr, size); + if (it == m_free.end() || !SplitUnlocked(it, vaddr, size)) { + return false; + } + it = m_free.find(vaddr); + if (it == m_free.end() || it->second != size) { + return false; + } + m_free.erase(it); + return true; + } + + std::map::iterator FindMappedContainingUnlocked(uint64_t vaddr) { + auto next = m_mapped.upper_bound(vaddr); + if (next == m_mapped.begin()) { + return m_mapped.end(); + } + auto it = std::prev(next); + return vaddr < it->second.vaddr + it->second.size ? it : m_mapped.end(); + } + + bool ContainsMappedUnlocked(uint64_t vaddr, uint64_t size, MappingType type) { + if (size == 0 || UINT64_MAX - vaddr < size) { + return false; + } + const auto end = vaddr + size; + auto current = vaddr; + while (current < end) { + const auto it = FindMappedContainingUnlocked(current); + if (it == m_mapped.end() || it->second.type != type) { + return false; + } + current = std::min(end, it->second.vaddr + it->second.size); + } + return true; + } + + bool AddMappedUnlocked(uint64_t vaddr, uint64_t size, MappingType type) { + if (size == 0 || UINT64_MAX - vaddr < size) { + return false; + } + const auto end = vaddr + size; + auto next = m_mapped.lower_bound(vaddr); + if (next != m_mapped.end() && next->second.vaddr < end) { + return false; + } + if (next != m_mapped.begin()) { + const auto& previous = std::prev(next)->second; + if (previous.vaddr + previous.size > vaddr) { + return false; + } + } + return m_mapped.emplace(vaddr, MappedRegion {vaddr, size, type}).second; + } + + bool RemoveMappedUnlocked(uint64_t vaddr, uint64_t size, MappingType type) { + if (!ContainsMappedUnlocked(vaddr, size, type)) { + return false; + } + const auto end = vaddr + size; + auto current = vaddr; + while (current < end) { + auto it = FindMappedContainingUnlocked(current); + const auto original = it->second; + const auto part_end = std::min(end, original.vaddr + original.size); + m_mapped.erase(it); + if (original.vaddr < current) { + const auto left_size = current - original.vaddr; + EXIT_IF(!AddMappedUnlocked(original.vaddr, left_size, original.type)); + } + if (part_end < original.vaddr + original.size) { + const auto right_size = original.vaddr + original.size - part_end; + EXIT_IF(!AddMappedUnlocked(part_end, right_size, original.type)); + } + current = part_end; + } + return true; + } + +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS static DWORD GetProtect(VirtualMemory::Mode mode) { switch (mode) { case VirtualMemory::Mode::Read: return PAGE_READONLY; @@ -1337,6 +1011,89 @@ private: default: return PAGE_NOACCESS; } } +#endif + + void ReserveGuestRegions() { +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + constexpr uint64_t MinimumRegionSize = 0x1000000; + const auto process = GetCurrentProcess(); + const auto guest_end = HOST_USER_MAX + 1u; + auto next_addr = HOST_SYSTEM_MANAGED_MIN; + + while (next_addr < guest_end) { + MEMORY_BASIC_INFORMATION info {}; + if (VirtualQuery(reinterpret_cast(next_addr), &info, sizeof(info)) == 0) { + EXIT("failed to inspect guest address space at 0x%016" PRIx64 "\n", next_addr); + } + + const auto region_base = reinterpret_cast(info.BaseAddress); + const auto region_end = + std::min(guest_end, region_base + static_cast(info.RegionSize)); + const auto reserve_start = + AlignUp(std::max(region_base, HOST_SYSTEM_MANAGED_MIN), m_granularity); + const auto reserve_end = AlignDown(region_end, m_granularity); + + if (info.State == MEM_FREE && reserve_end > reserve_start && + reserve_end - reserve_start > MinimumRegionSize) { + auto* ptr = VirtualAlloc2( + process, reinterpret_cast(reserve_start), reserve_end - reserve_start, + MEM_RESERVE | MEM_RESERVE_PLACEHOLDER, PAGE_NOACCESS, nullptr, 0); + if (reinterpret_cast(ptr) != reserve_start) { + EXIT("failed to reserve guest address space at 0x%016" PRIx64 + ", size 0x%016" PRIx64 "\n", + reserve_start, reserve_end - reserve_start); + } + AddFreeUnlocked(reserve_start, reserve_end - reserve_start); + m_owned.emplace_back(reserve_start, reserve_end - reserve_start); + } + + if (region_end <= next_addr) { + EXIT("guest address-space scan did not advance at 0x%016" PRIx64 "\n", next_addr); + } + next_addr = region_end; + } +#else + const std::array, 3> regions {{ + {HOST_SYSTEM_MANAGED_MIN, HOST_SYSTEM_MANAGED_MAX + 1u}, + {HOST_SYSTEM_RESERVED_MIN, HOST_SYSTEM_RESERVED_MAX + 1u}, + {HOST_USER_MIN, HOST_USER_MAX + 1u}, + }}; + for (const auto& [start, end]: regions) { + int flags = MAP_PRIVATE | MAP_ANON | MAP_NORESERVE; +#if defined(KYTY_LINKED_GUEST_ADDRESS_SPACE) + flags |= MAP_FIXED; +#elif defined(MAP_FIXED_NOREPLACE) + flags |= MAP_FIXED_NOREPLACE; +#endif + void* ptr = mmap(reinterpret_cast(start), end - start, PROT_NONE, flags, -1, 0); + if (ptr == MAP_FAILED || reinterpret_cast(ptr) != start) { + if (ptr != MAP_FAILED) { + munmap(ptr, end - start); + } + EXIT("failed to reserve guest address space at 0x%016" PRIx64 ", size 0x%016" PRIx64 + "\n", + start, end - start); + } + AddFreeUnlocked(start, end - start); + m_owned.emplace_back(start, end - start); + } +#endif + } + + uint64_t FindFreeAlignedUnlocked(uint64_t search_start, uint64_t search_end, uint64_t size, + uint64_t alignment) { + if (search_start >= search_end || size > search_end - search_start) { + return 0; + } + for (const auto& [base, range_size]: m_free) { + const auto candidate = AlignUp(std::max(base, search_start), alignment); + if (candidate < search_end && size <= search_end - candidate && + GuestFreeRangeContains(base, range_size, candidate, size)) { + return candidate; + } + } + return 0; + } std::map::iterator FindContaining(uint64_t vaddr, uint64_t size) { auto next = m_free.upper_bound(vaddr); @@ -1357,19 +1114,34 @@ private: if (base != vaddr) { const auto left_size = vaddr - base; +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS if (VirtualFree(reinterpret_cast(base), left_size, MEM_RELEASE | MEM_PRESERVE_PLACEHOLDER) == 0) { + LOGF_COLOR(Log::Color::Red, + "\t guest placeholder split failed: base=0x%016" PRIx64 + " size=0x%016" PRIx64 " requested=0x%016" PRIx64 + " request_size=0x%016" PRIx64 " error=0x%08" PRIx32 "\n", + base, end - base, vaddr, size, static_cast(GetLastError())); return false; } +#endif it->second = left_size; it = m_free.emplace(vaddr, end - vaddr).first; } if (it->second != size) { +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS if (VirtualFree(reinterpret_cast(it->first), size, MEM_RELEASE | MEM_PRESERVE_PLACEHOLDER) == 0) { + LOGF_COLOR(Log::Color::Red, + "\t guest placeholder split failed: base=0x%016" PRIx64 + " size=0x%016" PRIx64 " requested=0x%016" PRIx64 + " request_size=0x%016" PRIx64 " error=0x%08" PRIx32 "\n", + it->first, it->second, vaddr, size, + static_cast(GetLastError())); return false; } +#endif const auto right_start = it->first + size; const auto right_size = it->second - size; it->second = size; @@ -1408,16 +1180,36 @@ private: } if (base != it->first || end != it->first + it->second) { +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS if (VirtualFree(reinterpret_cast(base), end - base, MEM_RELEASE | MEM_COALESCE_PLACEHOLDERS) != 0) { m_free.erase(first, last); m_free.emplace(base, end - base); } +#else + m_free.erase(first, last); + m_free.emplace(base, end - base); +#endif } } - std::mutex m_mutex; - std::map m_free; + void RestorePlaceholder(uint64_t vaddr, uint64_t size) { +#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS + // The address remains owned for the process lifetime; MAP_FIXED only replaces one of our + // mappings and never acquires a new guest address. + void* ptr = mmap(reinterpret_cast(vaddr), size, PROT_NONE, + MAP_PRIVATE | MAP_ANON | MAP_FIXED, -1, 0); + EXIT_IF(ptr == MAP_FAILED || reinterpret_cast(ptr) != vaddr); +#else + (void)vaddr; + (void)size; #endif - uint64_t m_granularity = 0x10000; + } + + std::mutex m_mutex; + std::map m_free; + std::map m_mapped; + std::vector> m_owned; + uint64_t m_granularity = 0x10000; + GuestBackingStore* m_backing = nullptr; }; diff --git a/src/kernel/pthread.cpp b/src/kernel/pthread.cpp index f51c384..7cb980d 100644 --- a/src/kernel/pthread.cpp +++ b/src/kernel/pthread.cpp @@ -75,16 +75,16 @@ LIB_NAME("libkernel", "libkernel"); #undef PTHREAD_STACK_MIN #endif -constexpr int KEYS_MAX = 256; -constexpr int DESTRUCTOR_ITERATIONS = 4; -constexpr size_t PTHREAD_STACK_DEFAULT = 0x100000; -constexpr size_t GUEST_PTHREAD_STACK_MIN = 0x4000; -constexpr size_t PTHREAD_STACK_PAGE = 0x4000; -constexpr size_t PTHREAD_STACK_GRANULARITY = 0x10000; -constexpr size_t PTHREAD_STACK_INITIAL = 0x200000; -constexpr size_t PTHREAD_STACK_EXTRA = 0x100000; -constexpr uint64_t PTHREAD_STACK_TOP = 0x7efff8000ull; -constexpr uint32_t SIGNAL_APC_POLL_MICROS = 10000; +constexpr int KEYS_MAX = 256; +constexpr int DESTRUCTOR_ITERATIONS = 4; +constexpr size_t PTHREAD_STACK_DEFAULT = 0x100000; +constexpr size_t GUEST_PTHREAD_STACK_MIN = 0x4000; +constexpr size_t PTHREAD_STACK_PAGE = 0x4000; +constexpr size_t PTHREAD_STACK_INITIAL = 0x200000; +constexpr size_t PTHREAD_STACK_EXTRA = 0x100000; +constexpr uint64_t PTHREAD_STACK_TOP = 0x7efff8000ull; +constexpr uint64_t PTHREAD_STACK_BOTTOM = 0x0000040000ull; +constexpr uint32_t SIGNAL_APC_POLL_MICROS = 10000; static constexpr KernelClockid KERNEL_CLOCK_REALTIME = 0; static constexpr KernelClockid KERNEL_CLOCK_VIRTUAL = 1; @@ -697,16 +697,17 @@ static std::atomic g_pthread_thread_id = 0; static Common::Mutex g_guest_stack_mutex; static uint64_t g_guest_stack_last = 0; +struct CachedGuestStack { + uint64_t address; + size_t map_size; + size_t guard_size; +}; +static std::vector g_guest_stack_cache; static size_t RoundStackSize(size_t size) { return ((size + PTHREAD_STACK_PAGE - 1) / PTHREAD_STACK_PAGE) * PTHREAD_STACK_PAGE; } -static size_t RoundStackMappingSize(size_t size) { - return ((size + PTHREAD_STACK_GRANULARITY - 1) / PTHREAD_STACK_GRANULARITY) * - PTHREAD_STACK_GRANULARITY; -} - static int CreateGuestStack(PthreadAttr attr) { if (attr == nullptr) { return KERNEL_ERROR_EINVAL; @@ -722,34 +723,41 @@ static int CreateGuestStack(PthreadAttr attr) { const auto stack_size = RoundStackSize(attr->stack_size); const auto guard_size = RoundStackSize(attr->guard_size); - const auto map_size = RoundStackMappingSize(stack_size + guard_size); + const auto map_size = stack_size + guard_size; uint64_t stack_addr = 0; + bool cached = false; { Common::LockGuard lock(g_guest_stack_mutex); - if (g_guest_stack_last == 0) { - g_guest_stack_last = (PTHREAD_STACK_TOP - PTHREAD_STACK_INITIAL - PTHREAD_STACK_PAGE) & - ~(static_cast(PTHREAD_STACK_GRANULARITY) - 1); + auto cached_stack = + std::find_if(g_guest_stack_cache.begin(), g_guest_stack_cache.end(), + [map_size, guard_size](const auto& stack) { + return stack.map_size == map_size && stack.guard_size == guard_size; + }); + if (cached_stack != g_guest_stack_cache.end()) { + stack_addr = cached_stack->address; + g_guest_stack_cache.erase(cached_stack); + cached = true; + } else { + if (g_guest_stack_last == 0) { + g_guest_stack_last = PTHREAD_STACK_TOP - PTHREAD_STACK_INITIAL - PTHREAD_STACK_PAGE; + } + if (map_size > g_guest_stack_last - PTHREAD_STACK_BOTTOM) { + return KERNEL_ERROR_EAGAIN; + } + stack_addr = g_guest_stack_last - map_size; + g_guest_stack_last -= map_size; } - - stack_addr = g_guest_stack_last - map_size; - g_guest_stack_last -= map_size; } - void* mapped_addr = reinterpret_cast(stack_addr); - - constexpr int GUEST_PROT_READ_WRITE = 0x03; - constexpr int GUEST_MAP_PRIVATE = 0x02; - constexpr int GUEST_MAP_FIXED = 0x10; - constexpr int GUEST_MAP_STACK = 0x400; - constexpr int GUEST_MAP_ANON = 0x1000; - - int result = Memory::KernelMapNamedFlexibleMemory( - &mapped_addr, map_size, GUEST_PROT_READ_WRITE, - GUEST_MAP_PRIVATE | GUEST_MAP_FIXED | GUEST_MAP_STACK | GUEST_MAP_ANON, "stack"); - if (result != OK) { - return KERNEL_ERROR_EAGAIN; + int result = OK; + if (!cached) { + stack_addr = Memory::AllocateGuestStackMemory( + stack_addr, map_size, Common::VirtualMemory::Mode::ReadWrite, "stack"); + if (stack_addr == 0) { + return KERNEL_ERROR_EAGAIN; + } } if (guard_size != 0) { @@ -761,7 +769,7 @@ static int CreateGuestStack(PthreadAttr attr) { } attr->stack_addr = reinterpret_cast(stack_addr + guard_size); - attr->stack_size = map_size - guard_size; + attr->stack_size = stack_size; attr->stack_user = false; attr->stack_map_addr = stack_addr; attr->stack_map_size = map_size; @@ -777,13 +785,90 @@ static void FreeGuestStack(PthreadAttr attr) { return; } - Memory::KernelMunmap(attr->stack_map_addr, attr->stack_map_size); + const auto guard_size = attr->stack_map_size - attr->stack_size; + { + Common::LockGuard lock(g_guest_stack_mutex); + g_guest_stack_cache.push_back({attr->stack_map_addr, attr->stack_map_size, guard_size}); + } attr->stack_addr = nullptr; attr->stack_map_addr = 0; attr->stack_map_size = 0; } +#if defined(KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS) +bool TestGuestStackOwnerLifecycle(uint64_t* first_address, uint64_t* second_address, + uint64_t* map_size) { + if (first_address == nullptr || second_address == nullptr || map_size == nullptr) { + return false; + } + + size_t flexible_before = 0; + if (Memory::KernelAvailableFlexibleMemorySize(&flexible_before) != OK) { + return false; + } + + PthreadAttr attr = nullptr; + if (PthreadAttrInit(&attr) != OK) { + return false; + } + if (CreateGuestStack(attr) != OK) { + PthreadAttrDestroy(&attr); + return false; + } + *first_address = attr->stack_map_addr; + *map_size = attr->stack_map_size; + const bool first_owned = + Memory::TestGuestAddressRangeIsOwned(*first_address, static_cast(*map_size)); + uint64_t backing_value = 0; + const bool first_private = + !Memory::TryReadBacking(*first_address, &backing_value, sizeof(backing_value)); + size_t flexible_during_first = 0; + const bool first_capacity_unchanged = + Memory::KernelAvailableFlexibleMemorySize(&flexible_during_first) == OK && + flexible_during_first == flexible_before; + FreeGuestStack(attr); + + if (CreateGuestStack(attr) != OK) { + PthreadAttrDestroy(&attr); + return false; + } + *second_address = attr->stack_map_addr; + const bool second_owned = + Memory::TestGuestAddressRangeIsOwned(*second_address, static_cast(*map_size)); + const bool second_private = + !Memory::TryReadBacking(*second_address, &backing_value, sizeof(backing_value)); + size_t flexible_during_second = 0; + const bool second_capacity_unchanged = + Memory::KernelAvailableFlexibleMemorySize(&flexible_during_second) == OK && + flexible_during_second == flexible_before; + FreeGuestStack(attr); + + CachedGuestStack cached {}; + bool found = false; + { + Common::LockGuard lock(g_guest_stack_mutex); + const auto entry = std::find_if( + g_guest_stack_cache.begin(), g_guest_stack_cache.end(), + [second_address](const auto& stack) { return stack.address == *second_address; }); + if (entry != g_guest_stack_cache.end()) { + cached = *entry; + g_guest_stack_cache.erase(entry); + found = true; + } + } + + const bool unmapped = found && Memory::KernelMunmap(cached.address, cached.map_size) == OK; + size_t flexible_after = 0; + const bool final_capacity_unchanged = + Memory::KernelAvailableFlexibleMemorySize(&flexible_after) == OK && + flexible_after == flexible_before; + return PthreadAttrDestroy(&attr) == OK && first_owned && first_private && + first_capacity_unchanged && second_owned && second_private && + second_capacity_unchanged && unmapped && final_capacity_unchanged; +} +#endif + static KYTY_SYSV_ABI void* RunOnGuestStack(void* arg, pthread_entry_func_t func, void* stack_top) { #if defined(__x86_64__) || defined(_M_X64) void* ret = nullptr; diff --git a/src/kernel/pthread.h b/src/kernel/pthread.h index f8ffc97..d9364d5 100644 --- a/src/kernel/pthread.h +++ b/src/kernel/pthread.h @@ -112,11 +112,15 @@ void PthreadQueuePendingSignal(Pthread thread, int signum); bool PthreadHasPendingSignal(Pthread thread, int signum); bool PthreadTakePendingSignal(Pthread thread, int signum); bool PthreadGetGuestStack(Pthread thread, uint64_t* stack_addr, uint64_t* stack_size); +#if defined(KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS) +bool TestGuestStackOwnerLifecycle(uint64_t* first_address, uint64_t* second_address, + uint64_t* map_size); +#endif #if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS bool PthreadKillHost(Pthread thread, int host_signal); #endif -int PthreadGetPriorityForKernel(Pthread thread); -int PthreadGetCurrentPriorityForKernel(); +int PthreadGetPriorityForKernel(Pthread thread); +int PthreadGetCurrentPriorityForKernel(); int KYTY_SYSV_ABI KernelUsleep(KernelUseconds microseconds); unsigned int KYTY_SYSV_ABI KernelSleep(unsigned int seconds); diff --git a/src/loader/gamePatch.cpp b/src/loader/gamePatch.cpp index d376b67..b751549 100644 --- a/src/loader/gamePatch.cpp +++ b/src/loader/gamePatch.cpp @@ -2,6 +2,7 @@ #include "common/stringUtils.h" #include "common/virtualMemory.h" +#include "kernel/memory.h" #include "loader/elf.h" #include "loader/runtimeLinker.h" #include "loader/systemContent.h" @@ -153,18 +154,6 @@ bool ValidateTarget(const Plan& plan, const Program* program, std::string* error return true; } -Common::VirtualMemory::Mode ReadableMode(Elf64_Word flags) { - const bool executable = (flags & PF_X) != 0; - const bool writable = (flags & PF_W) != 0; - if (executable && writable) { - return Common::VirtualMemory::Mode::ExecuteReadWrite; - } - if (executable) { - return Common::VirtualMemory::Mode::ExecuteRead; - } - return writable ? Common::VirtualMemory::Mode::ReadWrite : Common::VirtualMemory::Mode::Read; -} - bool ResolveWrite(const Program& program, Write* write, std::string* error) { const auto* ehdr = program.elf->GetEhdr(); const auto* phdr = program.elf->GetPhdr(); @@ -178,16 +167,9 @@ bool ResolveWrite(const Program& program, Write* write, std::string* error) { continue; } - const auto segment_address = program.base_vaddr + segment.p_vaddr; - const bool add_read = (segment.p_flags & PF_R) == 0; - Common::VirtualMemory::Mode old_mode {}; - if (add_read && !Common::VirtualMemory::Protect(segment_address, segment.p_memsz, - ReadableMode(segment.p_flags), &old_mode)) { - return Fail(error, "could not read a loaded executable segment"); - } - - const auto* begin = reinterpret_cast(segment_address); - const auto* end = begin + segment.p_filesz; + const auto segment_address = program.base_vaddr + segment.p_vaddr; + const auto* begin = reinterpret_cast(segment_address); + const auto* end = begin + segment.p_filesz; for (auto* current = begin; current < end;) { const auto* found = std::search(current, end, write->expected.begin(), write->expected.end()); @@ -202,15 +184,9 @@ bool ResolveWrite(const Program& program, Write* write, std::string* error) { match_count++; current = found + 1; } - - if (add_read && - !Common::VirtualMemory::Protect(segment_address, segment.p_memsz, old_mode)) { - return Fail(error, "could not restore executable segment protection"); - } } - ::printf("Game patch: found %zu entries for '%s'\n", match_count, - write->patch_name.c_str()); + ::printf("Game patch: found %zu entries for '%s'\n", match_count, write->patch_name.c_str()); if (match == 0) { return Fail(error, "original bytes not found for '" + write->patch_name + "'"); } @@ -229,16 +205,9 @@ bool PrepareWrites(Plan* plan, const Program& program, std::string* error) { bool ApplyWrites(Plan* plan, std::string* error) { for (auto& write: plan->writes) { - Common::VirtualMemory::Mode old_mode {}; - const auto size = write.replacement.size(); - if (!Common::VirtualMemory::Protect( - write.address, size, Common::VirtualMemory::Mode::ExecuteReadWrite, &old_mode)) { - return Fail(error, "could not make patch memory writable"); - } - + const auto size = write.replacement.size(); std::memcpy(reinterpret_cast(write.address), write.replacement.data(), size); - if (!Common::VirtualMemory::Protect(write.address, size, old_mode) || - !Common::VirtualMemory::FlushInstructionCache(write.address, size)) { + if (!Common::VirtualMemory::FlushInstructionCache(write.address, size)) { return Fail(error, "could not finalize patch"); } } diff --git a/src/loader/runtimeLinker.cpp b/src/loader/runtimeLinker.cpp index 91f6ef6..460be61 100644 --- a/src/loader/runtimeLinker.cpp +++ b/src/loader/runtimeLinker.cpp @@ -62,14 +62,16 @@ static void FreeTlsBlock(ThreadLocalStorage::Block* block) { if (block->free_func != nullptr) { block->free_func(block->ptr); } else if (block->vm_alloc) { - Common::VirtualMemory::Free(reinterpret_cast(block->ptr)); + EXIT_IF(!Libs::LibKernel::Memory::FreeGuestMemory(reinterpret_cast(block->ptr), + block->alloc_size)); } else { delete[] block->ptr; } - block->ptr = nullptr; - block->free_func = nullptr; - block->vm_alloc = false; + block->ptr = nullptr; + block->free_func = nullptr; + block->vm_alloc = false; + block->alloc_size = 0; } static uint64_t AlignUp(uint64_t value, uint64_t alignment) { @@ -131,17 +133,25 @@ static std::vector g_stubbed_imports; static std::atomic_uint32_t g_unresolved_stub_call_log_count {0}; static std::vector g_unresolved_stub_thunk_pages; static uint64_t g_unresolved_stub_thunk_offset = 0; +static constexpr uint64_t UNRESOLVED_STUB_PAGE_SIZE = 4096; static KYTY_SYSV_ABI uint64_t ResolveImportStubWithId(uint64_t record_id); +static bool PatchGuestMemory64(uint64_t vaddr, uint64_t value) { + auto* ptr = reinterpret_cast(vaddr); + bool changed = (*ptr != value); + std::memcpy(ptr, &value, sizeof(value)); + return changed; +} + static uint64_t AllocateUnresolvedImportThunk(uint64_t record_id) { - constexpr uint64_t page_size = 4096; constexpr uint64_t thunk_size = 162; if (g_unresolved_stub_thunk_pages.empty() || - g_unresolved_stub_thunk_offset + thunk_size > page_size) { - auto page = Common::VirtualMemory::Alloc(0, page_size, - Common::VirtualMemory::Mode::ExecuteReadWrite); + g_unresolved_stub_thunk_offset + thunk_size > UNRESOLVED_STUB_PAGE_SIZE) { + auto page = Libs::LibKernel::Memory::AllocateRuntimeMemory( + 0, UNRESOLVED_STUB_PAGE_SIZE, Common::VirtualMemory::Mode::ExecuteReadWrite, + "unresolved_import_thunk"); EXIT_NOT_IMPLEMENTED(page == 0); g_unresolved_stub_thunk_pages.push_back(page); g_unresolved_stub_thunk_offset = 0; @@ -298,7 +308,7 @@ static KYTY_SYSV_ABI uint64_t ResolveImportStubWithId(uint64_t record_id) { resolved.name.c_str(), resolved.vaddr); if (record.patch_vaddr != 0) { - *reinterpret_cast(record.patch_vaddr) = resolved.vaddr; + PatchGuestMemory64(record.patch_vaddr, resolved.vaddr); } return resolved.vaddr; @@ -360,7 +370,7 @@ static KYTY_SYSV_ABI void RunEntry(uint64_t addr, EntryParams* params, atexit_fu register uintptr_t guest_rbp_reg asm("r15") = guest_rbp; #endif -#if defined(__APPLE__) || KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS +#if defined(__APPLE__) asm volatile( "pushq %%r12\n\t" "pushq %%r13\n\t" @@ -374,16 +384,44 @@ static KYTY_SYSV_ABI void RunEntry(uint64_t addr, EntryParams* params, atexit_fu "popq %%r13\n\t" "popq %%r12\n\t" : -#if defined(__APPLE__) : [func] "r"(func_reg), "D"(params), "S"(atexit_func), [guest_rsp] "r"(guest_rsp_reg), [guest_rbp] "r"(guest_rbp_reg) -#else - : [func] "r"(func), "D"(params), - "S"(atexit_func), [guest_rsp] "r"(guest_rsp), [guest_rbp] "r"(guest_rbp) -#endif : "cc", "memory", "rax", "rcx", "rdx", "r8", "r9", "r10", "r11", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7", "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13", "xmm14", "xmm15"); +#elif KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + // Windows stack probes use the TEB stack limits during the guest stack switch. + // bounds, which describe the host stack and are invalid while RSP is in guest memory. + register entry_func_t func_reg asm("rbx") = func; + register uintptr_t guest_rsp_reg asm("r8") = guest_rsp; + register uintptr_t guest_rbp_reg asm("r9") = guest_rbp; + asm volatile("pushq %%r12\n\t" + "pushq %%r13\n\t" + "pushq %%r14\n\t" + "pushq %%r15\n\t" + "movq %%gs:0x08, %%r14\n\t" + "movq %%gs:0x10, %%r15\n\t" + "xorq %%rcx, %%rcx\n\t" + "movq %%rcx, %%gs:0x08\n\t" + "movq %%rcx, %%gs:0x10\n\t" + "movq %%rsp, %%r12\n\t" + "movq %%rbp, %%r13\n\t" + "movq %[guest_rsp], %%rsp\n\t" + "movq %[guest_rbp], %%rbp\n\t" + "callq *%[func]\n\t" + "movq %%r13, %%rbp\n\t" + "movq %%r12, %%rsp\n\t" + "movq %%r14, %%gs:0x08\n\t" + "movq %%r15, %%gs:0x10\n\t" + "popq %%r15\n\t" + "popq %%r14\n\t" + "popq %%r13\n\t" + "popq %%r12\n\t" + : [guest_rsp] "+r"(guest_rsp_reg), [guest_rbp] "+r"(guest_rbp_reg) + : [func] "r"(func_reg), "D"(params), "S"(atexit_func) + : "cc", "memory", "rax", "rcx", "rdx", "r10", "r11", "xmm0", "xmm1", "xmm2", + "xmm3", "xmm4", "xmm5", "xmm6", "xmm7", "xmm8", "xmm9", "xmm10", "xmm11", + "xmm12", "xmm13", "xmm14", "xmm15"); #else // Clobbers prevent inputs from being allocated to r12/r13. asm volatile("movq %%rsp, %%r12\n\t" @@ -449,6 +487,110 @@ static KYTY_SYSV_ABI void RunEntry(uint64_t addr, EntryParams* params, atexit_fu #endif } +#if defined(KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS) +struct MainEntryStackTestState { + bool called = false; + uintptr_t rsp = 0; +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + uintptr_t teb_stack_base = UINTPTR_MAX; + uintptr_t teb_stack_limit = UINTPTR_MAX; +#endif +}; + +static KYTY_SYSV_ABI void TestMainEntryStackCallback(EntryParams* params, + atexit_func_t /*atexit_func*/) { + auto* state = reinterpret_cast(const_cast(params->argv[0])); + asm volatile("movq %%rsp, %0" : "=r"(state->rsp) : : "memory"); +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + asm volatile("movq %%gs:0x08, %0\n\t" + "movq %%gs:0x10, %1\n\t" + : "=r"(state->teb_stack_base), "=r"(state->teb_stack_limit) + : + : "memory"); +#endif + state->called = true; +} + +bool TestMainEntryUsesGuestStack() { + constexpr uint64_t stack_size = 0x10000; + const auto stack_base = Libs::LibKernel::Memory::AllocateRuntimeMemory( + 0, stack_size, Common::VirtualMemory::Mode::ReadWrite, "main_entry_stack_test"); + if (stack_base == 0) { + return false; + } + + MainEntryStackTestState state {}; + EntryParams params {}; + params.argv[0] = reinterpret_cast(&state); + +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + uintptr_t original_teb_stack_base = 0; + uintptr_t original_teb_stack_limit = 0; + asm volatile("movq %%gs:0x08, %0\n\t" + "movq %%gs:0x10, %1\n\t" + : "=r"(original_teb_stack_base), "=r"(original_teb_stack_limit) + : + : "memory"); +#endif + + RunEntry(reinterpret_cast(TestMainEntryStackCallback), ¶ms, nullptr, + reinterpret_cast(stack_base + stack_size)); + +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + uintptr_t restored_teb_stack_base = 0; + uintptr_t restored_teb_stack_limit = 0; + asm volatile("movq %%gs:0x08, %0\n\t" + "movq %%gs:0x10, %1\n\t" + : "=r"(restored_teb_stack_base), "=r"(restored_teb_stack_limit) + : + : "memory"); + const bool teb_ok = state.teb_stack_base == 0 && state.teb_stack_limit == 0 && + restored_teb_stack_base == original_teb_stack_base && + restored_teb_stack_limit == original_teb_stack_limit; +#else + constexpr bool teb_ok = true; +#endif + + const bool rsp_ok = state.rsp >= stack_base && state.rsp < stack_base + stack_size; + const bool freed = Libs::LibKernel::Memory::FreeGuestMemory(stack_base, stack_size); + return state.called && rsp_ok && teb_ok && freed; +} + +bool TestModuleRelocationUsesWritableHostMapping() { + constexpr uint64_t page_size = 0x4000; + constexpr uint64_t value = 0x4b59545950415443; + const auto base = Libs::LibKernel::Memory::AllocateProgramMemory( + 0, page_size, Common::VirtualMemory::Mode::ReadWrite, "host_only_patch_test"); + if (base == 0) { + return false; + } + Libs::LibKernel::Memory::SetProgramMemoryProtection(base, page_size, + Common::VirtualMemory::Mode::Read); + + Libs::LibKernel::Memory::VirtualQueryInfo before {}; + Libs::LibKernel::Memory::VirtualQueryInfo after {}; + const bool before_ok = + Libs::LibKernel::Memory::KernelVirtualQuery(reinterpret_cast(base), 0, &before, + sizeof(before)) == 0; + const bool changed = PatchGuestMemory64(base, value); + const bool after_ok = Libs::LibKernel::Memory::KernelVirtualQuery( + reinterpret_cast(base), 0, &after, sizeof(after)) == 0; + const bool value_ok = *reinterpret_cast(base) == value; +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + MEMORY_BASIC_INFORMATION mbi {}; + const bool host_mode_ok = + VirtualQuery(reinterpret_cast(base), &mbi, sizeof(mbi)) != 0 && + mbi.Protect == PAGE_READWRITE; +#else + constexpr bool host_mode_ok = true; +#endif + const bool freed = Libs::LibKernel::Memory::FreeGuestMemory(base, page_size); + + return before_ok && after_ok && changed && value_ok && host_mode_ok && freed && + before.protection == after.protection; +} +#endif + static uint64_t GetAlignedSize(const Elf64_Phdr* p) { return (p->p_align != 0 ? (p->p_memsz + (p->p_align - 1)) & ~(p->p_align - 1) : p->p_memsz); } @@ -1038,7 +1180,7 @@ static void RelocateRecord(uint32_t index, Elf64_Rela* r, Program* program, bool // KYTY_PROFILER_BLOCK("patch"); if (ri.resolved) { - patched = Common::VirtualMemory::PatchReplace(ri.vaddr, ri.value); + patched = PatchGuestMemory64(ri.vaddr, ri.value); } else { uint64_t value = 0; bool weak = (ri.bind == BindType::Weak || !program->fail_if_global_not_resolved); @@ -1056,7 +1198,7 @@ static void RelocateRecord(uint32_t index, Elf64_Rela* r, Program* program, bool } if (value != 0) { - patched = Common::VirtualMemory::PatchReplace(ri.vaddr, value); + patched = PatchGuestMemory64(ri.vaddr, value); } else { auto dbg_str = fmt::format("[{:016x}] <- {:016x}, {}, {}, {}, {}", ri.vaddr, ri.value, ri.name.c_str(), Common::EnumName(ri.type).c_str(), @@ -1079,7 +1221,7 @@ static void RelocateRecord(uint32_t index, Elf64_Rela* r, Program* program, bool } if (value != 0) { - patched = Common::VirtualMemory::PatchReplace(ri.vaddr, value); + patched = PatchGuestMemory64(ri.vaddr, value); } } } @@ -1459,6 +1601,7 @@ void RuntimeLinker::Execute(const std::filesystem::path& game_patch) { PreloadAdjacentPrograms(); RelocateAll(); + if (!game_patch.empty()) { GamePatch::Apply(game_patch, m_programs.empty() ? nullptr : m_programs.front()); } @@ -1489,6 +1632,21 @@ void RuntimeLinker::Clear() { DeleteProgram(p); } m_programs.clear(); + for (const auto page: g_unresolved_stub_thunk_pages) { + EXIT_IF(!Libs::LibKernel::Memory::FreeGuestMemory(page, UNRESOLVED_STUB_PAGE_SIZE)); + } + g_unresolved_stub_thunk_pages.clear(); + g_unresolved_stub_thunk_offset = 0; + g_stubbed_imports.clear(); + g_unresolved_stub_call_log_count.store(0); + if (g_invalid_memory != 0) { + EXIT_IF(!Libs::LibKernel::Memory::FreeGuestMemory(g_invalid_memory, 4096)); + g_invalid_memory = 0; + } + g_tls_main_program = nullptr; + g_tls_cached_main_program = nullptr; + g_tls_cached_main_tcb = nullptr; + g_desired_base_addr = SYSTEM_RESERVED + CODE_BASE_OFFSET; m_symbols.reset(); m_relocated = false; } @@ -1926,10 +2084,11 @@ uint8_t* RuntimeLinker::TlsGetAddr(Program* program) { const auto tcb_offset = program->tls.tcb_offset != 0 ? program->tls.tcb_offset : program->tls.image_size; const auto alloc_size = AlignUp(tcb_offset, TCB_ALIGN) + TCB_SIZE; - tls.ptr = reinterpret_cast( - Common::VirtualMemory::Alloc(0, alloc_size, Common::VirtualMemory::Mode::ReadWrite)); - tls.free_func = nullptr; - tls.vm_alloc = true; + tls.ptr = reinterpret_cast(Libs::LibKernel::Memory::AllocateRuntimeMemory( + 0, alloc_size, Common::VirtualMemory::Mode::ReadWrite, "thread_local_storage")); + tls.free_func = nullptr; + tls.vm_alloc = true; + tls.alloc_size = alloc_size; EXIT_IF(tls.ptr == nullptr); @@ -2006,8 +2165,9 @@ void RuntimeLinker::LoadProgramToMemory(Program* program) { EXIT_IF(tls_handler_size > UINT64_MAX - program->base_size_aligned); program->mapped_size = program->base_size_aligned + tls_handler_size; - program->base_vaddr = Common::VirtualMemory::Alloc( - g_desired_base_addr, program->mapped_size, Common::VirtualMemory::Mode::ExecuteReadWrite); + program->base_vaddr = Libs::LibKernel::Memory::AllocateProgramMemory( + g_desired_base_addr, program->mapped_size, Common::VirtualMemory::Mode::ExecuteReadWrite, + Common::PathToString(program->file_name.filename()).c_str()); if (!is_shared) { program->tls.handler_vaddr = program->base_vaddr + program->base_size_aligned; @@ -2017,10 +2177,6 @@ void RuntimeLinker::LoadProgramToMemory(Program* program) { EXIT_IF(program->base_vaddr == 0); EXIT_IF(program->base_size_aligned < program->base_size); - Libs::LibKernel::Memory::RegisterProgramMemory( - program->base_vaddr, program->mapped_size, Common::VirtualMemory::Mode::ExecuteReadWrite, - Common::PathToString(program->file_name.filename()).c_str()); - LOGF("base_vaddr = 0x%016" PRIx64 "\n" "base_size = 0x%016" PRIx64 "\n" "base_size_aligned = 0x%016" PRIx64 "\n" @@ -2060,11 +2216,8 @@ void RuntimeLinker::LoadProgramToMemory(Program* program) { } if (!skip_protect) { - if (!Common::VirtualMemory::Protect(segment_addr, segment_memory_size, mode)) { - EXIT("failed to protect ELF segment %u\n", static_cast(i)); - } - Libs::LibKernel::Memory::UpdateProgramMemoryProtection(segment_addr, - segment_memory_size, mode); + Libs::LibKernel::Memory::SetProgramMemoryProtection(segment_addr, + segment_memory_size, mode); if (Common::VirtualMemory::IsExecute(mode)) { Common::VirtualMemory::FlushInstructionCache(segment_addr, segment_memory_size); @@ -2105,15 +2258,29 @@ void RuntimeLinker::LoadProgramToMemory(Program* program) { void RuntimeLinker::DeleteProgram(Program* p) { auto program = std::unique_ptr(p); + if (g_tls_main_program == program.get()) { + g_tls_main_program = nullptr; + } + if (g_tls_cached_main_program == program.get()) { + g_tls_cached_main_program = nullptr; + g_tls_cached_main_tcb = nullptr; + } + for (auto& record: g_stubbed_imports) { + if (record.patch_vaddr >= program->base_vaddr && + record.patch_vaddr < program->base_vaddr + program->mapped_size) { + record.patch_vaddr = 0; + } + } if (program->base_vaddr != 0 || program->mapped_size != 0) { EXIT_IF(program->base_vaddr == 0 || program->mapped_size == 0); - Libs::LibKernel::Memory::UnregisterProgramMemory(program->base_vaddr, program->mapped_size); - EXIT_IF(!Common::VirtualMemory::Free(program->base_vaddr)); + EXIT_IF( + !Libs::LibKernel::Memory::FreeGuestMemory(program->base_vaddr, program->mapped_size)); } if (program->custom_call_plt_vaddr != 0 || program->custom_call_plt_num != 0) { - Common::VirtualMemory::Free(program->custom_call_plt_vaddr); + const auto size = Jit::CallPlt::GetSize(program->custom_call_plt_num); + EXIT_IF(!Libs::LibKernel::Memory::FreeGuestMemory(program->custom_call_plt_vaddr, size)); } } @@ -2237,13 +2404,13 @@ static void InstallRelocateHandler(Program* program) { void** pltgot = reinterpret_cast(pltgot_vaddr); Common::VirtualMemory::Mode old_mode {}; - Common::VirtualMemory::Protect(pltgot_vaddr, pltgot_size, Common::VirtualMemory::Mode::Write, - &old_mode); + EXIT_IF(!Libs::LibKernel::Memory::ProtectGuestMemory( + pltgot_vaddr, pltgot_size, Common::VirtualMemory::Mode::Write, &old_mode)); pltgot[1] = program; pltgot[2] = reinterpret_cast(RelocateHandler); - Common::VirtualMemory::Protect(pltgot_vaddr, pltgot_size, old_mode); + EXIT_IF(!Libs::LibKernel::Memory::ProtectGuestMemory(pltgot_vaddr, pltgot_size, old_mode)); if (Common::VirtualMemory::IsExecute(old_mode)) { Common::VirtualMemory::FlushInstructionCache(pltgot_vaddr, pltgot_size); @@ -2253,15 +2420,15 @@ static void InstallRelocateHandler(Program* program) { if (program->custom_call_plt_vaddr == 0) { program->custom_call_plt_num = program->dynamic_info->jmprela_table_size / sizeof(Elf64_Rela); - auto size = Jit::CallPlt::GetSize(program->custom_call_plt_num); - program->custom_call_plt_vaddr = - Common::VirtualMemory::Alloc(SYSTEM_RESERVED, size, Common::VirtualMemory::Mode::Write); + auto size = Jit::CallPlt::GetSize(program->custom_call_plt_num); + program->custom_call_plt_vaddr = Libs::LibKernel::Memory::AllocateRuntimeMemory( + SYSTEM_RESERVED, size, Common::VirtualMemory::Mode::Write, "custom_call_plt"); EXIT_NOT_IMPLEMENTED(program->custom_call_plt_vaddr == 0); auto* code = new (reinterpret_cast(program->custom_call_plt_vaddr)) Jit::CallPlt(program->custom_call_plt_num); code->SetPltGot(pltgot_vaddr); - Common::VirtualMemory::Protect(program->custom_call_plt_vaddr, size, - Common::VirtualMemory::Mode::Execute); + EXIT_IF(!Libs::LibKernel::Memory::ProtectGuestMemory(program->custom_call_plt_vaddr, size, + Common::VirtualMemory::Mode::Execute)); Common::VirtualMemory::FlushInstructionCache(program->custom_call_plt_vaddr, size); } } @@ -2272,8 +2439,8 @@ void RuntimeLinker::Relocate(Program* program) { EXIT_IF(program == nullptr); if (g_invalid_memory == 0) { - g_invalid_memory = Common::VirtualMemory::Alloc(INVALID_MEMORY, 4096, - Common::VirtualMemory::Mode::NoAccess); + g_invalid_memory = Libs::LibKernel::Memory::AllocateRuntimeMemory( + INVALID_MEMORY, 4096, Common::VirtualMemory::Mode::NoAccess, "invalid_memory", true); EXIT_NOT_IMPLEMENTED(g_invalid_memory == 0); } @@ -2447,12 +2614,9 @@ void RuntimeLinker::SetupTlsHandler(Program* program) { stub->SetOutputReg(reg); } - if (!Common::VirtualMemory::Protect(program->tls.handler_vaddr, Jit::SafeCall::GetSize(), - Common::VirtualMemory::Mode::Execute)) { - EXIT("failed to protect program TLS handler\n"); - } - Libs::LibKernel::Memory::UpdateProgramMemoryProtection( - program->tls.handler_vaddr, Jit::SafeCall::GetSize(), Common::VirtualMemory::Mode::Execute); + EXIT_IF(!Libs::LibKernel::Memory::ProtectGuestMemory(program->tls.handler_vaddr, + Jit::SafeCall::GetSize(), + Common::VirtualMemory::Mode::Execute)); Common::VirtualMemory::FlushInstructionCache(program->tls.handler_vaddr, Jit::SafeCall::GetSize()); } diff --git a/src/loader/runtimeLinker.h b/src/loader/runtimeLinker.h index 8133e78..86a83f1 100644 --- a/src/loader/runtimeLinker.h +++ b/src/loader/runtimeLinker.h @@ -48,9 +48,10 @@ struct LibraryId { struct ThreadLocalStorage { struct Block { - uint8_t* ptr = nullptr; - application_heap_free_func_t free_func = nullptr; - bool vm_alloc = false; + uint8_t* ptr = nullptr; + application_heap_free_func_t free_func = nullptr; + bool vm_alloc = false; + uint64_t alloc_size = 0; }; ~ThreadLocalStorage(); @@ -204,6 +205,11 @@ private: application_heap_posix_memalign_func_t m_application_heap_posix_memalign = nullptr; }; +#if defined(KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS) +bool TestMainEntryUsesGuestStack(); +bool TestModuleRelocationUsesWritableHostMapping(); +#endif + } // namespace Loader #endif /* EMULATOR_INCLUDE_EMULATOR_LOADER_RUNTIMELINKER_H_ */ diff --git a/tests/MemoryTrackerTests.cpp b/tests/MemoryTrackerTests.cpp index 78e55d8..e95540b 100644 --- a/tests/MemoryTrackerTests.cpp +++ b/tests/MemoryTrackerTests.cpp @@ -1,6 +1,7 @@ #include "graphics/host_gpu/memoryTracker.h" #include "graphics/host_gpu/rangeSet.h" #include "common/assert.h" +#include "common/virtualMemory.h" #include #include @@ -209,6 +210,20 @@ class SharedPage final { HANDLE mapping_ = nullptr; }; #endif + +bool ProtectAddressSpace(uint64_t vaddr, uint64_t size, + Common::VirtualMemory::Mode mode) { + uint32_t protection = PAGE_NOACCESS; + if (mode == Common::VirtualMemory::Mode::Read) { + protection = PAGE_READONLY; + } else if (mode == Common::VirtualMemory::Mode::ReadWrite) { + protection = PAGE_READWRITE; + } + DWORD old_protection = 0; + return VirtualProtect(reinterpret_cast(vaddr), size, protection, + &old_protection) != 0; +} + #if 1 bool DummyFault(void *, PageFaultAccess, uint64_t, uint64_t, PageFaultPhase) noexcept { @@ -353,7 +368,8 @@ struct DownloadTrackerHarness { return completed; } - DownloadTrackerHarness() : page_manager(Fault, this), tracker(page_manager) {} + DownloadTrackerHarness() + : page_manager(Fault, this), tracker(page_manager) {} PageFaultAccess pending_access = PageFaultAccess::Unknown; uint64_t download_address = 0; @@ -366,11 +382,6 @@ struct DownloadTrackerHarness { std::atomic g_native_page_manager{nullptr}; std::atomic_bool g_native_fault_entered{false}; -std::atomic_bool g_unmap_contended{false}; - -void UnmapContended() noexcept { - g_unmap_contended.store(true, std::memory_order_release); -} #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS LONG CALLBACK NativeTrackerFaultHandler(EXCEPTION_POINTERS *exception) { @@ -552,7 +563,7 @@ void TestGpuDownloadFaultOwnership() { !harness.tracker.IsRegionGpuModified(address, page_size) && harness.tracker.IsRegionCpuModified(address, page_size) && IsWritable(memory), "GPU write fault did not download before granting CPU ownership"); - harness.tracker.UnmapMemory(address, page_size); + harness.tracker.UntrackMemory(address, page_size); } void TestVirtualGpuWriteDiscard() { @@ -578,7 +589,7 @@ void TestVirtualGpuWriteDiscard() { Check(!harness.tracker.IsRegionGpuModified(address, page_size) && harness.tracker.IsRegionCpuModified(address, page_size) && IsWritable(memory), "virtual GPU discard did not transfer the page to CPU ownership"); - harness.tracker.UnmapMemory(address, page_size); + harness.tracker.UntrackMemory(address, page_size); Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); } @@ -684,6 +695,15 @@ void TestRangeSet() { "range set subtraction did not preserve both exact tails"); } +void TestQueriesDoNotRequireMappedOwnership() { + constexpr uint64_t address = 0x0000000203000000ull; + TrackerHarness harness; + const auto page_size = harness.page_manager.GetPageSize(); + Check(harness.tracker.IsRegionCpuModified(address, page_size) && + !harness.tracker.IsRegionGpuModified(address, page_size), + "unowned tracker range did not expose its initial CPU-dirty state"); +} + void TestRangeInvalidation() { constexpr uintptr_t base = 0x0000000201000000ull; TrackerHarness harness; @@ -767,7 +787,7 @@ void TestCpuDirtyUploadAndFault() { Check(IsWritable(memory), "explicit CPU dirty transition did not release the rearmed watch"); - tracker.UnmapMemory(address, page_size * 2); + tracker.UntrackMemory(address, page_size * 2); Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); } @@ -795,7 +815,7 @@ void TestFaultDuringUploadRemainsDirty() { }); Check(tracker.IsRegionCpuModified(address, page_size) && IsWritable(memory), "upload completion erased a racing CPU dirty transition"); - tracker.UnmapMemory(address, page_size); + tracker.UntrackMemory(address, page_size); Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); } @@ -840,7 +860,7 @@ void TestNativeStoreDuringRangeEnumeration() { IsWritable(memory), "native store during range enumeration was lost"); - tracker.UnmapMemory(address, page_size); + tracker.UntrackMemory(address, page_size); Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); } @@ -924,7 +944,7 @@ void TestFaultDuringDownloadSynchronization() { page_manager.IsTracked(address + page_size * 2), "uncontended dirty page did not retain its clean write watch"); - tracker.UnmapMemory(address, page_size * 3); + tracker.UntrackMemory(address, page_size * 3); } void TestFaultAndExplicitDirtyRace() { @@ -964,7 +984,7 @@ void TestFaultAndExplicitDirtyRace() { IsWritable(memory), "fault/explicit-dirty race lost dirty state or write access"); } - tracker.UnmapMemory(address, page_size); + tracker.UntrackMemory(address, page_size); Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); } @@ -1016,7 +1036,7 @@ void TestSharedTrackersAndConcurrentPageFaults() { } harness.first.UntrackMemory(address, page_size * 2); - harness.second.UnmapMemory(address, page_size * 2); + harness.second.UntrackMemory(address, page_size * 2); Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); } @@ -1049,7 +1069,7 @@ void TestGpuDirtyBits() { "explicit GPU dirty transition did not trap CPU access"); tracker.UnmarkRegionAsGpuModified(address, page_size); tracker.MarkRegionAsCpuModified(address, page_size); - tracker.UnmapMemory(address, page_size * 2); + tracker.UntrackMemory(address, page_size * 2); Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); } @@ -1084,7 +1104,7 @@ void TestCrossRegionUpload() { "cross-region written upload did not mark GPU dirty state"); tracker.UnmarkRegionAsGpuModified(boundary - page_size, page_size * 2); tracker.MarkRegionAsCpuModified(boundary - page_size, page_size * 2); - tracker.UnmapMemory(address, region_size * 2); + tracker.UntrackMemory(address, region_size * 2); Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); } @@ -1094,9 +1114,6 @@ void TestCrossRegionUpload() { auto &tracker = harness.tracker; auto &page_manager = harness.page_manager; const auto page_size = page_manager.GetPageSize(); - if (std::strcmp(name, "unmapped") == 0) { - (void)tracker.IsRegionCpuModified(base, page_size); - } const auto allocation_size = std::strcmp(name, "missing-download-bytes") == 0 ? page_size * 2 : page_size; @@ -1149,20 +1166,6 @@ void TestCrossRegionUpload() { } }); fault.join(); - } else if (std::strcmp(name, "gpu-dirty-unmap-race") == 0) { - g_unmap_contended.store(false, std::memory_order_release); - MemoryTracker::SetUnmapContentionHook(UnmapContended); - std::thread unmap; - tracker.ForEachUploadRange( - address, page_size, true, [](uint64_t, uint64_t) noexcept {}, - [&]() noexcept { - unmap = std::thread( - [&] { tracker.UnmapMemory(address, page_size); }); - while (!g_unmap_contended.load(std::memory_order_acquire)) { - std::this_thread::yield(); - } - }); - unmap.join(); } else if (std::strcmp(name, "missing-download-bytes") == 0) { tracker.ForEachUploadRange( address, allocation_size, true, [](uint64_t, uint64_t) noexcept {}, @@ -1193,8 +1196,7 @@ void TestFatalPaths() { #endif for (const char *name : {"gpu-dirty-fault", "gpu-dirty-read", "virtual-gpu-read", "gpu-dirty-explicit-cpu", - "unmapped", "reentrant-upload", - "writable-upload-race", "gpu-dirty-unmap-race", + "reentrant-upload", "writable-upload-race", "missing-download-bytes"}) { #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS std::string command = std::string("\"") + path + "\" --death " + name; @@ -1236,6 +1238,14 @@ void TestFatalPaths() { } // namespace +namespace Libs::LibKernel::Memory { + +bool ProtectGuestHostMemory(uint64_t vaddr, uint64_t size, Common::VirtualMemory::Mode mode) { + return ProtectAddressSpace(vaddr, size, mode); +} + +} // namespace Libs::LibKernel::Memory + int main(int argc, char **argv) { #if 1 if (argc == 3 && std::strcmp(argv[1], "--death") == 0) { @@ -1249,6 +1259,7 @@ int main(int argc, char **argv) { TestSameSlabTrackerArbitration(); TestSharedMetadataAndImagePageFault(); TestRangeSet(); + TestQueriesDoNotRequireMappedOwnership(); TestRangeInvalidation(); TestGpuDirtyBits(); TestCrossRegionUpload(); diff --git a/tests/PageManagerTests.cpp b/tests/PageManagerTests.cpp index a0c02ed..2af8728 100644 --- a/tests/PageManagerTests.cpp +++ b/tests/PageManagerTests.cpp @@ -1,4 +1,5 @@ #include "graphics/host_gpu/pageManager.h" +#include "common/virtualMemory.h" #include #include @@ -28,7 +29,6 @@ namespace { -using Libs::Graphics::GpuAccess; using Libs::Graphics::PageFaultAccess; using Libs::Graphics::PageManager; @@ -122,6 +122,23 @@ uint32_t Protection(const void *address) { return info.Protect; } #endif + +std::atomic_uint64_t g_protection_calls{0}; + +bool ProtectAddressSpace(uint64_t vaddr, uint64_t size, + Common::VirtualMemory::Mode mode) { + uint32_t protection = PAGE_NOACCESS; + if (mode == Common::VirtualMemory::Mode::Read) { + protection = PAGE_READONLY; + } else if (mode == Common::VirtualMemory::Mode::ReadWrite) { + protection = PAGE_READWRITE; + } + DWORD old_protection = 0; + g_protection_calls.fetch_add(1, std::memory_order_relaxed); + return VirtualProtect(reinterpret_cast(vaddr), size, protection, + &old_protection) != 0; +} + #if 1 struct FaultContext { @@ -254,6 +271,7 @@ uint8_t *Allocate(uint64_t size, uint32_t protection = PAGE_READWRITE) { } void TestWatchFaultAndUnwatch() { + g_protection_calls.store(0, std::memory_order_relaxed); FaultContext context; PageManager manager(InvalidateFault, &context); context.manager = &manager; @@ -266,6 +284,8 @@ void TestWatchFaultAndUnwatch() { Check(manager.IsTracked(reinterpret_cast(memory)) && !IsWritable(memory), "watch did not protect the page"); + Check(g_protection_calls.load(std::memory_order_relaxed) != 0, + "watch protection bypassed the address-space owner callback"); Check(manager.HandleFault(PageFaultAccess::Write, reinterpret_cast(memory + 32)), "tracked write fault was not handled"); @@ -343,14 +363,6 @@ void TestPermittedMappedLateFaultsResume() { "second delayed mapped write was not accepted"); Check(manager.HandleFault(PageFaultAccess::Read, address), "delayed mapped read was not accepted on readable backing"); - DWORD old_protection = 0; - Check(VirtualProtect(memory, page_size, PAGE_READONLY, &old_protection) != 0 && - old_protection == PAGE_READWRITE, - "failed to prepare intentional read-only protection"); - Check(!manager.HandleFault(PageFaultAccess::Write, address), - "intentional read-only mapping accepted a write fault"); - Check(VirtualProtect(memory, page_size, PAGE_READWRITE, &old_protection) != 0, - "failed to restore writable protection"); manager.OnGpuUnmap(address, page_size); Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); } @@ -496,31 +508,6 @@ void TestNativeAccessViolation() { Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); } -void TestInvalidLateWriteTokenIsConsumed() { - FaultContext context; - PageManager manager(InvalidateFault, &context); - context.manager = &manager; - const auto page_size = manager.GetPageSize(); - auto *memory = Allocate(page_size); - const auto address = reinterpret_cast(memory); - manager.OnGpuMap(address, page_size); - manager.UpdatePageWatchers(true, address, page_size); - Check(manager.HandleFault(PageFaultAccess::Write, address), - "initial write fault was not handled"); - DWORD old_protection = 0; - Check(VirtualProtect(memory, page_size, PAGE_READONLY, &old_protection) != - 0 && - old_protection == PAGE_READWRITE, - "failed to create invalid late-write protection state"); - Check(!manager.HandleFault(PageFaultAccess::Write, address) && - !manager.HandleFault(PageFaultAccess::Write, address), - "invalid late-write token was accepted or retained"); - Check(VirtualProtect(memory, page_size, PAGE_READWRITE, &old_protection) != 0, - "failed to restore test protection"); - manager.OnGpuUnmap(address, page_size); - Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); -} - void TestCrossRegionRange() { FaultContext context; PageManager manager(InvalidateFault, &context); @@ -627,9 +614,7 @@ void TestBatchedWatcherRanges() { manager->UpdatePageWatchers(false, 0x1000, page_size); } else { const bool two_pages = std::strcmp(name, "cross-reentrant") == 0; - auto *memory = Allocate( - two_pages ? page_size * 2 : page_size, - std::strcmp(name, "protection") == 0 ? PAGE_READONLY : PAGE_READWRITE); + auto *memory = Allocate(two_pages ? page_size * 2 : page_size); const auto address = reinterpret_cast(memory); manager->OnGpuMap(address, two_pages ? page_size * 2 : page_size); manager->UpdatePageWatchers(true, address, page_size); @@ -659,9 +644,7 @@ void TestBatchedWatcherRanges() { } (void)manager->HandleFault(PageFaultAccess::Read, address); first.join(); - } else if (std::strcmp(name, "watched-unmap") == 0) { - manager->OnGpuUnmap(address, page_size); - } else if (std::strcmp(name, "protection") != 0) { + } else { std::_Exit(0x7f); } } @@ -712,7 +695,7 @@ void TestFatalPaths() { for (const char *name : {"invalid-range", "unknown-untrack", "destructor-watch", "non-write", "callback-false", "reentrant", "cross-reentrant", - "concurrent-non-write", "watched-unmap", "protection"}) { + "concurrent-non-write"}) { CheckDeathCase(name); } } @@ -773,51 +756,18 @@ void TestExternalDirtyTransferDuringResolution() { Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); } -void TestMappingDoesNotRequireCpuWriteAccess() { - FaultContext context; - PageManager manager(InvalidateFault, &context); - context.manager = &manager; - const auto page_size = manager.GetPageSize(); - auto *memory = Allocate(page_size); - DWORD old_protection = 0; - Check(VirtualProtect(memory, page_size, PAGE_NOACCESS, &old_protection) != 0 && - old_protection == PAGE_READWRITE, - "failed to prepare CPU-inaccessible mapping"); - const auto address = reinterpret_cast(memory); - manager.OnGpuMap(address, page_size); - Check(manager.IsMapped(address, page_size), - "CPU-inaccessible committed range was not GPU mapped"); - manager.OnGpuUnmap(address, page_size); - Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); -} - -void TestGpuAccessPermissions() { - FaultContext context; - PageManager manager(InvalidateFault, &context); - context.manager = &manager; - const auto page_size = manager.GetPageSize(); - auto *memory = Allocate(page_size); - const auto address = reinterpret_cast(memory); - manager.OnGpuMap(address, page_size, GpuAccess::Read); - Check(manager.HasGpuAccess(address, page_size, GpuAccess::Read) && - !manager.HasGpuAccess(address, page_size, GpuAccess::Write), - "read-only GPU mapping granted write access"); - manager.OnGpuMap(address, page_size, GpuAccess::Write); - Check(manager.HasGpuAccess(address, page_size, GpuAccess::ReadWrite), - "overlapping GPU mappings did not combine permissions"); - manager.OnGpuUnmap(address, page_size, GpuAccess::Read); - Check(!manager.HasGpuAccess(address, page_size, GpuAccess::Read) && - manager.HasGpuAccess(address, page_size, GpuAccess::Write), - "GPU read unmap removed the wrong permission"); - manager.OnGpuUnmap(address, page_size, GpuAccess::Write); - Check(!manager.IsMapped(address, page_size), - "GPU permission mappings were not fully balanced"); - Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); -} #endif } // namespace +namespace Libs::LibKernel::Memory { + +bool ProtectGuestHostMemory(uint64_t vaddr, uint64_t size, Common::VirtualMemory::Mode mode) { + return ProtectAddressSpace(vaddr, size, mode); +} + +} // namespace Libs::LibKernel::Memory + int main(int argc, char **argv) { #if 1 if (argc == 3 && std::strcmp(argv[1], "--death") == 0) { @@ -831,13 +781,10 @@ int main(int argc, char **argv) { TestNativeDelayedReadAfterModeDowngrade(); TestDelayedFaultAfterExplicitUnwatch(); TestNativeAccessViolation(); - TestInvalidLateWriteTokenIsConsumed(); TestCrossRegionRange(); TestBatchedWatcherRanges(); TestConcurrentFault(); TestExternalDirtyTransferDuringResolution(); - TestMappingDoesNotRequireCpuWriteAccess(); - TestGpuAccessPermissions(); TestFatalPaths(); std::puts("PageManagerTests: all cases passed"); return 0; diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index 94b7677..7f76f39 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -1274,6 +1274,59 @@ public: std::printf("[host] %-32s ok\n", "SchedulerTimeline"); } + void CheckGpuMappedRangeLifecycle() { + EnsureRuntimeContext(); + CommandScheduler scheduler(Renderer(), m_runtime_context); + HW::Context registers{}; + HW::UserConfig user_config{}; + HW::Shader shaders{}; + scheduler.Begin(registers, user_config, shaders); + Gpu gpu(Renderer()); + GpuResourceManager resources(m_runtime_context, scheduler); + resources.SetGpu(&gpu); + + constexpr uint64_t base = 0x0000000200000000ull; + constexpr uint64_t page = 0x4000; + resources.MapMemory(base, page * 4); + resources.MapMemory(base + page * 2, page * 4); + Require("GpuMappedRangeLifecycle", "union", + resources.IsMapped(base, page * 6) && + !resources.IsMapped(base, page * 7), + "overlapping maps did not form one interval union"); + + resources.UnmapMemory(base + page * 2, page * 2); + Require("GpuMappedRangeLifecycle", "subtract", + resources.IsMapped(base, page * 2) && + resources.IsMapped(base + page * 4, page * 2) && + !resources.IsMapped(base, page * 6), + "partial unmap did not punch the expected interval hole"); + + resources.UnmapMemory(base + page * 2, page * 2); + Require("GpuMappedRangeLifecycle", "idempotent unmap", + resources.IsMapped(base, page * 2) && + resources.IsMapped(base + page * 4, page * 2), + "unmapping an absent interval changed neighboring mappings"); + + resources.UnmapMemory(base, page * 6); + Require("GpuMappedRangeLifecycle", "clear", + !resources.IsMapped(base, page * 6), + "full unmap did not clear the interval union"); + + constexpr uint64_t old_prt = base + page * 8; + constexpr uint64_t new_prt = base + page * 16; + resources.MapMemory(old_prt, page * 4); + resources.UnmapMemory(old_prt, page * 4); + resources.MapMemory(new_prt, page * 6); + Require("GpuMappedRangeLifecycle", "PRT replacement", + !resources.IsMapped(old_prt, page * 4) && + resources.IsMapped(new_prt, page * 6), + "old-unmap/new-map did not replace full PRT coverage"); + + resources.SetGpu(nullptr); + scheduler.Finish(); + std::printf("[host] %-32s ok\n", "GpuMappedRangeLifecycle"); + } + void CheckStreamBufferRing() { EnsureRuntimeContext(); CommandScheduler scheduler(Renderer(), m_runtime_context); @@ -1526,7 +1579,7 @@ public: fault_memory == reinterpret_cast(fault_base), "fixed processor-fault allocation failed"); auto &resources = context.GetGpuResources(); - resources.MapMemory(fault_base, fault_size, GpuAccess::ReadWrite); + resources.MapMemory(fault_base, fault_size); constexpr uint64_t immediate_dst = fault_base + 0x1000; constexpr uint64_t immediate_memory_dst = fault_base + 0x2000; @@ -1683,7 +1736,7 @@ public: resources.InvalidateMemory(fault_base, sizeof(uint32_t)), "processor memory invalidation did not find its mapped range"); }); - resources.UnmapMemory(fault_base, fault_size, GpuAccess::ReadWrite); + resources.UnmapMemory(fault_base, fault_size); Require("GpuCommandLane", "processor fault unmap", Libs::LibKernel::Memory::KernelMunmap(fault_base, fault_size) == 0, "processor-fault direct-memory mapping release failed"); @@ -1953,7 +2006,7 @@ public: GpuResourceManager resources(m_runtime_context, scheduler); resources.SetGpu(&gpu); auto &cache = resources.GetBufferCache(); - resources.MapMemory(base, allocation_size, GpuAccess::ReadWrite); + resources.MapMemory(base, allocation_size); const auto MarkGpuWrite = [&](uint64_t address, uint64_t size) { auto allocation = @@ -2376,7 +2429,7 @@ public: sizeof(reacquire_value)); resources.SetGpu(nullptr); - resources.UnmapMemory(base, allocation_size, GpuAccess::ReadWrite); + resources.UnmapMemory(base, allocation_size); scheduler.Finish(); } gpu.Shutdown(); @@ -2431,7 +2484,7 @@ public: narrow_download != nullptr && narrow_download_offset % 4 == 0 && wide_download != nullptr && wide_download_offset % 16 == 0, "wide/block image readback was not aligned to its texel block"); - resources.MapMemory(base, allocation_size, GpuAccess::ReadWrite); + resources.MapMemory(base, allocation_size); ImageDesc sampled{}; sampled.type = BindingType::Texture; @@ -4834,7 +4887,7 @@ public: m_device.destroyShaderModule(ms_depth_module, nullptr); resources.SetGpu(nullptr); - resources.UnmapMemory(base, allocation_size, GpuAccess::ReadWrite); + resources.UnmapMemory(base, allocation_size); scheduler.Finish(); } gpu.Shutdown(); @@ -4879,7 +4932,7 @@ public: scheduler.Begin(registers, user_config, shaders); { GpuResourceManager resources(m_runtime_context, scheduler); - resources.MapMemory(base, allocation_size, GpuAccess::ReadWrite); + resources.MapMemory(base, allocation_size); const uint32_t pitch = TileGetTexturePitch(format, 1, 1, tile); TileSizeAlign total{}; TileSizeOffset mip{}; @@ -4978,7 +5031,7 @@ public: std::vector{0x40004200u, 0x44003c00u}, "tiled BGRA16 Buffer mirror changed guest component order"); DestroyBuffer(&mirror_readback); - resources.UnmapMemory(base, allocation_size, GpuAccess::ReadWrite); + resources.UnmapMemory(base, allocation_size); scheduler.Finish(); } Require(name, "unmap", @@ -5025,7 +5078,7 @@ public: auto &resources = context.GetGpuResources(); auto &texture_cache = resources.GetTextureCache(); auto &executor = context.GetRenderExecutor(); - resources.MapMemory(base, allocation_size, GpuAccess::ReadWrite); + resources.MapMemory(base, allocation_size); constexpr auto stencil_format = Prospero::GpuEnumValue(Prospero::BufferFormat::k8UInt); @@ -6011,10 +6064,8 @@ public: const auto stale_ordered_color = texture_cache.FindImage(ordered_color_desc); RenderExecutorTestAccess::BindRenderTarget(executor, stale_ordered_color); - resources.UnmapMemory(ordered_color_address, target_mip_size, - GpuAccess::ReadWrite); - resources.MapMemory(ordered_color_address, target_mip_size, - GpuAccess::ReadWrite); + resources.UnmapMemory(ordered_color_address, target_mip_size); + resources.MapMemory(ordered_color_address, target_mip_size); auto ordered_depth_desc = depth; ordered_depth_desc.info.stencil = {ordered_color_address, @@ -6167,7 +6218,7 @@ public: texture_cache.GetImage(depth_id).usage.storage, "storage stencil binding did not acquire the associated depth owner"); RenderExecutorTestAccess::ResetBindings(executor); - resources.UnmapMemory(base, allocation_size, GpuAccess::ReadWrite); + resources.UnmapMemory(base, allocation_size); scheduler.Finish(); } @@ -17198,12 +17249,12 @@ void CheckStandard64RenderTargetTileRoundTrip() { void CheckStorageTextureGpuOwnedRebindState() { constexpr uintptr_t base = 0x0000000200200000ull; constexpr uint64_t size = 0x10000; - auto *memory = static_cast( - VirtualAlloc(reinterpret_cast(base), size, - MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); + const auto guest_memory = Libs::LibKernel::Memory::AllocateRuntimeMemory( + base, size, Common::VirtualMemory::Mode::ReadWrite, + "storage_texture_gpu_owned_rebind", true); + auto *memory = reinterpret_cast(guest_memory); Require("StorageTextureGpuOwnedRebind", "allocation", - memory == reinterpret_cast(base), - "fixed VirtualAlloc failed"); + guest_memory == base, "fixed guest-owner allocation failed"); PageManager page_manager(CacheFault, nullptr); MemoryTracker tracker(page_manager); page_manager.OnGpuMap(base, size); @@ -17215,7 +17266,6 @@ void CheckStorageTextureGpuOwnedRebindState() { Require( "StorageTextureGpuOwnedRebind", "owned", tracker.IsRegionGpuModified(base, size) && - page_manager.IsMapped(base, size) && (!HostMemoryQueryReadable(base, size, readable) || readable < size) && HostMemoryQueryRange(base, size, HostMemoryAccess::Mapped, mapped) && mapped == size && @@ -17261,7 +17311,8 @@ void CheckStorageTextureGpuOwnedRebindState() { tracker.UntrackMemory(base, size); page_manager.OnGpuUnmap(base, size); Require("StorageTextureGpuOwnedRebind", "free", - VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); + Libs::LibKernel::Memory::FreeGuestMemory(base, size), + "guest-owner free failed"); std::printf("[host] %-32s ok\n", "StorageTextureGpuOwnedRebind"); } #endif @@ -18029,6 +18080,11 @@ int main(int argc, char **argv) { vulkan.CheckSchedulerTimeline(); return 0; } + if (argc == 2 && std::strcmp(argv[1], "--mapped-range-only") == 0) { + VulkanHarness vulkan; + vulkan.CheckGpuMappedRangeLifecycle(); + return 0; + } if (argc == 2 && std::strcmp(argv[1], "--stream-buffer-only") == 0) { VulkanHarness vulkan; vulkan.CheckStreamBufferRing(); @@ -18184,6 +18240,7 @@ int main(int argc, char **argv) { CheckEmbeddedFetchLaneSpill(); CheckPs5GameExampleImageClearRuntimeShape(); vulkan.CheckSchedulerTimeline(); + vulkan.CheckGpuMappedRangeLifecycle(); vulkan.CheckStreamBufferRing(); vulkan.CheckCommandPoolGrowth(); vulkan.CheckGpuTilerCpuParity(); diff --git a/tests/VirtualMemoryAllocationTests.cpp b/tests/VirtualMemoryAllocationTests.cpp index 6c337ce..adf7c82 100644 --- a/tests/VirtualMemoryAllocationTests.cpp +++ b/tests/VirtualMemoryAllocationTests.cpp @@ -1,15 +1,21 @@ #include "common/commonSubsystem.h" #include "common/emulatorConfig.h" +#include "common/file.h" #include "common/logging/log.h" #include "common/subsystems.h" #include "common/threads.h" +#include "common/virtualMemory.h" #include "kernel/memory.h" +#include "kernel/pthread.h" #include "libs/errno.h" +#include "loader/runtimeLinker.h" +#include "loader/systemContent.h" #include #include #include #include +#include #include namespace { @@ -18,11 +24,16 @@ using Libs::LibKernel::Memory::VirtualQueryInfo; // Prospero ABI? constexpr uint64_t SceKernelPageSize = 0x4000; +constexpr uint64_t SceKernelTotalPhysicalSize = 13824ull * 1024ull * 1024ull; +constexpr uint64_t TestFlexibleMemorySize = 3072ull * 1024ull * 1024ull; constexpr int SceKernelProtCpuRead = 0x01; constexpr int SceKernelProtCpuRw = 0x02; +constexpr int SceKernelProtCpuExec = 0x04; constexpr int SceKernelMapFixed = 0x10; constexpr int SceKernelMapNoOverwrite = 0x80; +constexpr int SceKernelMapDmemCompat = 0x400; constexpr int SceKernelMapNoCoalesce = 0x400000; +constexpr int SceKernelMapAligned64Kb = 16 << 24; constexpr int SceKernelVqFindNext = 1; constexpr int SceKernelMtypeC = 11; constexpr uint64_t SceKernelDirectMemoryStart = 0; @@ -94,6 +105,29 @@ void InitSubsystems() { Config::Load(options); slist->Add(log, {core, config}); + Check("InitSubsystems", slist->InitAll(false), "failed to initialize logging subsystem"); + + const auto param_json = + std::filesystem::temp_directory_path() / + ("kyty_virtual_memory_" + + std::to_string(reinterpret_cast(&initialized)) + ".json"); + constexpr char json[] = R"({"kernel":{"flexibleMemorySize":3221225472}})"; + Common::File param_file; + Check("InitSubsystems", param_file.Create(param_json), "failed to create temporary param.json"); + uint32_t bytes_written = 0; + param_file.Write(json, sizeof(json) - 1, &bytes_written); + param_file.Close(); + Check("InitSubsystems", bytes_written == sizeof(json) - 1, + "failed to write temporary param.json"); + + Loader::SystemContentLoadParamSfo(param_json); + const auto flexible_memory_size = Loader::SystemContentGetFlexibleMemorySize(); + Check("InitSubsystems", Common::File::DeleteFile(param_json), + "failed to remove temporary param.json"); + Check("InitSubsystems", flexible_memory_size == TestFlexibleMemorySize, + "failed to read flexible memory size from param.json"); + Libs::LibKernel::Memory::SetFlexibleMemorySize(flexible_memory_size); + slist->Add(memory, {core, log, thread}); Check("InitSubsystems", slist->InitAll(false), "failed to initialize memory subsystem"); @@ -131,6 +165,13 @@ size_t AvailableFlexibleMemory(const char* test) { return size; } +size_t ConfiguredFlexibleMemory(const char* test) { + size_t size = 0; + CheckOk(test, Libs::LibKernel::Memory::KernelConfiguredFlexibleMemorySize(&size), + "KernelConfiguredFlexibleMemorySize"); + return size; +} + uint64_t MapNamedFlexible(const char* test, uint64_t size, int prot, const char* name) { void* addr = nullptr; const int ret = @@ -190,6 +231,387 @@ void TestProsperoArgumentAndInfoSizeContracts() { std::printf("[host] %-48s ok\n", test); } +void TestGuestAddressSpaceOwnsReservationsBeforeBacking() { + const char* test = "GuestAddressSpaceOwnsReservationsBeforeBacking"; + void* addr = nullptr; + + Check(test, Libs::LibKernel::Memory::TestGuestBackingOutsideAddressSpace(), + "boot-time shared backing alias overlaps an owned guest interval"); + CheckOk(test, + Libs::LibKernel::Memory::KernelReserveVirtualRange(&addr, SceKernelPageSize, 0, + SceKernelPageSize), + "KernelReserveVirtualRange"); + const auto base = reinterpret_cast(addr); + Check(test, Libs::LibKernel::Memory::TestGuestAddressRangeIsOwned(base, SceKernelPageSize), + "guest reservation was allocated outside the early owner"); + Check(test, Libs::LibKernel::Memory::TestPlaceholderRangeIsFree(base, SceKernelPageSize), + "semantic reservation replaced the owner's placeholder"); + Check(test, + Libs::LibKernel::Memory::ProtectGuestHostMemory( + base, SceKernelPageSize, Common::VirtualMemory::Mode::NoAccess), + "owner rejected a sparse placeholder protection no-op"); + CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(base, SceKernelPageSize), "KernelMunmap"); + Check(test, Libs::LibKernel::Memory::TestPlaceholderRangeIsFree(base, SceKernelPageSize), + "released semantic reservation escaped owner control"); + + std::printf("[host] %-48s ok\n", test); +} + +void TestGuestAddressSpaceHasNoFixedFallback() { + const char* test = "GuestAddressSpaceHasNoFixedFallback"; + const auto unowned_address = reinterpret_cast(0x10000); + void* addr = unowned_address; + + CheckFailed(test, + Libs::LibKernel::Memory::KernelReserveVirtualRange( + &addr, SceKernelPageSize, SceKernelMapFixed | SceKernelMapNoOverwrite, + SceKernelPageSize), + "KernelReserveVirtualRange(unowned fixed address)"); + Check(test, reinterpret_cast(addr) == 0x10000, + "failed fixed reservation unexpectedly moved"); + + addr = unowned_address; + CheckFailed(test, + Libs::LibKernel::Memory::KernelMapNamedFlexibleMemory( + &addr, SceKernelPageSize, SceKernelProtCpuRw, + SceKernelMapFixed | SceKernelMapNoOverwrite, "unowned_flexible"), + "KernelMapNamedFlexibleMemory(unowned fixed address)"); + + int64_t phys_addr = -1; + CheckOk(test, + Libs::LibKernel::Memory::KernelAllocateDirectMemory( + 0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(), SceKernelPageSize, + SceKernelPageSize, SceKernelMtypeC, &phys_addr), + "KernelAllocateDirectMemory"); + addr = unowned_address; + CheckFailed(test, + Libs::LibKernel::Memory::KernelMapNamedDirectMemory( + &addr, SceKernelPageSize, SceKernelProtCpuRw, + SceKernelMapFixed | SceKernelMapNoOverwrite, phys_addr, SceKernelPageSize, + "unowned_direct"), + "KernelMapNamedDirectMemory(unowned fixed address)"); + CheckOk(test, + Libs::LibKernel::Memory::KernelCheckedReleaseDirectMemory(phys_addr, SceKernelPageSize), + "KernelCheckedReleaseDirectMemory"); + + std::printf("[host] %-48s ok\n", test); +} + +void TestGuestFreeRangeSearchDoesNotUnderflow() { + const char* test = "GuestFreeRangeSearchDoesNotUnderflow"; + + Check(test, Libs::LibKernel::Memory::TestGuestFreeRangeBounds(), + "free-range containment accepted a candidate beyond the range end"); + + std::printf("[host] %-48s ok\n", test); +} + +void TestFlexibleMemoryCapacityIsBootFixed() { + const char* test = "FlexibleMemoryCapacityIsBootFixed"; + const auto configured = ConfiguredFlexibleMemory(test); + const auto baseline = AvailableFlexibleMemory(test); + const auto backing = Libs::LibKernel::Memory::TestGuestBackingSize(); + + Check(test, configured == TestFlexibleMemorySize, + "boot flexible pool did not use the param.json value"); + Check(test, configured == baseline, "boot flexible pool did not start at configured capacity"); + Check(test, backing == SceKernelTotalPhysicalSize, + "boot backing is not the single 13.5 GiB physical file"); + Check(test, backing == Libs::LibKernel::Memory::KernelGetDirectMemorySize() + configured, + "direct and flexible regions do not partition the boot backing"); + + const auto address = + MapNamedFlexible(test, SceKernelPageSize, SceKernelProtCpuRw, "boot_fixed_flexible"); + Check(test, ConfiguredFlexibleMemory(test) == configured, + "configured flexible capacity changed after allocation"); + Check(test, Libs::LibKernel::Memory::TestGuestBackingSize() == backing, + "shared backing size changed after allocation"); + Check(test, AvailableFlexibleMemory(test) == baseline - SceKernelPageSize, + "flexible allocation did not consume the boot-time pool"); + + CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(address, SceKernelPageSize), + "KernelMunmap"); + Check(test, ConfiguredFlexibleMemory(test) == configured, + "configured flexible capacity changed after release"); + Check(test, AvailableFlexibleMemory(test) == baseline, + "flexible release did not restore the boot-time pool"); + + std::printf("[host] %-48s ok\n", test); +} + +void TestFlexibleMemoryUsesSharedBacking() { + const char* test = "FlexibleMemoryUsesSharedBacking"; + const auto baseline = AvailableFlexibleMemory(test); + void* address = nullptr; + CheckOk(test, + Libs::LibKernel::Memory::KernelMapNamedFlexibleMemory( + &address, SceKernelPageSize * 2, SceKernelProtCpuRw, 0, "shared_flexible"), + "KernelMapNamedFlexibleMemory"); + const auto base = reinterpret_cast(address); + Check(test, Libs::LibKernel::Memory::TestGuestAddressRangeIsOwned(base, SceKernelPageSize * 2), + "flexible mapping escaped the guest owner"); + + constexpr uint64_t first_value = 0x464c45584241434bull; // "FLEXBACK" + constexpr uint64_t second_value = 0x534841524544464cull; // "SHAREDFL" + *reinterpret_cast(base) = first_value; + uint64_t value = 0; + Check(test, Libs::LibKernel::Memory::TryReadBacking(base, &value, sizeof(value)), + "TryReadBacking did not resolve flexible memory"); + Check(test, value == first_value, "backing did not observe a flexible-memory CPU write"); + Check(test, + Libs::LibKernel::Memory::TryWriteBacking(base + SceKernelPageSize, &second_value, + sizeof(second_value)), + "TryWriteBacking did not resolve flexible memory"); + Check(test, *reinterpret_cast(base + SceKernelPageSize) == second_value, + "flexible-memory view did not observe a backing write"); + + CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(base, SceKernelPageSize * 2), + "KernelMunmap"); + Check(test, AvailableFlexibleMemory(test) == baseline, + "flexible backing offsets were not returned to the boot-time pool"); + Check(test, !Libs::LibKernel::Memory::TryReadBacking(base, &value, sizeof(value)), + "unmapped flexible memory remained registered in the backing owner"); + + std::printf("[host] %-48s ok\n", test); +} + +void TestFlexibleDmemCompatAndAlignmentFlags() { + const char* test = "FlexibleDmemCompatAndAlignmentFlags"; + const auto baseline = AvailableFlexibleMemory(test); + void* address = nullptr; + + CheckOk(test, + Libs::LibKernel::Memory::KernelMapNamedFlexibleMemory( + &address, SceKernelPageSize, SceKernelProtCpuRw, + SceKernelMapDmemCompat | SceKernelMapAligned64Kb, "dmem_compat"), + "KernelMapNamedFlexibleMemory(DMEM_COMPAT|ALIGNED_64KB)"); + const auto base = reinterpret_cast(address); + Check(test, (base & (0x10000 - 1u)) == 0, "SDK alignment flag was not honored"); + const auto info = Query(test, base); + Check(test, info.is_flexible == 1 && info.is_stack == 0, + "SCE_KERNEL_MAP_DMEM_COMPAT was misclassified as MAP_STACK"); + Check(test, AvailableFlexibleMemory(test) + SceKernelPageSize == baseline, + "DMEM_COMPAT mapping did not consume boot-time flexible backing"); + + void* stack_start = reinterpret_cast(UINT64_MAX); + void* stack_end = reinterpret_cast(UINT64_MAX); + CheckOk(test, + Libs::LibKernel::Memory::KernelIsStack(reinterpret_cast(base), &stack_start, + &stack_end), + "KernelIsStack"); + Check(test, stack_start == nullptr && stack_end == nullptr, + "DMEM_COMPAT flexible mapping was reported as a stack"); + + CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(base, SceKernelPageSize), + "KernelMunmap"); + Check(test, AvailableFlexibleMemory(test) == baseline, + "DMEM_COMPAT cleanup did not restore flexible capacity"); + + void* opaque = nullptr; + CheckOk(test, + Libs::LibKernel::Memory::KernelMapNamedFlexibleMemory( + &opaque, SceKernelPageSize, SceKernelProtCpuRw, 0x8000, "opaque_runtime_flag"), + "KernelMapNamedFlexibleMemory(opaque runtime flag)"); + Check(test, AvailableFlexibleMemory(test) + SceKernelPageSize == baseline, + "opaque runtime flag mapping did not consume boot-time flexible backing"); + CheckOk(test, + Libs::LibKernel::Memory::KernelMunmap(reinterpret_cast(opaque), + SceKernelPageSize), + "KernelMunmap(opaque runtime flag)"); + Check(test, AvailableFlexibleMemory(test) == baseline, + "opaque runtime flag cleanup did not restore flexible capacity"); + + void* invalid_flag = nullptr; + CheckFailed( + test, + Libs::LibKernel::Memory::KernelMapNamedFlexibleMemory( + &invalid_flag, SceKernelPageSize, SceKernelProtCpuRw, 0x10000, "unsupported_flag"), + "KernelMapNamedFlexibleMemory(unsupported flag)"); + + void* invalid_alignment = nullptr; + CheckFailed(test, + Libs::LibKernel::Memory::KernelMapNamedFlexibleMemory( + &invalid_alignment, SceKernelPageSize, SceKernelProtCpuRw, 13 << 24, + "invalid_alignment"), + "KernelMapNamedFlexibleMemory(invalid alignment)"); + + std::printf("[host] %-48s ok\n", test); +} + +void TestFlexibleNoCoalescePreservesBoundaries() { + const char* test = "FlexibleNoCoalescePreservesBoundaries"; + const auto baseline = AvailableFlexibleMemory(test); + void* reserve = nullptr; + CheckOk(test, + Libs::LibKernel::Memory::KernelReserveVirtualRange( + &reserve, SceKernelPageSize * 2, 0, SceKernelPageSize), + "KernelReserveVirtualRange"); + const auto base = reinterpret_cast(reserve); + + void* left = reinterpret_cast(base); + CheckOk(test, + Libs::LibKernel::Memory::KernelMapNamedFlexibleMemory( + &left, SceKernelPageSize, SceKernelProtCpuRw, + SceKernelMapFixed | SceKernelMapNoCoalesce, "no_coalesce"), + "KernelMapNamedFlexibleMemory(left)"); + void* right = reinterpret_cast(base + SceKernelPageSize); + CheckOk(test, + Libs::LibKernel::Memory::KernelMapNamedFlexibleMemory( + &right, SceKernelPageSize, SceKernelProtCpuRw, + SceKernelMapFixed | SceKernelMapNoCoalesce, "no_coalesce"), + "KernelMapNamedFlexibleMemory(right)"); + + ExpectRange(test, Query(test, base), base, base + SceKernelPageSize, SceKernelProtCpuRw, 1, 0, + 0, 1, "no_coalesce"); + ExpectRange(test, Query(test, base + SceKernelPageSize), base + SceKernelPageSize, + base + SceKernelPageSize * 2, SceKernelProtCpuRw, 1, 0, 0, 1, "no_coalesce"); + + CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(base, SceKernelPageSize * 2), + "KernelMunmap"); + Check(test, AvailableFlexibleMemory(test) == baseline, + "NO_COALESCE cleanup did not restore flexible capacity"); + + std::printf("[host] %-48s ok\n", test); +} + +void TestFlexibleMemoryReuseIsZeroFilled() { + const char* test = "FlexibleMemoryReuseIsZeroFilled"; + const auto baseline = AvailableFlexibleMemory(test); + const auto first = + MapNamedFlexible(test, SceKernelPageSize, SceKernelProtCpuRw, "flexible_zero_source"); + std::memset(reinterpret_cast(first), 0xa5, SceKernelPageSize); + CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(first, SceKernelPageSize), + "KernelMunmap(source)"); + + const auto reused = + MapNamedFlexible(test, SceKernelPageSize, SceKernelProtCpuRw, "flexible_zero_reuse"); + const auto* bytes = reinterpret_cast(reused); + Check(test, + std::all_of(bytes, bytes + SceKernelPageSize, [](uint8_t value) { return value == 0; }), + "reused flexible backing exposed stale bytes"); + CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(reused, SceKernelPageSize), + "KernelMunmap(reuse)"); + Check(test, AvailableFlexibleMemory(test) == baseline, + "zero-fill test leaked flexible backing capacity"); + + std::printf("[host] %-48s ok\n", test); +} + +void TestGuestStackUsesPrivateOwnerMemoryAndCache() { + const char* test = "GuestStackUsesPrivateOwnerMemoryAndCache"; + const auto baseline = AvailableFlexibleMemory(test); + uint64_t first = 0; + uint64_t second = 0; + uint64_t map_size = 0; + + Check(test, Libs::LibKernel::TestGuestStackOwnerLifecycle(&first, &second, &map_size), + "guest stack owner lifecycle failed"); + Check(test, first != 0 && first == second, "guest stack cache did not reuse its owner mapping"); + Check(test, map_size != 0 && (map_size & (SceKernelPageSize - 1u)) == 0, + "guest stack mapping is not 16 KiB aligned"); + Check(test, AvailableFlexibleMemory(test) == baseline, + "private guest stack changed flexible backing capacity"); + + std::printf("[host] %-48s ok\n", test); +} + +void TestMainEntryUsesGuestStackAndDisablesHostChecks() { + const char* test = "MainEntryUsesGuestStackAndDisablesHostChecks"; + + Check(test, Loader::TestMainEntryUsesGuestStack(), + "main-entry stack switch did not preserve the guest/host stack invariants"); + + std::printf("[host] %-48s ok\n", test); +} + +void TestFragmentedBackingUnmapRollback() { + const char* test = "FragmentedBackingUnmapRollback"; + const auto baseline = AvailableFlexibleMemory(test); + const auto left = + MapNamedFlexible(test, SceKernelPageSize, SceKernelProtCpuRw, "backing_hole_left"); + const auto blocker = + MapNamedFlexible(test, SceKernelPageSize, SceKernelProtCpuRw, "backing_blocker"); + const auto right = + MapNamedFlexible(test, SceKernelPageSize, SceKernelProtCpuRw, "backing_hole_right"); + CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(left, SceKernelPageSize), + "KernelMunmap(left hole)"); + CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(right, SceKernelPageSize), + "KernelMunmap(right hole)"); + + const auto fragmented = + MapNamedFlexible(test, SceKernelPageSize * 2, SceKernelProtCpuRw, "fragmented_backing"); + auto* first_word = reinterpret_cast(fragmented); + auto* last_word = + reinterpret_cast(fragmented + SceKernelPageSize * 2 - sizeof(uint64_t)); + *first_word = 0x465241474c454654ull; // "FRAGLEFT" + *last_word = 0x4652414752474854ull; // "FRAGRGHT" + + Libs::LibKernel::Memory::TestFailGuestBackingStoreUnmapAfter(1); + CheckFailed(test, Libs::LibKernel::Memory::KernelMunmap(fragmented, SceKernelPageSize * 2), + "KernelMunmap(injected second-view failure)"); + ExpectRange(test, Query(test, fragmented), fragmented, fragmented + SceKernelPageSize * 2, + SceKernelProtCpuRw, 1, 0, 0, 1, "fragmented_backing"); + Check(test, *first_word == 0x465241474c454654ull && *last_word == 0x4652414752474854ull, + "transactional backing-unmap rollback lost mapped contents"); + + CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(fragmented, SceKernelPageSize * 2), + "KernelMunmap(retry)"); + CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(blocker, SceKernelPageSize), + "KernelMunmap(blocker)"); + Check(test, AvailableFlexibleMemory(test) == baseline, + "fragmented backing rollback test leaked flexible capacity"); + + std::printf("[host] %-48s ok\n", test); +} + +void TestRuntimeMemoryOwnerLifecycle() { + const char* test = "RuntimeMemoryOwnerLifecycle"; + Check(test, + Libs::LibKernel::Memory::AllocateRuntimeMemory(0x10000, SceKernelPageSize, + Common::VirtualMemory::Mode::ReadWrite, + "runtime_outside_owner", true) == 0, + "fixed runtime allocation escaped the guest owner"); + + const auto base = Libs::LibKernel::Memory::AllocateRuntimeMemory( + 0, SceKernelPageSize * 2, Common::VirtualMemory::Mode::ReadWrite, "runtime_lifecycle"); + Check(test, base != 0, "runtime allocation failed"); + Check(test, Libs::LibKernel::Memory::TestGuestAddressRangeIsOwned(base, SceKernelPageSize * 2), + "runtime allocation is outside the owner"); + *reinterpret_cast(base) = 0x52554e54494d454full; // "RUNTIMEO" + Check(test, + Libs::LibKernel::Memory::ProtectGuestMemory(base, SceKernelPageSize, + Common::VirtualMemory::Mode::Read), + "runtime protection failed"); + Check(test, Libs::LibKernel::Memory::FreeGuestMemory(base, SceKernelPageSize * 2), + "runtime free failed"); + Check(test, Libs::LibKernel::Memory::TestPlaceholderRangeIsFree(base, SceKernelPageSize * 2), + "runtime free did not restore the owner placeholder"); + + const auto reused = Libs::LibKernel::Memory::AllocateRuntimeMemory( + base, SceKernelPageSize * 2, Common::VirtualMemory::Mode::ReadWrite, "runtime_reuse", true); + Check(test, reused == base, "fixed runtime allocation did not reuse the owner placeholder"); + Check(test, Libs::LibKernel::Memory::FreeGuestMemory(reused, SceKernelPageSize * 2), + "reused runtime free failed"); + + const auto adjacent_first = Libs::LibKernel::Memory::AllocateRuntimeMemory( + 0, SceKernelPageSize, Common::VirtualMemory::Mode::ReadWrite, "runtime_adjacent_first"); + Check(test, adjacent_first != 0, "first adjacent runtime allocation failed"); + const auto adjacent_second = Libs::LibKernel::Memory::AllocateRuntimeMemory( + adjacent_first + SceKernelPageSize, SceKernelPageSize, + Common::VirtualMemory::Mode::ReadWrite, "runtime_adjacent_second", true); + Check(test, adjacent_second == adjacent_first + SceKernelPageSize, + "second adjacent runtime allocation failed"); + Check(test, + Libs::LibKernel::Memory::FreeGuestMemory(adjacent_first, SceKernelPageSize * 2), + "combined adjacent runtime free failed"); + Check(test, + Libs::LibKernel::Memory::TestPlaceholderRangeIsFree(adjacent_first, + SceKernelPageSize * 2), + "combined adjacent runtime free did not restore one owner placeholder"); + + std::printf("[host] %-48s ok\n", test); +} + void TestFlexibleMapQueryAndWholeMunmap() { const char* test = "FlexibleMapQueryAndWholeMunmap"; const auto baseline = AvailableFlexibleMemory(test); @@ -341,9 +763,11 @@ void TestDirectMapQueryOffsetAndPartialMunmap() { &addr, SceKernelPageSize * 4, SceKernelProtCpuRw, 0, phys_addr, SceKernelPageSize, "prospero_direct"), "KernelMapNamedDirectMemory"); - const auto base = reinterpret_cast(addr); - const auto phys = static_cast(phys_addr); - void* alias = nullptr; + const auto base = reinterpret_cast(addr); + const auto phys = static_cast(phys_addr); + Check(test, Libs::LibKernel::Memory::TestGuestAddressRangeIsOwned(base, SceKernelPageSize * 4), + "direct mapping escaped the guest owner"); + void* alias = nullptr; CheckOk(test, Libs::LibKernel::Memory::KernelMapNamedDirectMemory( &alias, SceKernelPageSize * 4, SceKernelProtCpuRw, 0, phys_addr, SceKernelPageSize, @@ -430,6 +854,192 @@ void TestDirectMapQueryOffsetAndPartialMunmap() { std::printf("[host] %-48s ok\n", test); } +void TestDirectPartialProtectUnmapPreservesNeighbors() { + const char* test = "DirectPartialProtectUnmapPreservesNeighbors"; + const auto size = SceKernelPageSize * 3; + int64_t phys_addr = 0; + CheckOk(test, + Libs::LibKernel::Memory::KernelAllocateDirectMemory( + 0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(), size, SceKernelPageSize, + SceKernelMtypeC, &phys_addr), + "KernelAllocateDirectMemory"); + + void* address = nullptr; + CheckOk(test, + Libs::LibKernel::Memory::KernelMapNamedDirectMemory(&address, size, SceKernelProtCpuRw, + 0, phys_addr, SceKernelPageSize, + "partial_protect_direct"), + "KernelMapNamedDirectMemory"); + const auto base = reinterpret_cast(address); + CheckOk( + test, + Libs::LibKernel::Memory::KernelMprotect(reinterpret_cast(base + SceKernelPageSize), + SceKernelPageSize, SceKernelProtCpuRead), + "KernelMprotect(middle)"); + Check(test, + Libs::LibKernel::Memory::ProtectGuestHostMemory( + base, size, Common::VirtualMemory::Mode::Read), + "owner could not protect fragmented backing views"); + Check(test, + Libs::LibKernel::Memory::ProtectGuestHostMemory( + base, size, Common::VirtualMemory::Mode::ReadWrite), + "owner could not restore fragmented backing views"); + CheckOk(test, + Libs::LibKernel::Memory::KernelMunmap(base + SceKernelPageSize, SceKernelPageSize), + "KernelMunmap(middle)"); + + Common::VirtualMemory::Mode old_left {}; + Common::VirtualMemory::Mode old_right {}; + Check(test, + Common::VirtualMemory::Protect(base, SceKernelPageSize, + Common::VirtualMemory::Mode::ReadWrite, &old_left), + "could not inspect left-page protection"); + Check(test, + Common::VirtualMemory::Protect(base + SceKernelPageSize * 2, SceKernelPageSize, + Common::VirtualMemory::Mode::ReadWrite, &old_right), + "could not inspect right-page protection"); + Check(test, old_left == Common::VirtualMemory::Mode::ReadWrite, + "partial unmap changed the left neighbor protection"); + Check(test, old_right == Common::VirtualMemory::Mode::ReadWrite, + "partial unmap changed the right neighbor protection"); + *reinterpret_cast(base) = 0x4c45465450524f54ull; // "LEFTPROT" + *reinterpret_cast(base + SceKernelPageSize * 2) = + 0x5247485450524f54ull; // "RGHTPROT" + + CheckOk(test, Libs::LibKernel::Memory::KernelReleaseDirectMemory(phys_addr, size), + "KernelReleaseDirectMemory"); + ExpectUnmapped(test, base); + ExpectUnmapped(test, base + SceKernelPageSize * 2); + + std::printf("[host] %-48s ok\n", test); +} + +void TestDirectMapValidationBeforeOwnerMutation() { + const char* test = "DirectMapValidationBeforeOwnerMutation"; + int64_t invalid = -1; + CheckFailed(test, + Libs::LibKernel::Memory::KernelAllocateDirectMemory( + 0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(), SceKernelPageSize + 1, + SceKernelPageSize, SceKernelMtypeC, &invalid), + "KernelAllocateDirectMemory(unaligned size)"); + Check(test, invalid == -1, "invalid direct allocation changed the output address"); + CheckFailed(test, + Libs::LibKernel::Memory::KernelAllocateDirectMemory( + 0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(), SceKernelPageSize, + 0x1000, SceKernelMtypeC, &invalid), + "KernelAllocateDirectMemory(sub-page alignment)"); + Check(test, invalid == -1, "invalid alignment changed the output address"); + + int64_t phys_addr = 0; + CheckOk(test, + Libs::LibKernel::Memory::KernelAllocateDirectMemory( + 0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(), SceKernelPageSize * 2, + SceKernelPageSize, SceKernelMtypeC, &phys_addr), + "KernelAllocateDirectMemory"); + + auto expect_invalid = [&](size_t len, int prot, int flags, int64_t phys, size_t alignment, + const char* action) { + void* address = nullptr; + CheckFailed(test, + Libs::LibKernel::Memory::KernelMapDirectMemory(&address, len, prot, flags, phys, + alignment), + action); + Check(test, address == nullptr, "invalid direct map changed the output address"); + }; + expect_invalid(SceKernelPageSize + 1, SceKernelProtCpuRw, 0, phys_addr, SceKernelPageSize, + "KernelMapDirectMemory(unaligned size)"); + expect_invalid(SceKernelPageSize, SceKernelProtCpuRw, 0, phys_addr + 1, SceKernelPageSize, + "KernelMapDirectMemory(unaligned physical address)"); + expect_invalid(SceKernelPageSize, SceKernelProtCpuExec, 0, phys_addr, SceKernelPageSize, + "KernelMapDirectMemory(executable)"); + + void* aligned = nullptr; + CheckOk(test, + Libs::LibKernel::Memory::KernelMapDirectMemory( + &aligned, SceKernelPageSize, SceKernelProtCpuRw, 0, phys_addr, 0xc000), + "KernelMapDirectMemory(16K-multiple alignment)"); + Check(test, reinterpret_cast(aligned) % 0xc000 == 0, + "non-power-of-two 16K alignment was not honored"); + CheckOk(test, + Libs::LibKernel::Memory::KernelMunmap(reinterpret_cast(aligned), + SceKernelPageSize), + "KernelMunmap(16K-multiple alignment)"); + + void* ignored_flag = nullptr; + CheckOk(test, + Libs::LibKernel::Memory::KernelMapDirectMemory(&ignored_flag, SceKernelPageSize, + SceKernelProtCpuRw, 0x08, phys_addr, + SceKernelPageSize), + "KernelMapDirectMemory(ignored flag)"); + CheckOk(test, + Libs::LibKernel::Memory::KernelMunmap(reinterpret_cast(ignored_flag), + SceKernelPageSize), + "KernelMunmap(ignored flag)"); + + CheckOk(test, + Libs::LibKernel::Memory::KernelReleaseDirectMemory(phys_addr, SceKernelPageSize * 2), + "KernelReleaseDirectMemory"); + std::printf("[host] %-48s ok\n", test); +} + +void TestDirectReleaseRollbackRestoresOwnerMapping() { + const char* test = "DirectReleaseRollbackRestoresOwnerMapping"; + int64_t phys_addr = 0; + CheckOk(test, + Libs::LibKernel::Memory::KernelAllocateDirectMemory( + 0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(), SceKernelPageSize, + SceKernelPageSize, SceKernelMtypeC, &phys_addr), + "KernelAllocateDirectMemory"); + void* address = nullptr; + CheckOk(test, + Libs::LibKernel::Memory::KernelMapNamedDirectMemory( + &address, SceKernelPageSize, SceKernelProtCpuRw, 0, phys_addr, SceKernelPageSize, + "release_rollback"), + "KernelMapNamedDirectMemory"); + const auto base = reinterpret_cast(address); + *reinterpret_cast(base) = 0x52454c524f4c4c42ull; // "RELROLLB" + + Libs::LibKernel::Memory::TestFailNextPhysicalMemoryUnmap(); + CheckFailed( + test, + Libs::LibKernel::Memory::KernelCheckedReleaseDirectMemory(phys_addr, SceKernelPageSize), + "KernelCheckedReleaseDirectMemory(injected failure)"); + ExpectRange(test, Query(test, base), base, base + SceKernelPageSize, SceKernelProtCpuRw, 0, 1, + 0, 1, "release_rollback", static_cast(phys_addr)); + Check(test, *reinterpret_cast(base) == 0x52454c524f4c4c42ull, + "release rollback lost the shared-backing contents"); + + CheckOk(test, + Libs::LibKernel::Memory::KernelCheckedReleaseDirectMemory(phys_addr, SceKernelPageSize), + "KernelCheckedReleaseDirectMemory(retry)"); + ExpectUnmapped(test, base); + std::printf("[host] %-48s ok\n", test); +} + +void TestDirectReleaseContracts() { + const char* test = "DirectReleaseContracts"; + CheckOk(test, Libs::LibKernel::Memory::KernelReleaseDirectMemory(0, 0), + "KernelReleaseDirectMemory(zero length)"); + CheckOk(test, Libs::LibKernel::Memory::KernelCheckedReleaseDirectMemory(0, 0), + "KernelCheckedReleaseDirectMemory(zero length)"); + CheckFailed(test, Libs::LibKernel::Memory::KernelReleaseDirectMemory(1, SceKernelPageSize), + "KernelReleaseDirectMemory(unaligned start)"); + CheckFailed(test, Libs::LibKernel::Memory::KernelReleaseDirectMemory(0, SceKernelPageSize + 1), + "KernelReleaseDirectMemory(unaligned size)"); + + const auto free_offset = static_cast( + Libs::LibKernel::Memory::KernelGetDirectMemorySize() - SceKernelPageSize); + CheckOk(test, + Libs::LibKernel::Memory::KernelReleaseDirectMemory(free_offset, SceKernelPageSize), + "KernelReleaseDirectMemory(unallocated range)"); + Check(test, + Libs::LibKernel::Memory::KernelCheckedReleaseDirectMemory( + free_offset, SceKernelPageSize) == Libs::LibKernel::KERNEL_ERROR_ENOENT, + "checked release did not report an unallocated range"); + + std::printf("[host] %-48s ok\n", test); +} + void TestReleasedReserveCanBeReused() { const char* test = "ReleasedReserveCanBeReused"; void* addr = nullptr; @@ -478,21 +1088,23 @@ void TestMunmapAcrossAdjacentFlexibleMappings() { "KernelMapNamedFlexibleMemory(right)"); Check(test, - Libs::LibKernel::Memory::ClampRangeSize(base + SceKernelPageSize - 0x100, 0x200) == - 0x200, + Libs::LibKernel::Memory::ClampRangeSize(base + SceKernelPageSize - 0x100, 0x200) == 0x200, "ClampRangeSize did not cross adjacent committed mappings"); + Check(test, + Libs::LibKernel::Memory::ProtectGuestHostMemory( + base, SceKernelPageSize * 2, Common::VirtualMemory::Mode::Read), + "owner could not protect adjacent backing mappings"); + Check(test, + Libs::LibKernel::Memory::ProtectGuestHostMemory( + base, SceKernelPageSize * 2, Common::VirtualMemory::Mode::ReadWrite), + "owner could not restore adjacent backing mappings"); CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(base, SceKernelPageSize * 2), "KernelMunmap(adjacent mappings)"); Check(test, AvailableFlexibleMemory(test) == baseline, "multi-range unmap leaked flexible-memory budget"); - ExpectRange(test, Query(test, base), base, base + SceKernelPageSize, 0, 0, 0, 0, 0, - "adjacent_left"); - ExpectRange(test, Query(test, base + SceKernelPageSize), base + SceKernelPageSize, - base + SceKernelPageSize * 2, 0, 0, 0, 0, 0, "adjacent_right"); - - CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(base, SceKernelPageSize * 2), - "KernelMunmap(restored reserve)"); + ExpectUnmapped(test, base); + ExpectUnmapped(test, base + SceKernelPageSize); std::printf("[host] %-48s ok\n", test); } @@ -548,6 +1160,52 @@ void TestNonzeroDirectOffsetAliasesSharedBacking() { std::printf("[host] %-48s ok\n", test); } +void TestDirectMapAcrossContiguousAllocations() { + const char* test = "DirectMapAcrossContiguousAllocations"; + const auto end = Libs::LibKernel::Memory::KernelGetDirectMemorySize(); + int64_t first = 0; + int64_t second = 0; + CheckOk(test, + Libs::LibKernel::Memory::KernelAllocateDirectMemory( + 0, end, SceKernelPageSize, SceKernelPageSize, SceKernelMtypeC, &first), + "KernelAllocateDirectMemory(first)"); + CheckOk(test, + Libs::LibKernel::Memory::KernelAllocateDirectMemory( + 0, end, SceKernelPageSize, SceKernelPageSize, SceKernelMtypeC, &second), + "KernelAllocateDirectMemory(second)"); + Check(test, second == first + static_cast(SceKernelPageSize), + "test allocations are not physically contiguous"); + + void* mapping = nullptr; + CheckOk(test, + Libs::LibKernel::Memory::KernelMapNamedDirectMemory( + &mapping, SceKernelPageSize * 2, SceKernelProtCpuRw, 0, first, SceKernelPageSize, + "contiguous_allocations"), + "KernelMapNamedDirectMemory"); + auto* words = reinterpret_cast(mapping); + words[0] = 0x434f4e5449474c46ull; // "CONTIGLF" + *reinterpret_cast(reinterpret_cast(mapping) + SceKernelPageSize) = + 0x434f4e5449475254ull; // "CONTIGRT" + + CheckOk(test, + Libs::LibKernel::Memory::KernelCheckedReleaseDirectMemory(first, SceKernelPageSize * 2), + "KernelCheckedReleaseDirectMemory(contiguous span)"); + ExpectUnmapped(test, reinterpret_cast(mapping)); + + int64_t reclaimed = -1; + CheckOk(test, + Libs::LibKernel::Memory::KernelAllocateDirectMemory( + 0, end, SceKernelPageSize * 2, SceKernelPageSize, SceKernelMtypeC, &reclaimed), + "KernelAllocateDirectMemory(reclaimed)"); + Check(test, reclaimed == first, "released contiguous span was not coalesced"); + CheckOk( + test, + Libs::LibKernel::Memory::KernelCheckedReleaseDirectMemory(reclaimed, SceKernelPageSize * 2), + "KernelCheckedReleaseDirectMemory(reclaimed)"); + + std::printf("[host] %-48s ok\n", test); +} + void TestDirectPhysicalFreeRangeReuseAndCoalescing() { const char* test = "DirectPhysicalFreeRangeReuseAndCoalescing"; const auto end = Libs::LibKernel::Memory::KernelGetDirectMemorySize(); @@ -933,38 +1591,6 @@ void TestFixedReserveRollbackConsumesRestoredPlaceholder() { std::printf("[host] %-48s ok\n", test); } -void TestFixedReserveRollbackRestoresDecommittedHostPages() { - const char* test = "FixedReserveRollbackRestoresDecommittedHostPages"; - constexpr uint64_t size = SceKernelPageSize * 3; - void* mapped = nullptr; - - CheckOk(test, - Libs::LibKernel::Memory::KernelMapNamedFlexibleMemory(&mapped, size, SceKernelProtCpuRw, - 0, "host_reserve_rollback"), - "KernelMapNamedFlexibleMemory"); - const auto base = reinterpret_cast(mapped); - *reinterpret_cast(base) = 0x4b595459484f5354ull; // "KYTYHOST" - *reinterpret_cast(base + SceKernelPageSize * 2) = - 0x4b5954595441494cull; // "KYTYTAIL" - - Libs::LibKernel::Memory::TestFailHostReservationAfter(1); - void* replacement = mapped; - CheckFailed( - test, - Libs::LibKernel::Memory::KernelReserveVirtualRange( - &replacement, size, SceKernelMapFixed | SceKernelMapNoCoalesce, SceKernelPageSize), - "KernelReserveVirtualRange(partial host reservation)"); - Check(test, *reinterpret_cast(base) == 0x4b595459484f5354ull, - "rollback did not restore the first flexible page"); - Check(test, *reinterpret_cast(base + SceKernelPageSize * 2) == 0x4b5954595441494cull, - "rollback damaged the flexible tail page"); - ExpectRange(test, Query(test, base), base, base + size, SceKernelProtCpuRw, 1, 0, 0, 1, - "host_reserve_rollback"); - - CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(base, size), "KernelMunmap"); - std::printf("[host] %-48s ok\n", test); -} - void TestFixedReserveRangeAddRollbackKeepsPlaceholder() { const char* test = "FixedReserveRangeAddRollbackKeepsPlaceholder"; constexpr uint64_t size = SceKernelPageSize * 4; @@ -1055,8 +1681,10 @@ void TestLargeHintedReserveHostsSmallDirectMap() { CheckOk(test, Libs::LibKernel::Memory::KernelReleaseDirectMemory(phys, SceKernelPageSize * 2), "KernelReleaseDirectMemory"); CheckOk(test, - Libs::LibKernel::Memory::KernelMunmap(reinterpret_cast(window), window_size), - "KernelMunmap(window reserve)"); + Libs::LibKernel::Memory::KernelMunmap(reinterpret_cast(window) + + SceKernelPageSize * 2, + window_size - SceKernelPageSize * 2), + "KernelMunmap(window reserve remainder)"); CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(reinterpret_cast(arena), arena_size), "KernelMunmap(arena reserve)"); @@ -1153,18 +1781,25 @@ void TestProsperoSampleMemoryPoolExpandCommit() { SceKernelProtCpuRw, 0), "KernelMemoryPoolCommit"); ExpectRange(test, Query(test, base), base, base + commit_len, SceKernelProtCpuRw, 0, 0, 1, 1); + Check(test, Libs::LibKernel::Memory::TestGuestAddressRangeIsOwned(base, commit_len), + "pooled commit escaped the guest owner"); Check(test, AvailableFlexibleMemory(test) == flexible_baseline, "pooled commit consumed flexible memory instead of expanded direct " "backing"); CheckFailed(test, - Libs::LibKernel::Memory::KernelReleaseDirectMemory(pool_offset, - SceKernelMemoryPoolExpandLen), - "KernelReleaseDirectMemory(committed pool expansion)"); + Libs::LibKernel::Memory::KernelCheckedReleaseDirectMemory( + pool_offset, SceKernelMemoryPoolExpandLen), + "KernelCheckedReleaseDirectMemory(committed pool expansion)"); constexpr uint64_t first_value = 0x504f4f4c4241434bull; // "POOLBACK" constexpr uint64_t second_value = 0x5348415245444d45ull; // "SHAREDME" *reinterpret_cast(base) = first_value; *reinterpret_cast(base + SceKernelMemoryPoolCommitLen) = second_value; + uint64_t backing_read = 0; + Check(test, Libs::LibKernel::Memory::TryReadBacking(base, &backing_read, sizeof(backing_read)), + "TryReadBacking did not resolve pooled memory"); + Check(test, backing_read == first_value, + "shared backing did not observe a pooled-memory CPU write"); CheckOk( test, @@ -1276,8 +1911,10 @@ void TestFragmentedMemoryPoolBacking() { "KernelMemoryPoolCommit(fragmented recommit)"); CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(base, commit_len), "KernelMunmap(fragmented commit)"); - CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(base, SceKernelMemoryPoolReserveLen), - "KernelMunmap(fragmented reserve cleanup)"); + CheckOk(test, + Libs::LibKernel::Memory::KernelMunmap(base + commit_len, + SceKernelMemoryPoolReserveLen - commit_len), + "KernelMunmap(fragmented reserve remainder)"); CheckOk(test, Libs::LibKernel::Memory::KernelReleaseDirectMemory(first_pool, @@ -1427,22 +2064,32 @@ void TestMemoryPoolCommitDecommitQueryFlags() { std::printf("[host] %-48s ok\n", test); } -void TestProgramMemoryRegistrationAndProtection() { - const char* test = "ProgramMemoryRegistrationAndProtection"; +void TestProgramMemoryAllocationAndProtection() { + const char* test = "ProgramMemoryAllocationAndProtection"; const auto size = SceKernelPageSize * 3; - const auto base = Common::VirtualMemory::Alloc(0, size, Common::VirtualMemory::Mode::ReadWrite); - Check(test, base != 0, "program host allocation failed"); - - Libs::LibKernel::Memory::RegisterProgramMemory( - base, size, Common::VirtualMemory::Mode::ReadWrite, "program_test"); + const auto base = Libs::LibKernel::Memory::AllocateProgramMemory( + 0x900000000, size, Common::VirtualMemory::Mode::ReadWrite, "program_test"); + Check(test, base != 0, "program guest allocation failed"); + Check(test, Libs::LibKernel::Memory::TestGuestAddressRangeIsOwned(base, size), + "program allocation escaped the guest owner"); ExpectRange(test, Query(test, base), base, base + size, SceKernelProtCpuRead | SceKernelProtCpuRw, 0, 0, 0, 1, "program_test"); - Libs::LibKernel::Memory::UpdateProgramMemoryProtection(base, SceKernelPageSize, - Common::VirtualMemory::Mode::Read); + Check(test, + Libs::LibKernel::Memory::ProtectGuestMemory(base, SceKernelPageSize, + Common::VirtualMemory::Mode::Read), + "ProtectGuestMemory(first page) failed"); ExpectRange(test, Query(test, base), base, base + SceKernelPageSize, SceKernelProtCpuRead, 0, 0, 0, 1, "program_test"); + Common::VirtualMemory::Mode previous_mode = Common::VirtualMemory::Mode::NoAccess; + Check(test, + Libs::LibKernel::Memory::ProtectGuestMemory( + base, SceKernelPageSize, Common::VirtualMemory::Mode::ReadWrite, &previous_mode), + "ProtectGuestMemory(tracked restore) failed"); + Check(test, previous_mode == Common::VirtualMemory::Mode::Read, + "semantic guest protection did not preserve its tracked old mode"); + CheckOk(test, Libs::LibKernel::Memory::KernelMprotect( reinterpret_cast(base + SceKernelPageSize - 0x10), 0x20, @@ -1451,24 +2098,44 @@ void TestProgramMemoryRegistrationAndProtection() { ExpectRange(test, Query(test, base), base, base + size, SceKernelProtCpuRead | SceKernelProtCpuRw, 0, 0, 0, 1, "program_test"); - Libs::LibKernel::Memory::UpdateProgramMemoryProtection( - base + SceKernelPageSize * 2, SceKernelPageSize, Common::VirtualMemory::Mode::Read); + Check(test, + Libs::LibKernel::Memory::ProtectGuestMemory( + base + SceKernelPageSize * 2, SceKernelPageSize, Common::VirtualMemory::Mode::Read), + "ProtectGuestMemory(last page) failed"); ExpectRange(test, Query(test, base + SceKernelPageSize * 2), base + SceKernelPageSize * 2, base + size, SceKernelProtCpuRead, 0, 0, 0, 1, "program_test"); - Libs::LibKernel::Memory::UnregisterProgramMemory(base, size); + Check(test, Libs::LibKernel::Memory::FreeGuestMemory(base, size), "program guest free failed"); ExpectUnmapped(test, base); - Check(test, Common::VirtualMemory::Free(base), "program host free failed"); std::printf("[host] %-48s ok\n", test); } +void TestModuleRelocationUsesWritableHostMapping() { + const char* test = "ModuleRelocationUsesWritableHostMapping"; + Check(test, Loader::TestModuleRelocationUsesWritableHostMapping(), + "module relocation did not retain writable host memory and semantic guest protection"); + std::printf("[host] %-48s ok\n", test); +} + } // namespace int main() { InitSubsystems(); RunTest(TestProsperoArgumentAndInfoSizeContracts); + RunTest(TestGuestAddressSpaceOwnsReservationsBeforeBacking); + RunTest(TestGuestAddressSpaceHasNoFixedFallback); + RunTest(TestGuestFreeRangeSearchDoesNotUnderflow); + RunTest(TestFlexibleMemoryCapacityIsBootFixed); + RunTest(TestFlexibleMemoryUsesSharedBacking); + RunTest(TestFlexibleDmemCompatAndAlignmentFlags); + RunTest(TestFlexibleNoCoalescePreservesBoundaries); + RunTest(TestFlexibleMemoryReuseIsZeroFilled); + RunTest(TestGuestStackUsesPrivateOwnerMemoryAndCache); + RunTest(TestMainEntryUsesGuestStackAndDisablesHostChecks); + RunTest(TestFragmentedBackingUnmapRollback); + RunTest(TestRuntimeMemoryOwnerLifecycle); RunTest(TestFlexibleMapQueryAndWholeMunmap); RunTest(TestPartialFlexibleMunmapAndFindNext); RunTest(TestReserveMapFixedAndNoOverwrite); @@ -1476,7 +2143,12 @@ int main() { RunTest(TestReleasedReserveCanBeReused); RunTest(TestMunmapAcrossAdjacentFlexibleMappings); RunTest(TestDirectMapQueryOffsetAndPartialMunmap); + RunTest(TestDirectPartialProtectUnmapPreservesNeighbors); + RunTest(TestDirectMapValidationBeforeOwnerMutation); + RunTest(TestDirectReleaseRollbackRestoresOwnerMapping); + RunTest(TestDirectReleaseContracts); RunTest(TestNonzeroDirectOffsetAliasesSharedBacking); + RunTest(TestDirectMapAcrossContiguousAllocations); RunTest(TestDirectPhysicalFreeRangeReuseAndCoalescing); RunTest(TestDirectAlignmentStaysWithinSearchRange); RunTest(TestDefaultDirectMapUsesSystemAddressRange); @@ -1485,7 +2157,6 @@ int main() { RunTest(TestFixedReserveReplacesPartialDirectMapping); RunTest(TestFixedReserveRollbackConsumesRestoredPlaceholder); RunTest(TestFixedReserveRollbackSkipsUntouchedChunks); - RunTest(TestFixedReserveRollbackRestoresDecommittedHostPages); RunTest(TestFixedReserveRangeAddRollbackKeepsPlaceholder); RunTest(TestLargeHintedReserveHostsSmallDirectMap); RunTest(TestMemoryPoolAlignmentContracts); @@ -1493,7 +2164,8 @@ int main() { RunTest(TestFragmentedMemoryPoolBacking); RunTest(TestMemoryPoolMultiRangeDecommit); RunTest(TestMemoryPoolCommitDecommitQueryFlags); - RunTest(TestProgramMemoryRegistrationAndProtection); + RunTest(TestProgramMemoryAllocationAndProtection); + RunTest(TestModuleRelocationUsesWritableHostMapping); if (g_failed_tests != 0) { std::printf("VirtualMemoryAllocationTests: %d case(s) failed\n", g_failed_tests);