H3cJP/KytyPS5
1
0
Fork 0
mirror of https://github.com/KytyPS5/KytyPS5.git synced 2026-08-03 11:23:49 +00:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Rework guest memory tracking

Browse Source
This commit is contained in:
nmzik
2026-07-31 02:14:17 +02:00
parent ecb48f90bb
commit e87ae56544
27 changed files with 3110 additions and 2887 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/build.yml
+17 -3
View File
@@ -83,7 +83,12 @@ jobs:
- name: Build
shell: cmd
run: |
cmake --build _Build/windows --target launcher --parallel
cmake --build _Build/windows --target launcher virtual_memory_allocation_tests --parallel
- name: Test
shell: cmd
run: |
ctest --test-dir _Build/windows --output-on-failure -R "^virtual_memory_allocation$"
- name: Install
shell: cmd
@@ -153,7 +158,15 @@ jobs:
- name: Build
shell: bash
run: |
cmake --build _Build/macos --target launcher --parallel
cmake --build _Build/macos \
--target launcher virtual_memory_allocation_tests \
--parallel
- name: Test
shell: bash
run: |
ctest --test-dir _Build/macos --output-on-failure \
-R '^virtual_memory_allocation$'
- name: Install
shell: bash
@@ -284,13 +297,14 @@ jobs:
run: |
cmake --build _Build/linux \
--target launcher page_manager_tests memory_tracker_tests \
virtual_memory_allocation_tests \
--parallel
- name: Test
shell: bash
run: |
ctest --test-dir _Build/linux --output-on-failure \
-R '^(page_manager|memory_tracker)$'
-R '^(page_manager|memory_tracker|virtual_memory_allocation)$'
- name: Install
shell: bash
src/CMakeLists.txt
+2 -1
View File
@@ -338,7 +338,6 @@ add_executable(memory_tracker_tests EXCLUDE_FROM_ALL
)
target_link_libraries(memory_tracker_tests fmt::fmt common)
target_include_directories(memory_tracker_tests PRIVATE ${inc_headers})
target_compile_definitions(memory_tracker_tests PRIVATE KYTY_MEMORY_TRACKER_TESTS=1)
add_executable(shader_vertex_metadata_tests EXCLUDE_FROM_ALL
../tests/ShaderVertexMetadataTests.cpp
@@ -437,6 +436,8 @@ if(BUILD_TESTING)
add_test(NAME resource_mutex COMMAND $<TARGET_FILE:resource_mutex_tests>)
add_test(NAME event_queue_lifetime COMMAND $<TARGET_FILE:event_queue_lifetime_tests>)
add_test(NAME shader_recompiler_compute COMMAND $<TARGET_FILE:shader_recompiler_compute_tests>)
add_test(NAME virtual_memory_allocation
COMMAND $<TARGET_FILE:virtual_memory_allocation_tests>)
add_test(NAME command_scheduler_timeline
COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --scheduler-only)
add_test(NAME stream_buffer_ring
src/common/virtualMemory.cpp
-19
View File
@@ -58,25 +58,6 @@ bool FlushInstructionCache(uint64_t address, uint64_t size) {
return SysVirtualFlushInstructionCache(address, size);
}
bool PatchReplace(uint64_t vaddr, uint64_t value) {
Mode old_mode {};
Protect(vaddr, 8, Mode::ReadWrite, &old_mode);
auto* ptr = reinterpret_cast<uint64_t*>(vaddr);
bool ret = (*ptr != value);
*ptr = value;
Protect(vaddr, 8, old_mode);
if (IsExecute(old_mode)) {
FlushInstructionCache(vaddr, 8);
}
return ret;
}
} // namespace VirtualMemory
} // namespace Common
src/common/virtualMemory.h
-1
View File
@@ -37,7 +37,6 @@ bool Free(uint64_t address);
bool FreeRange(uint64_t address, uint64_t size);
bool Protect(uint64_t address, uint64_t size, Mode mode, Mode* old_mode = nullptr);
bool FlushInstructionCache(uint64_t address, uint64_t size);
bool PatchReplace(uint64_t vaddr, uint64_t value);
} // namespace VirtualMemory
src/emulator.cpp
+13 -12
View File
@@ -105,7 +105,7 @@ static void ClearDebugTextureFolder() {
}
}
static void Init(const Config::ConfigOptions& cfg) {
static void Init(const Config::ConfigOptions& cfg, const std::filesystem::path& param_json) {
EXIT_IF(!Common::Thread::IsMainThread());
auto* slist = Common::SubsystemsList::Instance();
@@ -127,12 +127,21 @@ static void Init(const Config::ConfigOptions& cfg) {
slist->InitAll(true);
Config::Load(cfg);
slist->Add(log, {core, config});
slist->InitAll(true);
if (Common::File::IsFileExisting(param_json)) {
Loader::SystemContentLoadParamSfo(param_json);
if (const auto flexible_memory_size = Loader::SystemContentGetFlexibleMemorySize();
flexible_memory_size != 0) {
Libs::LibKernel::Memory::SetFlexibleMemorySize(flexible_memory_size);
}
}
slist->Add(audio, {core, log, pthread, memory});
slist->Add(controller, {core, log, config});
slist->Add(file_system, {core, log, pthread});
slist->Add(graphics, {core, log, pthread, memory, config, profiler, controller});
slist->Add(log, {core, config});
slist->Add(memory, {core, log});
slist->Add(network, {core, log, pthread});
slist->Add(profiler, {core, config});
@@ -180,7 +189,8 @@ void Run(const RunOptions& options) {
EXIT("ELF is required\n");
}
Init(options.config);
const auto param_json = options.app0_dir / "sce_sys" / "param.json";
Init(options.config, param_json);
ClearDebugTextureFolder();
@@ -192,15 +202,6 @@ void Run(const RunOptions& options) {
Libs::LibKernel::FileSystem::Mount(options.app0_dir, "/app0");
Libs::LibKernel::FileSystem::Mount(options.app0_dir, "/hostapp");
auto param_json = options.app0_dir / "sce_sys" / "param.json";
if (Common::File::IsFileExisting(param_json)) {
Loader::SystemContentLoadParamSfo(param_json);
if (auto flexible_memory_size = Loader::SystemContentGetFlexibleMemorySize();
flexible_memory_size != 0) {
Libs::LibKernel::Memory::SetFlexibleMemorySize(flexible_memory_size);
}
}
MountSandboxDirs();
auto* rt = Common::Singleton<Loader::RuntimeLinker>::Instance();
src/graphics/host_gpu/memoryTracker.cpp
-33
View File
@@ -4,16 +4,6 @@
namespace Libs::Graphics {
#if defined(KYTY_MEMORY_TRACKER_TESTS)
namespace {
std::atomic<MemoryTracker::UnmapContentionHook> g_unmap_contention_hook {nullptr};
}
void MemoryTracker::SetUnmapContentionHook(UnmapContentionHook hook) noexcept {
g_unmap_contention_hook.store(hook, std::memory_order_release);
}
#endif
static_assert(std::atomic<void*>::is_always_lock_free);
MemoryTracker::MemoryTracker(PageManager& page_manager, PageWatchMode gpu_watch_mode)
@@ -94,7 +84,6 @@ RegionManager* MemoryTracker::GetOrCreateRegion(uint64_t index) {
bool MemoryTracker::IsRegionCpuModified(uint64_t vaddr, uint64_t size) {
CheckNotInUploadCallback();
std::lock_guard access(m_access_mutex);
RequireMapped(vaddr, size);
return Iterate<true>(vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) {
std::scoped_lock lock(manager->lock);
return manager->IsModified<DirtySource::Cpu>(offset, bytes);
@@ -104,7 +93,6 @@ bool MemoryTracker::IsRegionCpuModified(uint64_t vaddr, uint64_t size) {
bool MemoryTracker::IsRegionGpuModified(uint64_t vaddr, uint64_t size) {
CheckNotInUploadCallback();
std::lock_guard access(m_access_mutex);
RequireMapped(vaddr, size);
return Iterate<false>(vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) {
std::scoped_lock lock(manager->lock);
return manager->IsModified<DirtySource::Gpu>(offset, bytes);
@@ -114,7 +102,6 @@ bool MemoryTracker::IsRegionGpuModified(uint64_t vaddr, uint64_t size) {
void MemoryTracker::MarkRegionAsCpuModified(uint64_t vaddr, uint64_t size) {
CheckNotInUploadCallback();
std::lock_guard access(m_access_mutex);
RequireMapped(vaddr, size);
Iterate<true>(vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) {
std::scoped_lock lock(manager->lock);
const auto changed =
@@ -126,7 +113,6 @@ void MemoryTracker::MarkRegionAsCpuModified(uint64_t vaddr, uint64_t size) {
void MemoryTracker::MarkRegionAsGpuModified(uint64_t vaddr, uint64_t size) {
CheckNotInUploadCallback();
std::lock_guard access(m_access_mutex);
RequireMapped(vaddr, size);
Iterate<true>(vaddr, size, [this](RegionManager* manager, uint64_t offset, uint64_t bytes) {
std::scoped_lock lock(manager->lock);
const auto changed =
@@ -138,7 +124,6 @@ void MemoryTracker::MarkRegionAsGpuModified(uint64_t vaddr, uint64_t size) {
void MemoryTracker::UnmarkRegionAsGpuModified(uint64_t vaddr, uint64_t size) {
CheckNotInUploadCallback();
std::lock_guard access(m_access_mutex);
RequireMapped(vaddr, size);
Iterate<true>(vaddr, size, [this](RegionManager* manager, uint64_t offset, uint64_t bytes) {
std::scoped_lock lock(manager->lock);
if (!manager->IsFullyModified<DirtySource::Gpu>(offset, bytes)) {
@@ -151,8 +136,6 @@ void MemoryTracker::UnmarkRegionAsGpuModified(uint64_t vaddr, uint64_t size) {
}
void MemoryTracker::UntrackMemoryLocked(uint64_t vaddr, uint64_t size) {
RequireMapped(vaddr, size);
std::vector<RegionManager*> managers;
managers.reserve((vaddr % TRACKER_REGION_SIZE + size + TRACKER_REGION_SIZE - 1) /
TRACKER_REGION_SIZE);
@@ -185,22 +168,6 @@ void MemoryTracker::UntrackMemory(uint64_t vaddr, uint64_t size) {
UntrackMemoryLocked(vaddr, size);
}
void MemoryTracker::UnmapMemory(uint64_t vaddr, uint64_t size) {
CheckNotInUploadCallback();
std::unique_lock access(m_access_mutex, std::try_to_lock);
if (!access.owns_lock()) {
#if defined(KYTY_MEMORY_TRACKER_TESTS)
if (const auto hook = g_unmap_contention_hook.load(std::memory_order_acquire);
hook != nullptr) {
hook();
}
#endif
access.lock();
}
UntrackMemoryLocked(vaddr, size);
m_page_manager.OnGpuUnmap(vaddr, size);
}
bool MemoryTracker::InvalidateRegion(uint64_t vaddr, uint64_t size, PageFaultPhase phase) noexcept {
switch (phase) {
case PageFaultPhase::Release: return true;
src/graphics/host_gpu/memoryTracker.h
+3 -19
View File
@@ -30,7 +30,6 @@ public:
void MarkRegionAsGpuModified(uint64_t vaddr, uint64_t size);
void UnmarkRegionAsGpuModified(uint64_t vaddr, uint64_t size);
void UntrackMemory(uint64_t vaddr, uint64_t size);
void UnmapMemory(uint64_t vaddr, uint64_t size);
[[nodiscard]] CpuFaultAction
BeginCpuFault(uint64_t vaddr, uint64_t size,
PageFaultAccess access = PageFaultAccess::Write) noexcept;
@@ -91,8 +90,7 @@ public:
static_assert(std::is_nothrow_invocable_v<Preflight&, uint64_t, uint64_t>);
static_assert(std::is_nothrow_invocable_v<Func&, uint64_t, uint64_t>);
CheckNotInUploadCallback();
std::lock_guard access(m_access_mutex);
RequireMapped(vaddr, size);
std::lock_guard access(m_access_mutex);
std::vector<RegionManager*> managers;
Iterate<false>(vaddr, size, [&](RegionManager* manager, uint64_t, uint64_t) {
managers.push_back(manager);
@@ -132,11 +130,6 @@ public:
vaddr, size, [](uint64_t, uint64_t) noexcept {}, std::forward<Func>(func));
}
#if defined(KYTY_MEMORY_TRACKER_TESTS)
using UnmapContentionHook = void (*)() noexcept;
static void SetUnmapContentionHook(UnmapContentionHook hook) noexcept;
#endif
template <typename RangeFunc, typename UploadFunc>
void ForEachUploadRange(uint64_t vaddr, uint64_t size, bool is_written, RangeFunc&& range_func,
UploadFunc&& upload_func) {
@@ -144,7 +137,6 @@ public:
static_assert(std::is_nothrow_invocable_v<UploadFunc&>);
CheckNotInUploadCallback();
std::unique_lock access(m_access_mutex);
RequireMapped(vaddr, size);
Iterate<true>(vaddr, size, [](RegionManager*, uint64_t, uint64_t) {});
const auto* previous_upload_owner = std::exchange(s_upload_owner, this);
Iterate<false>(vaddr, size, [&](RegionManager* manager, uint64_t offset, uint64_t bytes) {
@@ -209,16 +201,8 @@ private:
return false;
}
static void ValidateRange(uint64_t vaddr, uint64_t size);
void UntrackMemoryLocked(uint64_t vaddr, uint64_t size);
void RequireMapped(uint64_t vaddr, uint64_t size) const {
ValidateRange(vaddr, size);
if (!m_page_manager.IsMapped(vaddr, size)) {
EXIT("memory tracker range [0x%llx, 0x%llx) is not mapped\n",
static_cast<unsigned long long>(vaddr),
static_cast<unsigned long long>(vaddr + size));
}
}
static void ValidateRange(uint64_t vaddr, uint64_t size);
void UntrackMemoryLocked(uint64_t vaddr, uint64_t size);
RegionManager* GetOrCreateRegion(uint64_t index);
std::unique_ptr<std::atomic<RegionManager*>[]> m_regions;
src/graphics/host_gpu/pageManager.cpp
+43 -464
View File
@@ -1,6 +1,7 @@
#include "graphics/host_gpu/pageManager.h"
#include "graphics/host_gpu/regionDefinitions.h"
#include "kernel/memory.h"
#include <algorithm>
#include <array>
@@ -21,16 +22,11 @@
#undef min
#undef max
#elif defined(__APPLE__)
#include <mach/mach.h>
#include <mach/mach_vm.h>
#include <pthread.h>
#include <sys/mman.h>
#include <unistd.h>
#else
#include <cerrno>
#include <cstring>
#include <execinfo.h>
#include <fcntl.h>
#include <sys/mman.h>
#include <sys/syscall.h>
#include <unistd.h>
@@ -57,45 +53,8 @@ constexpr uint64_t REGION_PAGES = REGION_SIZE / PAGE_SIZE;
constexpr uint32_t NO_ACCESS_PROTECTION = PAGE_NOACCESS;
constexpr uint32_t READ_ONLY_PROTECTION = PAGE_READONLY;
constexpr uint32_t READ_WRITE_PROTECTION = PAGE_READWRITE;
#if defined(__APPLE__)
// Map the tracker's Win32-style protection tags to POSIX mprotect flags.
static int PageProtToPosix(uint32_t protection) {
switch (protection) {
case PAGE_NOACCESS: return PROT_NONE;
case PAGE_READONLY: return PROT_READ;
case PAGE_READWRITE: return PROT_READ | PROT_WRITE;
default: return PROT_NONE;
}
}
// Query the current protection of the page containing vaddr via the Mach VM map and
// collapse it to the tracker's read/write tags (execute is irrelevant to write tracking).
static uint32_t MachQueryPageProt(uint64_t vaddr) {
auto region_addr = static_cast<mach_vm_address_t>(vaddr);
mach_vm_size_t region_size = 0;
vm_region_basic_info_data_64_t info {};
mach_msg_type_number_t count = VM_REGION_BASIC_INFO_COUNT_64;
mach_port_t object_name = MACH_PORT_NULL;
kern_return_t kr =
mach_vm_region(mach_task_self(), &region_addr, &region_size, VM_REGION_BASIC_INFO_64,
reinterpret_cast<vm_region_info_t>(&info), &count, &object_name);
if (kr != KERN_SUCCESS || region_addr > vaddr) {
return PAGE_NOACCESS; // no region covering vaddr
}
if ((info.protection & VM_PROT_WRITE) != 0) {
return PAGE_READWRITE;
}
if ((info.protection & VM_PROT_READ) != 0) {
return PAGE_READONLY;
}
return PAGE_NOACCESS;
}
#elif defined(__linux__)
// Zero is the unknown protection sentinel.
constexpr uint32_t UNKNOWN_PROTECTION = 0;
#endif
thread_local bool g_in_fault_resolution = false;
@@ -136,6 +95,15 @@ thread_local bool g_in_fault_resolution = false;
std::_Exit(322);
}
Common::VirtualMemory::Mode ToMemoryMode(uint32_t protection) {
switch (protection) {
case NO_ACCESS_PROTECTION: return Common::VirtualMemory::Mode::NoAccess;
case READ_ONLY_PROTECTION: return Common::VirtualMemory::Mode::Read;
case READ_WRITE_PROTECTION: return Common::VirtualMemory::Mode::ReadWrite;
default: Fatal("unmappable protection 0x%08" PRIx32, protection);
}
}
uint32_t CurrentThread() noexcept {
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
return GetCurrentThreadId();
@@ -155,130 +123,6 @@ uint32_t CurrentThread() noexcept {
#endif
}
#if defined(__linux__)
int ToHostProtection(uint32_t protection) {
switch (protection) {
case NO_ACCESS_PROTECTION: return PROT_NONE;
case READ_ONLY_PROTECTION: return PROT_READ;
case READ_WRITE_PROTECTION: return PROT_READ | PROT_WRITE;
default: Fatal("unmappable protection 0x%08" PRIx32, protection);
}
}
struct HostMapping {
uint64_t end = 0;
uint32_t protection = UNKNOWN_PROTECTION;
};
// Async-signal-safe lookup in the address-ordered /proc/self/maps.
HostMapping QueryHostMapping(uint64_t vaddr) noexcept {
int fd = ::open("/proc/self/maps", O_RDONLY | O_CLOEXEC); // NOLINT
if (fd < 0) {
return {};
}
enum class Field { Start, End, Perms, Rest };
HostMapping result {};
auto field = Field::Start;
uint64_t start = 0;
uint64_t end = 0;
char perms[4] = {};
uint32_t perms_len = 0;
bool line_valid = true;
char buffer[8192];
for (bool done = false; !done;) {
const auto got = ::read(fd, buffer, sizeof(buffer));
if (got < 0) {
if (errno == EINTR) {
continue;
}
break;
}
if (got == 0) {
break;
}
for (ssize_t i = 0; i < got && !done; i++) {
const char c = buffer[i];
if (c == '\n') {
field = Field::Start;
start = 0;
end = 0;
perms_len = 0;
line_valid = true;
continue;
}
if (!line_valid) {
continue;
}
switch (field) {
case Field::Start:
case Field::End: {
uint64_t digit = 0;
if (c >= '0' && c <= '9') {
digit = static_cast<uint64_t>(c - '0');
} else if (c >= 'a' && c <= 'f') {
digit = static_cast<uint64_t>(c - 'a') + 10;
} else if (c == '-' && field == Field::Start) {
field = Field::End;
break;
} else if (c == ' ' && field == Field::End) {
field = Field::Perms;
perms_len = 0;
break;
} else {
line_valid = false;
break;
}
auto& value = (field == Field::Start ? start : end);
value = (value << 4u) | digit;
break;
}
case Field::Perms: {
if (c != ' ') {
if (perms_len < sizeof(perms)) {
perms[perms_len] = c;
}
perms_len++;
break;
}
if (vaddr < start) {
done = true;
} else if (vaddr < end && perms_len >= 2) {
result.end = end;
result.protection = perms[1] == 'w' ? READ_WRITE_PROTECTION
: perms[0] == 'r' ? READ_ONLY_PROTECTION
: NO_ACCESS_PROTECTION;
done = true;
} else {
field = Field::Rest;
}
break;
}
case Field::Rest: break;
}
}
}
::close(fd);
return result;
}
uint32_t QueryHostProtection(uint64_t vaddr) noexcept {
return QueryHostMapping(vaddr).protection;
}
#endif
class SpinGuard final {
public:
explicit SpinGuard(std::atomic_flag& lock): m_lock(lock) {
@@ -313,21 +157,16 @@ uint64_t PageEnd(uint64_t vaddr, uint64_t size) {
struct PageManager::Impl {
struct PageState {
std::atomic_flag lock = ATOMIC_FLAG_INIT;
uint32_t mappings = 0;
uint32_t gpu_read_mappings = 0;
uint32_t gpu_write_mappings = 0;
uint32_t write_watchers = 0;
uint32_t access_watchers = 0;
uint32_t original_protection = 0;
uint32_t backing_writer = 0;
#if defined(__linux__)
// Shadow the protection applied through Protect().
uint32_t current_protection = UNKNOWN_PROTECTION;
#endif
bool resolving = false;
bool resolving_read_write = false;
bool late_read_pending = false;
bool late_write_pending = false;
uint32_t current_protection = UNKNOWN_PROTECTION;
bool resolving = false;
bool resolving_read_write = false;
bool late_read_pending = false;
bool late_write_pending = false;
};
struct Region {
@@ -356,7 +195,7 @@ struct PageManager::Impl {
Impl(PageFaultHandler handler, void* context): fault_handler(handler), fault_context(context) {
if (fault_handler == nullptr) {
Fatal("null fault handler");
Fatal("null page-manager fault callback");
}
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
SYSTEM_INFO info {};
@@ -386,9 +225,8 @@ struct PageManager::Impl {
for (const auto& region: region_storage) {
for (auto& page: region->pages) {
SpinGuard lock(page.lock);
if (page.mappings != 0 || page.gpu_read_mappings != 0 ||
page.gpu_write_mappings != 0 || page.write_watchers != 0 ||
page.access_watchers != 0 || page.backing_writer != 0 || page.resolving) {
if (page.write_watchers != 0 || page.access_watchers != 0 ||
page.backing_writer != 0 || page.resolving) {
FailFast("PageManager destroyed with live page state");
}
}
@@ -441,179 +279,30 @@ struct PageManager::Impl {
}
}
static void ValidateInitialProtection(std::span<PageState*> pages, uint64_t vaddr) {
const auto end = vaddr + pages.size() * PAGE_SIZE;
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
for (auto address = vaddr; address < end;) {
MEMORY_BASIC_INFORMATION info {};
if (VirtualQuery(reinterpret_cast<const void*>(static_cast<uintptr_t>(address)), &info,
sizeof(info)) == 0 ||
info.State != MEM_COMMIT || info.Protect != PAGE_READWRITE) {
Fatal("basic path requires PAGE_READWRITE at 0x%016" PRIx64 " (state=0x%08" PRIx32
", protection=0x%08" PRIx32 ")",
address, static_cast<uint32_t>(info.State),
static_cast<uint32_t>(info.Protect));
}
const auto region_end = reinterpret_cast<uint64_t>(info.BaseAddress) + info.RegionSize;
if (region_end <= address) {
Fatal("VirtualQuery returned an invalid region at 0x%016" PRIx64, address);
}
address = std::min(end, region_end);
}
#elif defined(__APPLE__)
for (auto address = vaddr; address < end; address += PAGE_SIZE) {
const uint32_t protection = MachQueryPageProt(address);
if (protection != PAGE_READWRITE) {
Fatal("basic path requires PAGE_READWRITE at 0x%016" PRIx64
" (protection=0x%08" PRIx32 ")",
address, protection);
}
}
#else
for (auto address = vaddr; address < end;) {
const auto mapping = QueryHostMapping(address);
if (mapping.protection != READ_WRITE_PROTECTION || mapping.end <= address) {
Fatal("basic path requires a read/write mapping at 0x%016" PRIx64
" (protection=0x%08" PRIx32 ")",
address, mapping.protection);
}
address = std::min(end, mapping.end);
}
for (auto* page: pages) {
page->current_protection = READ_WRITE_PROTECTION;
}
#endif
static void InitializeProtection(std::span<PageState*> pages) {
for (auto* page: pages) {
page->original_protection = READ_WRITE_PROTECTION;
page->current_protection = READ_WRITE_PROTECTION;
}
}
static bool AllowsAccess([[maybe_unused]] const PageState& page, uint64_t vaddr,
static bool AllowsAccess(const PageState& page, [[maybe_unused]] uint64_t vaddr,
PageFaultAccess access) noexcept {
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
MEMORY_BASIC_INFORMATION info {};
if (VirtualQuery(reinterpret_cast<const void*>(static_cast<uintptr_t>(vaddr)), &info,
sizeof(info)) == 0 ||
info.State != MEM_COMMIT) {
return false;
}
switch (access) {
case PageFaultAccess::Read:
return info.Protect == PAGE_READONLY || info.Protect == PAGE_READWRITE;
case PageFaultAccess::Write: return info.Protect == PAGE_READWRITE;
return page.current_protection == READ_ONLY_PROTECTION ||
page.current_protection == READ_WRITE_PROTECTION;
case PageFaultAccess::Write: return page.current_protection == READ_WRITE_PROTECTION;
default: return false;
}
#elif defined(__APPLE__)
const uint32_t protection = MachQueryPageProt(vaddr);
switch (access) {
case PageFaultAccess::Read:
return protection == PAGE_READONLY || protection == PAGE_READWRITE;
case PageFaultAccess::Write: return protection == PAGE_READWRITE;
default: return false;
}
#else
const auto permitted = [](uint32_t protection, PageFaultAccess wanted) {
switch (wanted) {
case PageFaultAccess::Read:
return protection == READ_ONLY_PROTECTION ||
protection == READ_WRITE_PROTECTION;
case PageFaultAccess::Write: return protection == READ_WRITE_PROTECTION;
default: return false;
}
};
if (!permitted(page.current_protection, access)) {
return false;
}
return permitted(QueryHostProtection(vaddr), access);
#endif
}
static void ProtectRange(std::span<PageState*> pages, uint64_t vaddr, uint32_t protection,
std::span<const uint32_t> expected_old, bool fault_path) noexcept {
void ProtectRange(std::span<PageState*> pages, uint64_t vaddr, uint32_t protection,
std::span<const uint32_t> expected_old, bool fault_path) noexcept {
const auto size = pages.size() * PAGE_SIZE;
if (pages.size() != expected_old.size()) {
FailFast("protection range state size mismatch");
}
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
struct HostRange {
uint64_t begin = 0;
uint64_t end = 0;
};
std::vector<HostRange> host_ranges;
const auto end = vaddr + size;
for (auto address = vaddr; address < end;) {
MEMORY_BASIC_INFORMATION info {};
if (VirtualQuery(reinterpret_cast<const void*>(static_cast<uintptr_t>(address)), &info,
sizeof(info)) == 0 ||
info.State != MEM_COMMIT) {
if (fault_path) {
FailFast("VirtualProtect fault transition did not match expected protection");
}
Fatal("invalid protection transition at 0x%016" PRIx64 ", state=0x%08" PRIx32
", new=0x%08" PRIx32,
address, static_cast<uint32_t>(info.State), protection);
}
const auto region_end = reinterpret_cast<uint64_t>(info.BaseAddress) + info.RegionSize;
const auto query_end = std::min(end, region_end);
if (query_end <= address) {
if (fault_path) {
FailFast("VirtualQuery returned an invalid fault transition region");
}
Fatal("VirtualQuery returned an invalid region at 0x%016" PRIx64, address);
}
const auto first_page = static_cast<size_t>((address - vaddr) / PAGE_SIZE);
const auto last_page =
static_cast<size_t>((query_end - vaddr + PAGE_SIZE - 1) / PAGE_SIZE);
for (auto page = first_page; page < last_page; page++) {
if (info.Protect != expected_old[page]) {
if (fault_path) {
FailFast(
"VirtualProtect fault transition did not match expected protection");
}
Fatal("invalid protection transition at 0x%016" PRIx64 ", actual=0x%08" PRIx32
", expected=0x%08" PRIx32 ", new=0x%08" PRIx32,
vaddr + page * PAGE_SIZE, static_cast<uint32_t>(info.Protect),
expected_old[page], protection);
}
}
const auto allocation = reinterpret_cast<uint64_t>(info.AllocationBase);
if (host_ranges.empty() || allocation != host_ranges.back().begin) {
host_ranges.push_back({allocation, query_end});
} else {
host_ranges.back().end = query_end;
}
address = query_end;
}
for (auto range: host_ranges) {
range.begin = std::max(range.begin, vaddr);
DWORD old_protection = 0;
const auto first_page = static_cast<size_t>((range.begin - vaddr) / PAGE_SIZE);
if (VirtualProtect(reinterpret_cast<void*>(static_cast<uintptr_t>(range.begin)),
range.end - range.begin, protection, &old_protection) == 0 ||
old_protection != expected_old[first_page]) {
if (fault_path) {
FailFast("VirtualProtect fault transition did not match expected protection");
}
Fatal("invalid protection transition at 0x%016" PRIx64 ", old=0x%08" PRIx32
", expected=0x%08" PRIx32 ", new=0x%08" PRIx32,
range.begin, static_cast<uint32_t>(old_protection), expected_old[first_page],
protection);
}
}
#elif defined(__APPLE__)
// mprotect cannot report the previous protection, so the expected_old comparison
// is dropped; the tracker is the sole mutator of these pages and drives the
// transition from its own shadow state.
(void)expected_old;
if (mprotect(reinterpret_cast<void*>(static_cast<uintptr_t>(vaddr)), size,
PageProtToPosix(protection)) != 0) {
if (fault_path) {
FailFast("mprotect fault transition failed");
}
Fatal("mprotect failed at 0x%016" PRIx64 ", new=0x%08" PRIx32, vaddr, protection);
}
#else
for (size_t i = 0; i < pages.size(); i++) {
const auto actual = pages[i]->current_protection;
if (actual != UNKNOWN_PROTECTION && actual != expected_old[i]) {
@@ -625,22 +314,21 @@ struct PageManager::Impl {
vaddr + i * PAGE_SIZE, actual, expected_old[i], protection);
}
}
if (::mprotect(reinterpret_cast<void*>(static_cast<uintptr_t>(vaddr)), size,
ToHostProtection(protection)) != 0) {
if (!Libs::LibKernel::Memory::ProtectGuestHostMemory(vaddr, size,
ToMemoryMode(protection))) {
if (fault_path) {
FailFast("mprotect failed on the fault path");
FailFast("address-space fault protection transition failed");
}
Fatal("mprotect failed at 0x%016" PRIx64 ", new=0x%08" PRIx32 " (%s)", vaddr,
protection, std::strerror(errno));
Fatal("address-space protection failed at 0x%016" PRIx64 ", new=0x%08" PRIx32, vaddr,
protection);
}
for (auto* page: pages) {
page->current_protection = protection;
}
#endif
}
static void Protect(PageState& page, uint64_t vaddr, uint32_t protection, uint32_t expected_old,
bool fault_path) noexcept {
void Protect(PageState& page, uint64_t vaddr, uint32_t protection, uint32_t expected_old,
bool fault_path) noexcept {
PageState* pages[] = {&page};
uint32_t expected[] = {expected_old};
ProtectRange(pages, vaddr, protection, expected, fault_path);
@@ -680,50 +368,6 @@ bool PageManager::IsTracked(uint64_t vaddr) const noexcept {
return page.write_watchers != 0 || page.access_watchers != 0;
}
bool PageManager::IsMapped(uint64_t vaddr, uint64_t size) const noexcept {
if (vaddr == 0 || size == 0 || vaddr >= ADDRESS_SIZE || size > ADDRESS_SIZE - vaddr) {
return false;
}
const auto end = PageStart(vaddr + size - 1) + PAGE_SIZE;
for (auto page_vaddr = PageStart(vaddr); page_vaddr < end; page_vaddr += PAGE_SIZE) {
auto* region = m_impl->FindRegion(page_vaddr);
if (region == nullptr) {
return false;
}
auto& page = m_impl->GetPage(*region, page_vaddr);
SpinGuard lock(page.lock);
if (page.mappings == 0) {
return false;
}
}
return true;
}
bool PageManager::HasGpuAccess(uint64_t vaddr, uint64_t size, GpuAccess access) const noexcept {
if (access != GpuAccess::Read && access != GpuAccess::Write && access != GpuAccess::ReadWrite) {
FailFast("HasGpuAccess received an invalid GPU access mode");
}
const bool need_read = access == GpuAccess::Read || access == GpuAccess::ReadWrite;
const bool need_write = access == GpuAccess::Write || access == GpuAccess::ReadWrite;
if (vaddr == 0 || size == 0 || vaddr >= ADDRESS_SIZE || size > ADDRESS_SIZE - vaddr) {
return false;
}
const auto end = PageEnd(vaddr, size);
for (auto addr = PageStart(vaddr); addr < end; addr += PAGE_SIZE) {
auto* region = m_impl->FindRegion(addr);
if (region == nullptr) {
return false;
}
auto& page = m_impl->GetPage(*region, addr);
SpinGuard lock(page.lock);
if ((need_read && page.gpu_read_mappings == 0) ||
(need_write && page.gpu_write_mappings == 0)) {
return false;
}
}
return true;
}
void PageManager::UpdatePageWatchers(bool track, uint64_t vaddr, uint64_t size,
PageWatchMode mode) {
if (mode != PageWatchMode::Write && mode != PageWatchMode::ReadWrite) {
@@ -754,9 +398,6 @@ void PageManager::UpdatePageWatchers(bool track, uint64_t vaddr, uint64_t size,
if (page.resolving && track) {
FailFast("new page watcher raced active fault resolution");
}
if (page.mappings == 0) {
Fatal("watching unmapped page 0x%016" PRIx64, address);
}
auto& watchers =
(mode == PageWatchMode::ReadWrite ? page.access_watchers : page.write_watchers);
if (track) {
@@ -784,8 +425,7 @@ void PageManager::UpdatePageWatchers(bool track, uint64_t vaddr, uint64_t size,
last++;
}
if (first != last) {
Impl::ValidateInitialProtection(std::span {pages}.subspan(first, last - first),
chunk_begin + first * PAGE_SIZE);
Impl::InitializeProtection(std::span {pages}.subspan(first, last - first));
}
first = last;
}
@@ -831,9 +471,9 @@ void PageManager::UpdatePageWatchers(bool track, uint64_t vaddr, uint64_t size,
last = current + 1;
}
}
Impl::ProtectRange(std::span {pages}.subspan(first, last - first),
chunk_begin + first * PAGE_SIZE, protection,
std::span {old_protections}.subspan(first, last - first), false);
m_impl->ProtectRange(std::span {pages}.subspan(first, last - first),
chunk_begin + first * PAGE_SIZE, protection,
std::span {old_protections}.subspan(first, last - first), false);
first = current;
}
@@ -860,70 +500,9 @@ void PageManager::UpdatePageWatchers(bool track, uint64_t vaddr, uint64_t size,
}
}
void PageManager::OnGpuMap(uint64_t vaddr, uint64_t size, GpuAccess access) {
if (g_in_fault_resolution) {
FailFast("GPU mapping changed during fault resolution");
}
if (access != GpuAccess::Read && access != GpuAccess::Write && access != GpuAccess::ReadWrite) {
FailFast("GPU map received an invalid access mode");
}
const bool gpu_read = access == GpuAccess::Read || access == GpuAccess::ReadWrite;
const bool gpu_write = access == GpuAccess::Write || access == GpuAccess::ReadWrite;
const auto end = PageEnd(vaddr, size);
for (auto addr = PageStart(vaddr); addr < end; addr += PAGE_SIZE) {
auto& page = m_impl->GetPage(*m_impl->GetOrCreateRegion(addr), addr);
SpinGuard lock(page.lock);
if (page.resolving || page.mappings == std::numeric_limits<uint32_t>::max() ||
(gpu_read && page.gpu_read_mappings == std::numeric_limits<uint32_t>::max()) ||
(gpu_write && page.gpu_write_mappings == std::numeric_limits<uint32_t>::max())) {
Fatal("invalid map state at 0x%016" PRIx64, addr);
}
page.mappings++;
page.gpu_read_mappings += gpu_read ? 1u : 0u;
page.gpu_write_mappings += gpu_write ? 1u : 0u;
#if defined(__linux__)
// New guest mappings start read/write.
if (page.current_protection == UNKNOWN_PROTECTION) {
page.current_protection = READ_WRITE_PROTECTION;
}
#endif
}
}
void PageManager::OnGpuMap(uint64_t, uint64_t) {}
void PageManager::OnGpuUnmap(uint64_t vaddr, uint64_t size, GpuAccess access) {
if (g_in_fault_resolution) {
FailFast("GPU unmapping changed during fault resolution");
}
if (access != GpuAccess::Read && access != GpuAccess::Write && access != GpuAccess::ReadWrite) {
FailFast("GPU unmap received an invalid access mode");
}
const bool gpu_read = access == GpuAccess::Read || access == GpuAccess::ReadWrite;
const bool gpu_write = access == GpuAccess::Write || access == GpuAccess::ReadWrite;
const auto end = PageEnd(vaddr, size);
for (auto page_vaddr = PageStart(vaddr); page_vaddr < end; page_vaddr += PAGE_SIZE) {
auto* region = m_impl->FindRegion(page_vaddr);
if (region == nullptr) {
Fatal("unmapping unknown page 0x%016" PRIx64, page_vaddr);
}
auto& page = m_impl->GetPage(*region, page_vaddr);
SpinGuard lock(page.lock);
if (page.resolving || page.mappings == 0 || (gpu_read && page.gpu_read_mappings == 0) ||
(gpu_write && page.gpu_write_mappings == 0) ||
(page.mappings == 1 && (page.write_watchers != 0 || page.access_watchers != 0))) {
Fatal("invalid unmap state at 0x%016" PRIx64, page_vaddr);
}
page.mappings--;
page.gpu_read_mappings -= gpu_read ? 1u : 0u;
page.gpu_write_mappings -= gpu_write ? 1u : 0u;
if (page.mappings == 0) {
if (page.gpu_read_mappings != 0 || page.gpu_write_mappings != 0) {
FailFast("GPU unmap left nonzero GPU mapping counts");
}
page.late_read_pending = false;
page.late_write_pending = false;
}
}
}
void PageManager::OnGpuUnmap(uint64_t, uint64_t) {}
PageManager::BackingWrite::BackingWrite(PageManager& manager, uint64_t vaddr,
uint64_t size) noexcept
@@ -979,8 +558,7 @@ void PageManager::BeginBackingWrite(uint64_t vaddr, uint64_t size) noexcept {
}
auto& page = m_impl->GetPage(*region, address);
SpinGuard lock(page.lock);
if (page.mappings == 0 || page.resolving || page.backing_writer != 0 ||
page.access_watchers == 0) {
if (page.resolving || page.backing_writer != 0 || page.access_watchers == 0) {
Fatal("backing write races page resolution at 0x%016" PRIx64, address);
}
page.resolving = true;
@@ -1008,7 +586,7 @@ void PageManager::EndBackingWrite(uint64_t vaddr, uint64_t size) noexcept {
const auto old_protection = NO_ACCESS_PROTECTION;
const auto new_protection = Impl::WatcherProtection(page);
if (new_protection != old_protection) {
Impl::Protect(page, address, new_protection, old_protection, false);
m_impl->Protect(page, address, new_protection, old_protection, false);
}
Impl::PublishDelayedFaults(page, old_protection, new_protection);
if (page.write_watchers == 0 && page.access_watchers == 0) {
@@ -1109,7 +687,8 @@ bool PageManager::HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noex
page.write_watchers = 0;
}
const auto restored_protection = Impl::WatcherProtection(page);
Impl::Protect(page, PageStart(fault_vaddr), restored_protection, old_protection, true);
m_impl->Protect(page, PageStart(fault_vaddr), restored_protection, old_protection,
true);
if (page.write_watchers == 0) {
page.original_protection = 0;
}
src/graphics/host_gpu/pageManager.h
+2 -6
View File
@@ -13,11 +13,9 @@ namespace Libs::Graphics {
enum class PageFaultAccess { Read, Write, Execute, Unknown };
enum class PageFaultPhase { Invalidate, Complete, Release };
enum class PageWatchMode { Write, ReadWrite };
enum class GpuAccess { Read, Write, ReadWrite };
using PageFaultHandler = bool (*)(void* context, PageFaultAccess access, uint64_t vaddr,
uint64_t size, PageFaultPhase phase) noexcept;
class PageManager final {
public:
class BackingWrite final {
@@ -40,13 +38,11 @@ public:
[[nodiscard]] uint64_t GetPageSize() const;
[[nodiscard]] bool IsTracked(uint64_t vaddr) const noexcept;
[[nodiscard]] bool IsMapped(uint64_t vaddr, uint64_t size) const noexcept;
[[nodiscard]] bool HasGpuAccess(uint64_t vaddr, uint64_t size, GpuAccess access) const noexcept;
void UpdatePageWatchers(bool track, uint64_t vaddr, uint64_t size,
PageWatchMode mode = PageWatchMode::Write);
void OnGpuMap(uint64_t vaddr, uint64_t size, GpuAccess access = GpuAccess::ReadWrite);
void OnGpuUnmap(uint64_t vaddr, uint64_t size, GpuAccess access = GpuAccess::ReadWrite);
void OnGpuMap(uint64_t vaddr, uint64_t size);
void OnGpuUnmap(uint64_t vaddr, uint64_t size);
[[nodiscard]] bool HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noexcept;
[[nodiscard]] std::vector<std::unique_ptr<BackingWrite>>
src/graphics/host_gpu/renderer/cache/bufferCache.cpp
Vendored
-21
View File
@@ -714,7 +714,6 @@ BufferBinding BufferCache::ObtainBuffer(CommandBuffer& command, uint64_t vaddr,
if (command.IsInvalid() || command.IsExecute()) {
EXIT("BufferCache: buffer request requires a recording command buffer\n");
}
ValidateGpuAccess(vaddr, size, is_read, is_written);
std::lock_guard transaction(m_resource_mutex);
(void)SynchronizeBacking(vaddr, size);
@@ -999,7 +998,6 @@ void BufferCache::FillBuffer(uint64_t vaddr, uint64_t size, uint32_t value, bool
if (vaddr == 0) {
EXIT("BufferCache: invalid fill memory address\n");
}
ValidateGpuAccess(vaddr, size, false, true);
(void)m_texture_cache.ClearMeta(vaddr);
{
std::lock_guard transaction(m_resource_mutex);
@@ -1041,12 +1039,6 @@ void BufferCache::CopyBuffer(uint64_t dst_vaddr, uint64_t src_vaddr, uint64_t si
(src_gds && (src_vaddr > m_gds_buffer.Size() || size > m_gds_buffer.Size() - src_vaddr))) {
EXIT("BufferCache: invalid or overlapping copy range\n");
}
if (src_memory) {
ValidateGpuAccess(src_vaddr, size, true, false);
}
if (dst_memory) {
ValidateGpuAccess(dst_vaddr, size, false, true);
}
if (src_memory || dst_memory) {
std::lock_guard transaction(m_resource_mutex);
if (src_memory) {
@@ -1203,19 +1195,6 @@ void BufferCache::PublishImageBuffer(uint64_t vaddr, uint64_t size) {
owner->second->tick_accessed_last = m_gc_tick;
}
void BufferCache::ValidateGpuAccess(uint64_t vaddr, uint64_t size, bool is_read,
bool is_written) const {
if ((!is_read && !is_written) || vaddr == 0 || size == 0 || size > UINT64_MAX - vaddr) {
EXIT("BufferCache: invalid GPU access request\n");
}
if (is_read && !m_page_manager.HasGpuAccess(vaddr, size, GpuAccess::Read)) {
EXIT("BufferCache: GPU-read access denied\n");
}
if (is_written && !m_page_manager.HasGpuAccess(vaddr, size, GpuAccess::Write)) {
EXIT("BufferCache: GPU-write access denied\n");
}
}
void BufferCache::RunGarbageCollector() {
std::lock_guard transaction(m_resource_mutex);
const auto tick = m_gc_tick++;
src/graphics/host_gpu/renderer/cache/bufferCache.h
Vendored
+1 -2
View File
@@ -77,8 +77,7 @@ public:
void CompleteBackingPublication(uint64_t vaddr, uint64_t size, uint64_t tick);
[[nodiscard]] bool SynchronizeBacking(uint64_t vaddr, uint64_t size);
void PublishImageBuffer(uint64_t vaddr, uint64_t size);
void ValidateGpuAccess(uint64_t vaddr, uint64_t size, bool is_read, bool is_written) const;
void RunGarbageCollector();
void RunGarbageCollector();
private:
friend struct BufferCacheTestAccess;
src/graphics/host_gpu/renderer/cache/gpuResourceManager.cpp
Vendored
+6 -10
View File
@@ -4,7 +4,6 @@
#include "graphics/guest_gpu/command_processor/commandProcessor.h"
#include "graphics/guest_gpu/graphicsRun.h"
#include "graphics/host_gpu/renderer/commandScheduler.h"
namespace Libs::Graphics {
GpuResourceManager::GpuResourceManager(GraphicContext& graphics, CommandScheduler& scheduler)
@@ -115,22 +114,19 @@ bool GpuResourceManager::IsMapped(uint64_t vaddr, uint64_t size) const noexcept
return m_mapped_ranges.Contains(vaddr, size);
}
void GpuResourceManager::MapMemory(uint64_t vaddr, uint64_t size, GpuAccess access) {
void GpuResourceManager::MapMemory(uint64_t vaddr, uint64_t size) {
{
std::lock_guard lock(m_mapped_ranges_mutex);
m_mapped_ranges.Add(vaddr, size);
}
m_page_manager.OnGpuMap(vaddr, size, access);
m_page_manager.OnGpuMap(vaddr, size);
}
void GpuResourceManager::UnmapMemory(uint64_t vaddr, uint64_t size, GpuAccess access) {
if (!IsMapped(vaddr, size)) {
EXIT("cannot unmap an unmapped GPU resource range\n");
}
const auto unmap = [this, vaddr, size, access] {
m_texture_cache.UnmapMemory(vaddr, size);
void GpuResourceManager::UnmapMemory(uint64_t vaddr, uint64_t size) {
const auto unmap = [this, vaddr, size] {
m_buffer_cache.UnmapMemory(vaddr, size);
m_page_manager.OnGpuUnmap(vaddr, size, access);
m_texture_cache.UnmapMemory(vaddr, size);
m_page_manager.OnGpuUnmap(vaddr, size);
std::lock_guard lock(m_mapped_ranges_mutex);
m_mapped_ranges.Subtract(vaddr, size);
};
src/graphics/host_gpu/renderer/cache/gpuResourceManager.h
Vendored
+2 -2
View File
@@ -29,8 +29,8 @@ public:
[[nodiscard]] bool HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noexcept;
[[nodiscard]] bool InvalidateMemory(uint64_t vaddr, uint64_t size);
[[nodiscard]] bool IsMapped(uint64_t vaddr, uint64_t size) const noexcept;
void MapMemory(uint64_t vaddr, uint64_t size, GpuAccess access);
void UnmapMemory(uint64_t vaddr, uint64_t size, GpuAccess access);
void MapMemory(uint64_t vaddr, uint64_t size);
void UnmapMemory(uint64_t vaddr, uint64_t size);
void RunGarbageCollector();
private:
src/graphics/host_gpu/renderer/cache/textureCache.cpp
Vendored
-1
View File
@@ -1377,7 +1377,6 @@ bool TextureCache::ClearImageFromBuffer(CommandBuffer& command, uint64_t address
if (command.IsInvalid() || !GuestRange {address, size}.Valid()) {
EXIT("TextureCache: invalid image clear\n");
}
m_buffer_cache.ValidateGpuAccess(address, size, false, true);
std::lock_guard transaction(m_resource_mutex);
CacheLock lock(*this, m_lock);
ImageId selected {};
src/graphics/host_gpu/renderer/pipeline/descriptors.cpp
-2
View File
@@ -616,8 +616,6 @@ RenderExecutor::ResolveTexture(const ShaderRecompiler::IR::ImageResource& reso
(address & (static_cast<uint64_t>(size.align) - 1u)) != 0);
if (storage) {
ValidateStorageTexture(resource, descriptor, size.size);
m_context.GetBufferCache().ValidateGpuAccess(address, size.size, resource.read,
resource.written);
}
const auto pixel_format = TextureGetFormat(format);
src/kernel/memory.cpp
+1178 -1144
View File
File diff suppressed because it is too large Load Diff
src/kernel/memory.h
+23 -10
View File
@@ -145,7 +145,7 @@ int KYTY_SYSV_ABI KernelIsStack(void* addr, void** start, void** end);
int KYTY_SYSV_ABI KernelReserveVirtualRange(void** addr, size_t len, int flags, size_t alignment);
bool KernelHandleReservedRangeAccessViolation(uint64_t vaddr);
int KYTY_SYSV_ABI KernelAvailableFlexibleMemorySize(size_t* size);
int KYTY_SYSV_ABI KernelConfiguredFlexibleMemorySize(uint64_t* size);
int KYTY_SYSV_ABI KernelConfiguredFlexibleMemorySize(size_t* size);
int KYTY_SYSV_ABI KernelMprotect(const void* addr, size_t len, int prot);
int KYTY_SYSV_ABI KernelMtypeprotect(const void* addr, size_t len, int type, int prot);
int KYTY_SYSV_ABI KernelBatchMap(KernelBatchMapEntry* entries, int num_entries,
@@ -163,17 +163,30 @@ int KYTY_SYSV_ABI KernelMemoryPoolBatch(const KernelMemoryPoolBatchEntry* entrie
int KYTY_SYSV_ABI KernelMemoryPoolGetBlockStats(KernelMemoryPoolBlockStats* output,
size_t output_size);
void RegisterProgramMemory(uint64_t vaddr, uint64_t size, Common::VirtualMemory::Mode mode,
const char* name);
void UpdateProgramMemoryProtection(uint64_t vaddr, uint64_t size, Common::VirtualMemory::Mode mode);
void UnregisterProgramMemory(uint64_t vaddr, uint64_t size);
uint64_t AllocateProgramMemory(uint64_t search_addr, uint64_t size,
Common::VirtualMemory::Mode mode, const char* name);
void SetProgramMemoryProtection(uint64_t vaddr, uint64_t size, Common::VirtualMemory::Mode mode);
uint64_t AllocateRuntimeMemory(uint64_t search_addr, uint64_t size,
Common::VirtualMemory::Mode mode, const char* name,
bool fixed = false);
uint64_t AllocateGuestStackMemory(uint64_t search_addr, uint64_t size,
Common::VirtualMemory::Mode mode, const char* name);
bool ProtectGuestMemory(uint64_t vaddr, uint64_t size, Common::VirtualMemory::Mode mode,
Common::VirtualMemory::Mode* old_mode = nullptr);
// Transient PageManager watch state; does not change the guest mapping's semantic protection.
bool ProtectGuestHostMemory(uint64_t vaddr, uint64_t size, Common::VirtualMemory::Mode mode);
bool FreeGuestMemory(uint64_t vaddr, uint64_t size);
#if defined(KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS)
void TestFailNextPhysicalMemoryUnmap();
void TestFailPhysicalMemoryUnmapAfter(uint32_t successful_unmaps);
void TestFailHostReservationAfter(uint32_t successful_pages);
void TestFailNextFixedReserveRangeRegistration();
bool TestPlaceholderRangeIsFree(uint64_t vaddr, uint64_t size);
void TestFailNextPhysicalMemoryUnmap();
void TestFailPhysicalMemoryUnmapAfter(uint32_t successful_unmaps);
void TestFailGuestBackingStoreUnmapAfter(uint32_t successful_unmaps);
void TestFailNextFixedReserveRangeRegistration();
bool TestPlaceholderRangeIsFree(uint64_t vaddr, uint64_t size);
bool TestGuestAddressRangeIsOwned(uint64_t vaddr, uint64_t size);
bool TestGuestBackingOutsideAddressSpace();
uint64_t TestGuestBackingSize();
bool TestGuestFreeRangeBounds();
#endif
} // namespace Libs::LibKernel::Memory
src/kernel/memoryAddressSpace.inc
+588 -798
View File
File diff suppressed because it is too large Load Diff
src/kernel/pthread.cpp
+122 -37
View File
@@ -75,16 +75,16 @@ LIB_NAME("libkernel", "libkernel");
#undef PTHREAD_STACK_MIN
#endif
constexpr int KEYS_MAX = 256;
constexpr int DESTRUCTOR_ITERATIONS = 4;
constexpr size_t PTHREAD_STACK_DEFAULT = 0x100000;
constexpr size_t GUEST_PTHREAD_STACK_MIN = 0x4000;
constexpr size_t PTHREAD_STACK_PAGE = 0x4000;
constexpr size_t PTHREAD_STACK_GRANULARITY = 0x10000;
constexpr size_t PTHREAD_STACK_INITIAL = 0x200000;
constexpr size_t PTHREAD_STACK_EXTRA = 0x100000;
constexpr uint64_t PTHREAD_STACK_TOP = 0x7efff8000ull;
constexpr uint32_t SIGNAL_APC_POLL_MICROS = 10000;
constexpr int KEYS_MAX = 256;
constexpr int DESTRUCTOR_ITERATIONS = 4;
constexpr size_t PTHREAD_STACK_DEFAULT = 0x100000;
constexpr size_t GUEST_PTHREAD_STACK_MIN = 0x4000;
constexpr size_t PTHREAD_STACK_PAGE = 0x4000;
constexpr size_t PTHREAD_STACK_INITIAL = 0x200000;
constexpr size_t PTHREAD_STACK_EXTRA = 0x100000;
constexpr uint64_t PTHREAD_STACK_TOP = 0x7efff8000ull;
constexpr uint64_t PTHREAD_STACK_BOTTOM = 0x0000040000ull;
constexpr uint32_t SIGNAL_APC_POLL_MICROS = 10000;
static constexpr KernelClockid KERNEL_CLOCK_REALTIME = 0;
static constexpr KernelClockid KERNEL_CLOCK_VIRTUAL = 1;
@@ -697,16 +697,17 @@ static std::atomic<int32_t> g_pthread_thread_id = 0;
static Common::Mutex g_guest_stack_mutex;
static uint64_t g_guest_stack_last = 0;
struct CachedGuestStack {
uint64_t address;
size_t map_size;
size_t guard_size;
};
static std::vector<CachedGuestStack> g_guest_stack_cache;
static size_t RoundStackSize(size_t size) {
return ((size + PTHREAD_STACK_PAGE - 1) / PTHREAD_STACK_PAGE) * PTHREAD_STACK_PAGE;
}
static size_t RoundStackMappingSize(size_t size) {
return ((size + PTHREAD_STACK_GRANULARITY - 1) / PTHREAD_STACK_GRANULARITY) *
PTHREAD_STACK_GRANULARITY;
}
static int CreateGuestStack(PthreadAttr attr) {
if (attr == nullptr) {
return KERNEL_ERROR_EINVAL;
@@ -722,34 +723,41 @@ static int CreateGuestStack(PthreadAttr attr) {
const auto stack_size = RoundStackSize(attr->stack_size);
const auto guard_size = RoundStackSize(attr->guard_size);
const auto map_size = RoundStackMappingSize(stack_size + guard_size);
const auto map_size = stack_size + guard_size;
uint64_t stack_addr = 0;
bool cached = false;
{
Common::LockGuard lock(g_guest_stack_mutex);
if (g_guest_stack_last == 0) {
g_guest_stack_last = (PTHREAD_STACK_TOP - PTHREAD_STACK_INITIAL - PTHREAD_STACK_PAGE) &
~(static_cast<uint64_t>(PTHREAD_STACK_GRANULARITY) - 1);
auto cached_stack =
std::find_if(g_guest_stack_cache.begin(), g_guest_stack_cache.end(),
[map_size, guard_size](const auto& stack) {
return stack.map_size == map_size && stack.guard_size == guard_size;
});
if (cached_stack != g_guest_stack_cache.end()) {
stack_addr = cached_stack->address;
g_guest_stack_cache.erase(cached_stack);
cached = true;
} else {
if (g_guest_stack_last == 0) {
g_guest_stack_last = PTHREAD_STACK_TOP - PTHREAD_STACK_INITIAL - PTHREAD_STACK_PAGE;
}
if (map_size > g_guest_stack_last - PTHREAD_STACK_BOTTOM) {
return KERNEL_ERROR_EAGAIN;
}
stack_addr = g_guest_stack_last - map_size;
g_guest_stack_last -= map_size;
}
stack_addr = g_guest_stack_last - map_size;
g_guest_stack_last -= map_size;
}
void* mapped_addr = reinterpret_cast<void*>(stack_addr);
constexpr int GUEST_PROT_READ_WRITE = 0x03;
constexpr int GUEST_MAP_PRIVATE = 0x02;
constexpr int GUEST_MAP_FIXED = 0x10;
constexpr int GUEST_MAP_STACK = 0x400;
constexpr int GUEST_MAP_ANON = 0x1000;
int result = Memory::KernelMapNamedFlexibleMemory(
&mapped_addr, map_size, GUEST_PROT_READ_WRITE,
GUEST_MAP_PRIVATE | GUEST_MAP_FIXED | GUEST_MAP_STACK | GUEST_MAP_ANON, "stack");
if (result != OK) {
return KERNEL_ERROR_EAGAIN;
int result = OK;
if (!cached) {
stack_addr = Memory::AllocateGuestStackMemory(
stack_addr, map_size, Common::VirtualMemory::Mode::ReadWrite, "stack");
if (stack_addr == 0) {
return KERNEL_ERROR_EAGAIN;
}
}
if (guard_size != 0) {
@@ -761,7 +769,7 @@ static int CreateGuestStack(PthreadAttr attr) {
}
attr->stack_addr = reinterpret_cast<void*>(stack_addr + guard_size);
attr->stack_size = map_size - guard_size;
attr->stack_size = stack_size;
attr->stack_user = false;
attr->stack_map_addr = stack_addr;
attr->stack_map_size = map_size;
@@ -777,13 +785,90 @@ static void FreeGuestStack(PthreadAttr attr) {
return;
}
Memory::KernelMunmap(attr->stack_map_addr, attr->stack_map_size);
const auto guard_size = attr->stack_map_size - attr->stack_size;
{
Common::LockGuard lock(g_guest_stack_mutex);
g_guest_stack_cache.push_back({attr->stack_map_addr, attr->stack_map_size, guard_size});
}
attr->stack_addr = nullptr;
attr->stack_map_addr = 0;
attr->stack_map_size = 0;
}
#if defined(KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS)
bool TestGuestStackOwnerLifecycle(uint64_t* first_address, uint64_t* second_address,
uint64_t* map_size) {
if (first_address == nullptr || second_address == nullptr || map_size == nullptr) {
return false;
}
size_t flexible_before = 0;
if (Memory::KernelAvailableFlexibleMemorySize(&flexible_before) != OK) {
return false;
}
PthreadAttr attr = nullptr;
if (PthreadAttrInit(&attr) != OK) {
return false;
}
if (CreateGuestStack(attr) != OK) {
PthreadAttrDestroy(&attr);
return false;
}
*first_address = attr->stack_map_addr;
*map_size = attr->stack_map_size;
const bool first_owned =
Memory::TestGuestAddressRangeIsOwned(*first_address, static_cast<uint64_t>(*map_size));
uint64_t backing_value = 0;
const bool first_private =
!Memory::TryReadBacking(*first_address, &backing_value, sizeof(backing_value));
size_t flexible_during_first = 0;
const bool first_capacity_unchanged =
Memory::KernelAvailableFlexibleMemorySize(&flexible_during_first) == OK &&
flexible_during_first == flexible_before;
FreeGuestStack(attr);
if (CreateGuestStack(attr) != OK) {
PthreadAttrDestroy(&attr);
return false;
}
*second_address = attr->stack_map_addr;
const bool second_owned =
Memory::TestGuestAddressRangeIsOwned(*second_address, static_cast<uint64_t>(*map_size));
const bool second_private =
!Memory::TryReadBacking(*second_address, &backing_value, sizeof(backing_value));
size_t flexible_during_second = 0;
const bool second_capacity_unchanged =
Memory::KernelAvailableFlexibleMemorySize(&flexible_during_second) == OK &&
flexible_during_second == flexible_before;
FreeGuestStack(attr);
CachedGuestStack cached {};
bool found = false;
{
Common::LockGuard lock(g_guest_stack_mutex);
const auto entry = std::find_if(
g_guest_stack_cache.begin(), g_guest_stack_cache.end(),
[second_address](const auto& stack) { return stack.address == *second_address; });
if (entry != g_guest_stack_cache.end()) {
cached = *entry;
g_guest_stack_cache.erase(entry);
found = true;
}
}
const bool unmapped = found && Memory::KernelMunmap(cached.address, cached.map_size) == OK;
size_t flexible_after = 0;
const bool final_capacity_unchanged =
Memory::KernelAvailableFlexibleMemorySize(&flexible_after) == OK &&
flexible_after == flexible_before;
return PthreadAttrDestroy(&attr) == OK && first_owned && first_private &&
first_capacity_unchanged && second_owned && second_private &&
second_capacity_unchanged && unmapped && final_capacity_unchanged;
}
#endif
static KYTY_SYSV_ABI void* RunOnGuestStack(void* arg, pthread_entry_func_t func, void* stack_top) {
#if defined(__x86_64__) || defined(_M_X64)
void* ret = nullptr;
src/kernel/pthread.h
+6 -2
View File
@@ -112,11 +112,15 @@ void PthreadQueuePendingSignal(Pthread thread, int signum);
bool PthreadHasPendingSignal(Pthread thread, int signum);
bool PthreadTakePendingSignal(Pthread thread, int signum);
bool PthreadGetGuestStack(Pthread thread, uint64_t* stack_addr, uint64_t* stack_size);
#if defined(KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS)
bool TestGuestStackOwnerLifecycle(uint64_t* first_address, uint64_t* second_address,
uint64_t* map_size);
#endif
#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS
bool PthreadKillHost(Pthread thread, int host_signal);
#endif
int PthreadGetPriorityForKernel(Pthread thread);
int PthreadGetCurrentPriorityForKernel();
int PthreadGetPriorityForKernel(Pthread thread);
int PthreadGetCurrentPriorityForKernel();
int KYTY_SYSV_ABI KernelUsleep(KernelUseconds microseconds);
unsigned int KYTY_SYSV_ABI KernelSleep(unsigned int seconds);
src/loader/gamePatch.cpp
+7 -38
View File
@@ -2,6 +2,7 @@
#include "common/stringUtils.h"
#include "common/virtualMemory.h"
#include "kernel/memory.h"
#include "loader/elf.h"
#include "loader/runtimeLinker.h"
#include "loader/systemContent.h"
@@ -153,18 +154,6 @@ bool ValidateTarget(const Plan& plan, const Program* program, std::string* error
return true;
}
Common::VirtualMemory::Mode ReadableMode(Elf64_Word flags) {
const bool executable = (flags & PF_X) != 0;
const bool writable = (flags & PF_W) != 0;
if (executable && writable) {
return Common::VirtualMemory::Mode::ExecuteReadWrite;
}
if (executable) {
return Common::VirtualMemory::Mode::ExecuteRead;
}
return writable ? Common::VirtualMemory::Mode::ReadWrite : Common::VirtualMemory::Mode::Read;
}
bool ResolveWrite(const Program& program, Write* write, std::string* error) {
const auto* ehdr = program.elf->GetEhdr();
const auto* phdr = program.elf->GetPhdr();
@@ -178,16 +167,9 @@ bool ResolveWrite(const Program& program, Write* write, std::string* error) {
continue;
}
const auto segment_address = program.base_vaddr + segment.p_vaddr;
const bool add_read = (segment.p_flags & PF_R) == 0;
Common::VirtualMemory::Mode old_mode {};
if (add_read && !Common::VirtualMemory::Protect(segment_address, segment.p_memsz,
ReadableMode(segment.p_flags), &old_mode)) {
return Fail(error, "could not read a loaded executable segment");
}
const auto* begin = reinterpret_cast<const uint8_t*>(segment_address);
const auto* end = begin + segment.p_filesz;
const auto segment_address = program.base_vaddr + segment.p_vaddr;
const auto* begin = reinterpret_cast<const uint8_t*>(segment_address);
const auto* end = begin + segment.p_filesz;
for (auto* current = begin; current < end;) {
const auto* found =
std::search(current, end, write->expected.begin(), write->expected.end());
@@ -202,15 +184,9 @@ bool ResolveWrite(const Program& program, Write* write, std::string* error) {
match_count++;
current = found + 1;
}
if (add_read &&
!Common::VirtualMemory::Protect(segment_address, segment.p_memsz, old_mode)) {
return Fail(error, "could not restore executable segment protection");
}
}
::printf("Game patch: found %zu entries for '%s'\n", match_count,
write->patch_name.c_str());
::printf("Game patch: found %zu entries for '%s'\n", match_count, write->patch_name.c_str());
if (match == 0) {
return Fail(error, "original bytes not found for '" + write->patch_name + "'");
}
@@ -229,16 +205,9 @@ bool PrepareWrites(Plan* plan, const Program& program, std::string* error) {
bool ApplyWrites(Plan* plan, std::string* error) {
for (auto& write: plan->writes) {
Common::VirtualMemory::Mode old_mode {};
const auto size = write.replacement.size();
if (!Common::VirtualMemory::Protect(
write.address, size, Common::VirtualMemory::Mode::ExecuteReadWrite, &old_mode)) {
return Fail(error, "could not make patch memory writable");
}
const auto size = write.replacement.size();
std::memcpy(reinterpret_cast<void*>(write.address), write.replacement.data(), size);
if (!Common::VirtualMemory::Protect(write.address, size, old_mode) ||
!Common::VirtualMemory::FlushInstructionCache(write.address, size)) {
if (!Common::VirtualMemory::FlushInstructionCache(write.address, size)) {
return Fail(error, "could not finalize patch");
}
}
src/loader/runtimeLinker.cpp
+216 -52
View File
@@ -62,14 +62,16 @@ static void FreeTlsBlock(ThreadLocalStorage::Block* block) {
if (block->free_func != nullptr) {
block->free_func(block->ptr);
} else if (block->vm_alloc) {
Common::VirtualMemory::Free(reinterpret_cast<uint64_t>(block->ptr));
EXIT_IF(!Libs::LibKernel::Memory::FreeGuestMemory(reinterpret_cast<uint64_t>(block->ptr),
block->alloc_size));
} else {
delete[] block->ptr;
}
block->ptr = nullptr;
block->free_func = nullptr;
block->vm_alloc = false;
block->ptr = nullptr;
block->free_func = nullptr;
block->vm_alloc = false;
block->alloc_size = 0;
}
static uint64_t AlignUp(uint64_t value, uint64_t alignment) {
@@ -131,17 +133,25 @@ static std::vector<StubbedImportRecord> g_stubbed_imports;
static std::atomic_uint32_t g_unresolved_stub_call_log_count {0};
static std::vector<uint64_t> g_unresolved_stub_thunk_pages;
static uint64_t g_unresolved_stub_thunk_offset = 0;
static constexpr uint64_t UNRESOLVED_STUB_PAGE_SIZE = 4096;
static KYTY_SYSV_ABI uint64_t ResolveImportStubWithId(uint64_t record_id);
static bool PatchGuestMemory64(uint64_t vaddr, uint64_t value) {
auto* ptr = reinterpret_cast<uint64_t*>(vaddr);
bool changed = (*ptr != value);
std::memcpy(ptr, &value, sizeof(value));
return changed;
}
static uint64_t AllocateUnresolvedImportThunk(uint64_t record_id) {
constexpr uint64_t page_size = 4096;
constexpr uint64_t thunk_size = 162;
if (g_unresolved_stub_thunk_pages.empty() ||
g_unresolved_stub_thunk_offset + thunk_size > page_size) {
auto page = Common::VirtualMemory::Alloc(0, page_size,
Common::VirtualMemory::Mode::ExecuteReadWrite);
g_unresolved_stub_thunk_offset + thunk_size > UNRESOLVED_STUB_PAGE_SIZE) {
auto page = Libs::LibKernel::Memory::AllocateRuntimeMemory(
0, UNRESOLVED_STUB_PAGE_SIZE, Common::VirtualMemory::Mode::ExecuteReadWrite,
"unresolved_import_thunk");
EXIT_NOT_IMPLEMENTED(page == 0);
g_unresolved_stub_thunk_pages.push_back(page);
g_unresolved_stub_thunk_offset = 0;
@@ -298,7 +308,7 @@ static KYTY_SYSV_ABI uint64_t ResolveImportStubWithId(uint64_t record_id) {
resolved.name.c_str(), resolved.vaddr);
if (record.patch_vaddr != 0) {
*reinterpret_cast<uint64_t*>(record.patch_vaddr) = resolved.vaddr;
PatchGuestMemory64(record.patch_vaddr, resolved.vaddr);
}
return resolved.vaddr;
@@ -360,7 +370,7 @@ static KYTY_SYSV_ABI void RunEntry(uint64_t addr, EntryParams* params, atexit_fu
register uintptr_t guest_rbp_reg asm("r15") = guest_rbp;
#endif
#if defined(__APPLE__) || KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
#if defined(__APPLE__)
asm volatile(
"pushq %%r12\n\t"
"pushq %%r13\n\t"
@@ -374,16 +384,44 @@ static KYTY_SYSV_ABI void RunEntry(uint64_t addr, EntryParams* params, atexit_fu
"popq %%r13\n\t"
"popq %%r12\n\t"
:
#if defined(__APPLE__)
: [func] "r"(func_reg), "D"(params),
"S"(atexit_func), [guest_rsp] "r"(guest_rsp_reg), [guest_rbp] "r"(guest_rbp_reg)
#else
: [func] "r"(func), "D"(params),
"S"(atexit_func), [guest_rsp] "r"(guest_rsp), [guest_rbp] "r"(guest_rbp)
#endif
: "cc", "memory", "rax", "rcx", "rdx", "r8", "r9", "r10", "r11", "xmm0", "xmm1", "xmm2",
"xmm3", "xmm4", "xmm5", "xmm6", "xmm7", "xmm8", "xmm9", "xmm10", "xmm11", "xmm12",
"xmm13", "xmm14", "xmm15");
#elif KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
// Windows stack probes use the TEB stack limits during the guest stack switch.
// bounds, which describe the host stack and are invalid while RSP is in guest memory.
register entry_func_t func_reg asm("rbx") = func;
register uintptr_t guest_rsp_reg asm("r8") = guest_rsp;
register uintptr_t guest_rbp_reg asm("r9") = guest_rbp;
asm volatile("pushq %%r12\n\t"
"pushq %%r13\n\t"
"pushq %%r14\n\t"
"pushq %%r15\n\t"
"movq %%gs:0x08, %%r14\n\t"
"movq %%gs:0x10, %%r15\n\t"
"xorq %%rcx, %%rcx\n\t"
"movq %%rcx, %%gs:0x08\n\t"
"movq %%rcx, %%gs:0x10\n\t"
"movq %%rsp, %%r12\n\t"
"movq %%rbp, %%r13\n\t"
"movq %[guest_rsp], %%rsp\n\t"
"movq %[guest_rbp], %%rbp\n\t"
"callq *%[func]\n\t"
"movq %%r13, %%rbp\n\t"
"movq %%r12, %%rsp\n\t"
"movq %%r14, %%gs:0x08\n\t"
"movq %%r15, %%gs:0x10\n\t"
"popq %%r15\n\t"
"popq %%r14\n\t"
"popq %%r13\n\t"
"popq %%r12\n\t"
: [guest_rsp] "+r"(guest_rsp_reg), [guest_rbp] "+r"(guest_rbp_reg)
: [func] "r"(func_reg), "D"(params), "S"(atexit_func)
: "cc", "memory", "rax", "rcx", "rdx", "r10", "r11", "xmm0", "xmm1", "xmm2",
"xmm3", "xmm4", "xmm5", "xmm6", "xmm7", "xmm8", "xmm9", "xmm10", "xmm11",
"xmm12", "xmm13", "xmm14", "xmm15");
#else
// Clobbers prevent inputs from being allocated to r12/r13.
asm volatile("movq %%rsp, %%r12\n\t"
@@ -449,6 +487,110 @@ static KYTY_SYSV_ABI void RunEntry(uint64_t addr, EntryParams* params, atexit_fu
#endif
}
#if defined(KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS)
struct MainEntryStackTestState {
bool called = false;
uintptr_t rsp = 0;
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
uintptr_t teb_stack_base = UINTPTR_MAX;
uintptr_t teb_stack_limit = UINTPTR_MAX;
#endif
};
static KYTY_SYSV_ABI void TestMainEntryStackCallback(EntryParams* params,
atexit_func_t /*atexit_func*/) {
auto* state = reinterpret_cast<MainEntryStackTestState*>(const_cast<char*>(params->argv[0]));
asm volatile("movq %%rsp, %0" : "=r"(state->rsp) : : "memory");
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
asm volatile("movq %%gs:0x08, %0\n\t"
"movq %%gs:0x10, %1\n\t"
: "=r"(state->teb_stack_base), "=r"(state->teb_stack_limit)
:
: "memory");
#endif
state->called = true;
}
bool TestMainEntryUsesGuestStack() {
constexpr uint64_t stack_size = 0x10000;
const auto stack_base = Libs::LibKernel::Memory::AllocateRuntimeMemory(
0, stack_size, Common::VirtualMemory::Mode::ReadWrite, "main_entry_stack_test");
if (stack_base == 0) {
return false;
}
MainEntryStackTestState state {};
EntryParams params {};
params.argv[0] = reinterpret_cast<const char*>(&state);
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
uintptr_t original_teb_stack_base = 0;
uintptr_t original_teb_stack_limit = 0;
asm volatile("movq %%gs:0x08, %0\n\t"
"movq %%gs:0x10, %1\n\t"
: "=r"(original_teb_stack_base), "=r"(original_teb_stack_limit)
:
: "memory");
#endif
RunEntry(reinterpret_cast<uint64_t>(TestMainEntryStackCallback), &params, nullptr,
reinterpret_cast<void*>(stack_base + stack_size));
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
uintptr_t restored_teb_stack_base = 0;
uintptr_t restored_teb_stack_limit = 0;
asm volatile("movq %%gs:0x08, %0\n\t"
"movq %%gs:0x10, %1\n\t"
: "=r"(restored_teb_stack_base), "=r"(restored_teb_stack_limit)
:
: "memory");
const bool teb_ok = state.teb_stack_base == 0 && state.teb_stack_limit == 0 &&
restored_teb_stack_base == original_teb_stack_base &&
restored_teb_stack_limit == original_teb_stack_limit;
#else
constexpr bool teb_ok = true;
#endif
const bool rsp_ok = state.rsp >= stack_base && state.rsp < stack_base + stack_size;
const bool freed = Libs::LibKernel::Memory::FreeGuestMemory(stack_base, stack_size);
return state.called && rsp_ok && teb_ok && freed;
}
bool TestModuleRelocationUsesWritableHostMapping() {
constexpr uint64_t page_size = 0x4000;
constexpr uint64_t value = 0x4b59545950415443;
const auto base = Libs::LibKernel::Memory::AllocateProgramMemory(
0, page_size, Common::VirtualMemory::Mode::ReadWrite, "host_only_patch_test");
if (base == 0) {
return false;
}
Libs::LibKernel::Memory::SetProgramMemoryProtection(base, page_size,
Common::VirtualMemory::Mode::Read);
Libs::LibKernel::Memory::VirtualQueryInfo before {};
Libs::LibKernel::Memory::VirtualQueryInfo after {};
const bool before_ok =
Libs::LibKernel::Memory::KernelVirtualQuery(reinterpret_cast<const void*>(base), 0, &before,
sizeof(before)) == 0;
const bool changed = PatchGuestMemory64(base, value);
const bool after_ok = Libs::LibKernel::Memory::KernelVirtualQuery(
reinterpret_cast<const void*>(base), 0, &after, sizeof(after)) == 0;
const bool value_ok = *reinterpret_cast<const uint64_t*>(base) == value;
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
MEMORY_BASIC_INFORMATION mbi {};
const bool host_mode_ok =
VirtualQuery(reinterpret_cast<const void*>(base), &mbi, sizeof(mbi)) != 0 &&
mbi.Protect == PAGE_READWRITE;
#else
constexpr bool host_mode_ok = true;
#endif
const bool freed = Libs::LibKernel::Memory::FreeGuestMemory(base, page_size);
return before_ok && after_ok && changed && value_ok && host_mode_ok && freed &&
before.protection == after.protection;
}
#endif
static uint64_t GetAlignedSize(const Elf64_Phdr* p) {
return (p->p_align != 0 ? (p->p_memsz + (p->p_align - 1)) & ~(p->p_align - 1) : p->p_memsz);
}
@@ -1038,7 +1180,7 @@ static void RelocateRecord(uint32_t index, Elf64_Rela* r, Program* program, bool
// KYTY_PROFILER_BLOCK("patch");
if (ri.resolved) {
patched = Common::VirtualMemory::PatchReplace(ri.vaddr, ri.value);
patched = PatchGuestMemory64(ri.vaddr, ri.value);
} else {
uint64_t value = 0;
bool weak = (ri.bind == BindType::Weak || !program->fail_if_global_not_resolved);
@@ -1056,7 +1198,7 @@ static void RelocateRecord(uint32_t index, Elf64_Rela* r, Program* program, bool
}
if (value != 0) {
patched = Common::VirtualMemory::PatchReplace(ri.vaddr, value);
patched = PatchGuestMemory64(ri.vaddr, value);
} else {
auto dbg_str = fmt::format("[{:016x}] <- {:016x}, {}, {}, {}, {}", ri.vaddr, ri.value,
ri.name.c_str(), Common::EnumName(ri.type).c_str(),
@@ -1079,7 +1221,7 @@ static void RelocateRecord(uint32_t index, Elf64_Rela* r, Program* program, bool
}
if (value != 0) {
patched = Common::VirtualMemory::PatchReplace(ri.vaddr, value);
patched = PatchGuestMemory64(ri.vaddr, value);
}
}
}
@@ -1459,6 +1601,7 @@ void RuntimeLinker::Execute(const std::filesystem::path& game_patch) {
PreloadAdjacentPrograms();
RelocateAll();
if (!game_patch.empty()) {
GamePatch::Apply(game_patch, m_programs.empty() ? nullptr : m_programs.front());
}
@@ -1489,6 +1632,21 @@ void RuntimeLinker::Clear() {
DeleteProgram(p);
}
m_programs.clear();
for (const auto page: g_unresolved_stub_thunk_pages) {
EXIT_IF(!Libs::LibKernel::Memory::FreeGuestMemory(page, UNRESOLVED_STUB_PAGE_SIZE));
}
g_unresolved_stub_thunk_pages.clear();
g_unresolved_stub_thunk_offset = 0;
g_stubbed_imports.clear();
g_unresolved_stub_call_log_count.store(0);
if (g_invalid_memory != 0) {
EXIT_IF(!Libs::LibKernel::Memory::FreeGuestMemory(g_invalid_memory, 4096));
g_invalid_memory = 0;
}
g_tls_main_program = nullptr;
g_tls_cached_main_program = nullptr;
g_tls_cached_main_tcb = nullptr;
g_desired_base_addr = SYSTEM_RESERVED + CODE_BASE_OFFSET;
m_symbols.reset();
m_relocated = false;
}
@@ -1926,10 +2084,11 @@ uint8_t* RuntimeLinker::TlsGetAddr(Program* program) {
const auto tcb_offset =
program->tls.tcb_offset != 0 ? program->tls.tcb_offset : program->tls.image_size;
const auto alloc_size = AlignUp(tcb_offset, TCB_ALIGN) + TCB_SIZE;
tls.ptr = reinterpret_cast<uint8_t*>(
Common::VirtualMemory::Alloc(0, alloc_size, Common::VirtualMemory::Mode::ReadWrite));
tls.free_func = nullptr;
tls.vm_alloc = true;
tls.ptr = reinterpret_cast<uint8_t*>(Libs::LibKernel::Memory::AllocateRuntimeMemory(
0, alloc_size, Common::VirtualMemory::Mode::ReadWrite, "thread_local_storage"));
tls.free_func = nullptr;
tls.vm_alloc = true;
tls.alloc_size = alloc_size;
EXIT_IF(tls.ptr == nullptr);
@@ -2006,8 +2165,9 @@ void RuntimeLinker::LoadProgramToMemory(Program* program) {
EXIT_IF(tls_handler_size > UINT64_MAX - program->base_size_aligned);
program->mapped_size = program->base_size_aligned + tls_handler_size;
program->base_vaddr = Common::VirtualMemory::Alloc(
g_desired_base_addr, program->mapped_size, Common::VirtualMemory::Mode::ExecuteReadWrite);
program->base_vaddr = Libs::LibKernel::Memory::AllocateProgramMemory(
g_desired_base_addr, program->mapped_size, Common::VirtualMemory::Mode::ExecuteReadWrite,
Common::PathToString(program->file_name.filename()).c_str());
if (!is_shared) {
program->tls.handler_vaddr = program->base_vaddr + program->base_size_aligned;
@@ -2017,10 +2177,6 @@ void RuntimeLinker::LoadProgramToMemory(Program* program) {
EXIT_IF(program->base_vaddr == 0);
EXIT_IF(program->base_size_aligned < program->base_size);
Libs::LibKernel::Memory::RegisterProgramMemory(
program->base_vaddr, program->mapped_size, Common::VirtualMemory::Mode::ExecuteReadWrite,
Common::PathToString(program->file_name.filename()).c_str());
LOGF("base_vaddr = 0x%016" PRIx64 "\n"
"base_size = 0x%016" PRIx64 "\n"
"base_size_aligned = 0x%016" PRIx64 "\n"
@@ -2060,11 +2216,8 @@ void RuntimeLinker::LoadProgramToMemory(Program* program) {
}
if (!skip_protect) {
if (!Common::VirtualMemory::Protect(segment_addr, segment_memory_size, mode)) {
EXIT("failed to protect ELF segment %u\n", static_cast<unsigned>(i));
}
Libs::LibKernel::Memory::UpdateProgramMemoryProtection(segment_addr,
segment_memory_size, mode);
Libs::LibKernel::Memory::SetProgramMemoryProtection(segment_addr,
segment_memory_size, mode);
if (Common::VirtualMemory::IsExecute(mode)) {
Common::VirtualMemory::FlushInstructionCache(segment_addr, segment_memory_size);
@@ -2105,15 +2258,29 @@ void RuntimeLinker::LoadProgramToMemory(Program* program) {
void RuntimeLinker::DeleteProgram(Program* p) {
auto program = std::unique_ptr<Program>(p);
if (g_tls_main_program == program.get()) {
g_tls_main_program = nullptr;
}
if (g_tls_cached_main_program == program.get()) {
g_tls_cached_main_program = nullptr;
g_tls_cached_main_tcb = nullptr;
}
for (auto& record: g_stubbed_imports) {
if (record.patch_vaddr >= program->base_vaddr &&
record.patch_vaddr < program->base_vaddr + program->mapped_size) {
record.patch_vaddr = 0;
}
}
if (program->base_vaddr != 0 || program->mapped_size != 0) {
EXIT_IF(program->base_vaddr == 0 || program->mapped_size == 0);
Libs::LibKernel::Memory::UnregisterProgramMemory(program->base_vaddr, program->mapped_size);
EXIT_IF(!Common::VirtualMemory::Free(program->base_vaddr));
EXIT_IF(
!Libs::LibKernel::Memory::FreeGuestMemory(program->base_vaddr, program->mapped_size));
}
if (program->custom_call_plt_vaddr != 0 || program->custom_call_plt_num != 0) {
Common::VirtualMemory::Free(program->custom_call_plt_vaddr);
const auto size = Jit::CallPlt::GetSize(program->custom_call_plt_num);
EXIT_IF(!Libs::LibKernel::Memory::FreeGuestMemory(program->custom_call_plt_vaddr, size));
}
}
@@ -2237,13 +2404,13 @@ static void InstallRelocateHandler(Program* program) {
void** pltgot = reinterpret_cast<void**>(pltgot_vaddr);
Common::VirtualMemory::Mode old_mode {};
Common::VirtualMemory::Protect(pltgot_vaddr, pltgot_size, Common::VirtualMemory::Mode::Write,
&old_mode);
EXIT_IF(!Libs::LibKernel::Memory::ProtectGuestMemory(
pltgot_vaddr, pltgot_size, Common::VirtualMemory::Mode::Write, &old_mode));
pltgot[1] = program;
pltgot[2] = reinterpret_cast<void*>(RelocateHandler);
Common::VirtualMemory::Protect(pltgot_vaddr, pltgot_size, old_mode);
EXIT_IF(!Libs::LibKernel::Memory::ProtectGuestMemory(pltgot_vaddr, pltgot_size, old_mode));
if (Common::VirtualMemory::IsExecute(old_mode)) {
Common::VirtualMemory::FlushInstructionCache(pltgot_vaddr, pltgot_size);
@@ -2253,15 +2420,15 @@ static void InstallRelocateHandler(Program* program) {
if (program->custom_call_plt_vaddr == 0) {
program->custom_call_plt_num =
program->dynamic_info->jmprela_table_size / sizeof(Elf64_Rela);
auto size = Jit::CallPlt::GetSize(program->custom_call_plt_num);
program->custom_call_plt_vaddr =
Common::VirtualMemory::Alloc(SYSTEM_RESERVED, size, Common::VirtualMemory::Mode::Write);
auto size = Jit::CallPlt::GetSize(program->custom_call_plt_num);
program->custom_call_plt_vaddr = Libs::LibKernel::Memory::AllocateRuntimeMemory(
SYSTEM_RESERVED, size, Common::VirtualMemory::Mode::Write, "custom_call_plt");
EXIT_NOT_IMPLEMENTED(program->custom_call_plt_vaddr == 0);
auto* code = new (reinterpret_cast<void*>(program->custom_call_plt_vaddr))
Jit::CallPlt(program->custom_call_plt_num);
code->SetPltGot(pltgot_vaddr);
Common::VirtualMemory::Protect(program->custom_call_plt_vaddr, size,
Common::VirtualMemory::Mode::Execute);
EXIT_IF(!Libs::LibKernel::Memory::ProtectGuestMemory(program->custom_call_plt_vaddr, size,
Common::VirtualMemory::Mode::Execute));
Common::VirtualMemory::FlushInstructionCache(program->custom_call_plt_vaddr, size);
}
}
@@ -2272,8 +2439,8 @@ void RuntimeLinker::Relocate(Program* program) {
EXIT_IF(program == nullptr);
if (g_invalid_memory == 0) {
g_invalid_memory = Common::VirtualMemory::Alloc(INVALID_MEMORY, 4096,
Common::VirtualMemory::Mode::NoAccess);
g_invalid_memory = Libs::LibKernel::Memory::AllocateRuntimeMemory(
INVALID_MEMORY, 4096, Common::VirtualMemory::Mode::NoAccess, "invalid_memory", true);
EXIT_NOT_IMPLEMENTED(g_invalid_memory == 0);
}
@@ -2447,12 +2614,9 @@ void RuntimeLinker::SetupTlsHandler(Program* program) {
stub->SetOutputReg(reg);
}
if (!Common::VirtualMemory::Protect(program->tls.handler_vaddr, Jit::SafeCall::GetSize(),
Common::VirtualMemory::Mode::Execute)) {
EXIT("failed to protect program TLS handler\n");
}
Libs::LibKernel::Memory::UpdateProgramMemoryProtection(
program->tls.handler_vaddr, Jit::SafeCall::GetSize(), Common::VirtualMemory::Mode::Execute);
EXIT_IF(!Libs::LibKernel::Memory::ProtectGuestMemory(program->tls.handler_vaddr,
Jit::SafeCall::GetSize(),
Common::VirtualMemory::Mode::Execute));
Common::VirtualMemory::FlushInstructionCache(program->tls.handler_vaddr,
Jit::SafeCall::GetSize());
}
src/loader/runtimeLinker.h
+9 -3
View File
@@ -48,9 +48,10 @@ struct LibraryId {
struct ThreadLocalStorage {
struct Block {
uint8_t* ptr = nullptr;
application_heap_free_func_t free_func = nullptr;
bool vm_alloc = false;
uint8_t* ptr = nullptr;
application_heap_free_func_t free_func = nullptr;
bool vm_alloc = false;
uint64_t alloc_size = 0;
};
~ThreadLocalStorage();
@@ -204,6 +205,11 @@ private:
application_heap_posix_memalign_func_t m_application_heap_posix_memalign = nullptr;
};
#if defined(KYTY_VIRTUAL_MEMORY_ALLOCATION_TESTS)
bool TestMainEntryUsesGuestStack();
bool TestModuleRelocationUsesWritableHostMapping();
#endif
} // namespace Loader
#endif /* EMULATOR_INCLUDE_EMULATOR_LOADER_RUNTIMELINKER_H_ */
tests/MemoryTrackerTests.cpp
+46 -35
View File
@@ -1,6 +1,7 @@
#include "graphics/host_gpu/memoryTracker.h"
#include "graphics/host_gpu/rangeSet.h"
#include "common/assert.h"
#include "common/virtualMemory.h"
#include <atomic>
#include <cstdint>
@@ -209,6 +210,20 @@ class SharedPage final {
HANDLE mapping_ = nullptr;
};
#endif
bool ProtectAddressSpace(uint64_t vaddr, uint64_t size,
Common::VirtualMemory::Mode mode) {
uint32_t protection = PAGE_NOACCESS;
if (mode == Common::VirtualMemory::Mode::Read) {
protection = PAGE_READONLY;
} else if (mode == Common::VirtualMemory::Mode::ReadWrite) {
protection = PAGE_READWRITE;
}
DWORD old_protection = 0;
return VirtualProtect(reinterpret_cast<void *>(vaddr), size, protection,
&old_protection) != 0;
}
#if 1
bool DummyFault(void *, PageFaultAccess, uint64_t, uint64_t, PageFaultPhase) noexcept {
@@ -353,7 +368,8 @@ struct DownloadTrackerHarness {
return completed;
}
DownloadTrackerHarness() : page_manager(Fault, this), tracker(page_manager) {}
DownloadTrackerHarness()
: page_manager(Fault, this), tracker(page_manager) {}
PageFaultAccess pending_access = PageFaultAccess::Unknown;
uint64_t download_address = 0;
@@ -366,11 +382,6 @@ struct DownloadTrackerHarness {
std::atomic<PageManager *> g_native_page_manager{nullptr};
std::atomic_bool g_native_fault_entered{false};
std::atomic_bool g_unmap_contended{false};
void UnmapContended() noexcept {
g_unmap_contended.store(true, std::memory_order_release);
}
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
LONG CALLBACK NativeTrackerFaultHandler(EXCEPTION_POINTERS *exception) {
@@ -552,7 +563,7 @@ void TestGpuDownloadFaultOwnership() {
!harness.tracker.IsRegionGpuModified(address, page_size) &&
harness.tracker.IsRegionCpuModified(address, page_size) && IsWritable(memory),
"GPU write fault did not download before granting CPU ownership");
harness.tracker.UnmapMemory(address, page_size);
harness.tracker.UntrackMemory(address, page_size);
}
void TestVirtualGpuWriteDiscard() {
@@ -578,7 +589,7 @@ void TestVirtualGpuWriteDiscard() {
Check(!harness.tracker.IsRegionGpuModified(address, page_size) &&
harness.tracker.IsRegionCpuModified(address, page_size) && IsWritable(memory),
"virtual GPU discard did not transfer the page to CPU ownership");
harness.tracker.UnmapMemory(address, page_size);
harness.tracker.UntrackMemory(address, page_size);
Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
}
@@ -684,6 +695,15 @@ void TestRangeSet() {
"range set subtraction did not preserve both exact tails");
}
void TestQueriesDoNotRequireMappedOwnership() {
constexpr uint64_t address = 0x0000000203000000ull;
TrackerHarness harness;
const auto page_size = harness.page_manager.GetPageSize();
Check(harness.tracker.IsRegionCpuModified(address, page_size) &&
!harness.tracker.IsRegionGpuModified(address, page_size),
"unowned tracker range did not expose its initial CPU-dirty state");
}
void TestRangeInvalidation() {
constexpr uintptr_t base = 0x0000000201000000ull;
TrackerHarness harness;
@@ -767,7 +787,7 @@ void TestCpuDirtyUploadAndFault() {
Check(IsWritable(memory),
"explicit CPU dirty transition did not release the rearmed watch");
tracker.UnmapMemory(address, page_size * 2);
tracker.UntrackMemory(address, page_size * 2);
Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
}
@@ -795,7 +815,7 @@ void TestFaultDuringUploadRemainsDirty() {
});
Check(tracker.IsRegionCpuModified(address, page_size) && IsWritable(memory),
"upload completion erased a racing CPU dirty transition");
tracker.UnmapMemory(address, page_size);
tracker.UntrackMemory(address, page_size);
Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
}
@@ -840,7 +860,7 @@ void TestNativeStoreDuringRangeEnumeration() {
IsWritable(memory),
"native store during range enumeration was lost");
tracker.UnmapMemory(address, page_size);
tracker.UntrackMemory(address, page_size);
Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
}
@@ -924,7 +944,7 @@ void TestFaultDuringDownloadSynchronization() {
page_manager.IsTracked(address + page_size * 2),
"uncontended dirty page did not retain its clean write watch");
tracker.UnmapMemory(address, page_size * 3);
tracker.UntrackMemory(address, page_size * 3);
}
void TestFaultAndExplicitDirtyRace() {
@@ -964,7 +984,7 @@ void TestFaultAndExplicitDirtyRace() {
IsWritable(memory),
"fault/explicit-dirty race lost dirty state or write access");
}
tracker.UnmapMemory(address, page_size);
tracker.UntrackMemory(address, page_size);
Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
}
@@ -1016,7 +1036,7 @@ void TestSharedTrackersAndConcurrentPageFaults() {
}
harness.first.UntrackMemory(address, page_size * 2);
harness.second.UnmapMemory(address, page_size * 2);
harness.second.UntrackMemory(address, page_size * 2);
Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
}
@@ -1049,7 +1069,7 @@ void TestGpuDirtyBits() {
"explicit GPU dirty transition did not trap CPU access");
tracker.UnmarkRegionAsGpuModified(address, page_size);
tracker.MarkRegionAsCpuModified(address, page_size);
tracker.UnmapMemory(address, page_size * 2);
tracker.UntrackMemory(address, page_size * 2);
Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
}
@@ -1084,7 +1104,7 @@ void TestCrossRegionUpload() {
"cross-region written upload did not mark GPU dirty state");
tracker.UnmarkRegionAsGpuModified(boundary - page_size, page_size * 2);
tracker.MarkRegionAsCpuModified(boundary - page_size, page_size * 2);
tracker.UnmapMemory(address, region_size * 2);
tracker.UntrackMemory(address, region_size * 2);
Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
}
@@ -1094,9 +1114,6 @@ void TestCrossRegionUpload() {
auto &tracker = harness.tracker;
auto &page_manager = harness.page_manager;
const auto page_size = page_manager.GetPageSize();
if (std::strcmp(name, "unmapped") == 0) {
(void)tracker.IsRegionCpuModified(base, page_size);
}
const auto allocation_size =
std::strcmp(name, "missing-download-bytes") == 0 ? page_size * 2
: page_size;
@@ -1149,20 +1166,6 @@ void TestCrossRegionUpload() {
}
});
fault.join();
} else if (std::strcmp(name, "gpu-dirty-unmap-race") == 0) {
g_unmap_contended.store(false, std::memory_order_release);
MemoryTracker::SetUnmapContentionHook(UnmapContended);
std::thread unmap;
tracker.ForEachUploadRange(
address, page_size, true, [](uint64_t, uint64_t) noexcept {},
[&]() noexcept {
unmap = std::thread(
[&] { tracker.UnmapMemory(address, page_size); });
while (!g_unmap_contended.load(std::memory_order_acquire)) {
std::this_thread::yield();
}
});
unmap.join();
} else if (std::strcmp(name, "missing-download-bytes") == 0) {
tracker.ForEachUploadRange(
address, allocation_size, true, [](uint64_t, uint64_t) noexcept {},
@@ -1193,8 +1196,7 @@ void TestFatalPaths() {
#endif
for (const char *name : {"gpu-dirty-fault", "gpu-dirty-read", "virtual-gpu-read",
"gpu-dirty-explicit-cpu",
"unmapped", "reentrant-upload",
"writable-upload-race", "gpu-dirty-unmap-race",
"reentrant-upload", "writable-upload-race",
"missing-download-bytes"}) {
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
std::string command = std::string("\"") + path + "\" --death " + name;
@@ -1236,6 +1238,14 @@ void TestFatalPaths() {
} // namespace
namespace Libs::LibKernel::Memory {
bool ProtectGuestHostMemory(uint64_t vaddr, uint64_t size, Common::VirtualMemory::Mode mode) {
return ProtectAddressSpace(vaddr, size, mode);
}
} // namespace Libs::LibKernel::Memory
int main(int argc, char **argv) {
#if 1
if (argc == 3 && std::strcmp(argv[1], "--death") == 0) {
@@ -1249,6 +1259,7 @@ int main(int argc, char **argv) {
TestSameSlabTrackerArbitration();
TestSharedMetadataAndImagePageFault();
TestRangeSet();
TestQueriesDoNotRequireMappedOwnership();
TestRangeInvalidation();
TestGpuDirtyBits();
TestCrossRegionUpload();
tests/PageManagerTests.cpp
+32 -85
View File
@@ -1,4 +1,5 @@
#include "graphics/host_gpu/pageManager.h"
#include "common/virtualMemory.h"
#include <atomic>
#include <cstdint>
@@ -28,7 +29,6 @@
namespace {
using Libs::Graphics::GpuAccess;
using Libs::Graphics::PageFaultAccess;
using Libs::Graphics::PageManager;
@@ -122,6 +122,23 @@ uint32_t Protection(const void *address) {
return info.Protect;
}
#endif
std::atomic_uint64_t g_protection_calls{0};
bool ProtectAddressSpace(uint64_t vaddr, uint64_t size,
Common::VirtualMemory::Mode mode) {
uint32_t protection = PAGE_NOACCESS;
if (mode == Common::VirtualMemory::Mode::Read) {
protection = PAGE_READONLY;
} else if (mode == Common::VirtualMemory::Mode::ReadWrite) {
protection = PAGE_READWRITE;
}
DWORD old_protection = 0;
g_protection_calls.fetch_add(1, std::memory_order_relaxed);
return VirtualProtect(reinterpret_cast<void *>(vaddr), size, protection,
&old_protection) != 0;
}
#if 1
struct FaultContext {
@@ -254,6 +271,7 @@ uint8_t *Allocate(uint64_t size, uint32_t protection = PAGE_READWRITE) {
}
void TestWatchFaultAndUnwatch() {
g_protection_calls.store(0, std::memory_order_relaxed);
FaultContext context;
PageManager manager(InvalidateFault, &context);
context.manager = &manager;
@@ -266,6 +284,8 @@ void TestWatchFaultAndUnwatch() {
Check(manager.IsTracked(reinterpret_cast<uint64_t>(memory)) &&
!IsWritable(memory),
"watch did not protect the page");
Check(g_protection_calls.load(std::memory_order_relaxed) != 0,
"watch protection bypassed the address-space owner callback");
Check(manager.HandleFault(PageFaultAccess::Write,
reinterpret_cast<uint64_t>(memory + 32)),
"tracked write fault was not handled");
@@ -343,14 +363,6 @@ void TestPermittedMappedLateFaultsResume() {
"second delayed mapped write was not accepted");
Check(manager.HandleFault(PageFaultAccess::Read, address),
"delayed mapped read was not accepted on readable backing");
DWORD old_protection = 0;
Check(VirtualProtect(memory, page_size, PAGE_READONLY, &old_protection) != 0 &&
old_protection == PAGE_READWRITE,
"failed to prepare intentional read-only protection");
Check(!manager.HandleFault(PageFaultAccess::Write, address),
"intentional read-only mapping accepted a write fault");
Check(VirtualProtect(memory, page_size, PAGE_READWRITE, &old_protection) != 0,
"failed to restore writable protection");
manager.OnGpuUnmap(address, page_size);
Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
}
@@ -496,31 +508,6 @@ void TestNativeAccessViolation() {
Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
}
void TestInvalidLateWriteTokenIsConsumed() {
FaultContext context;
PageManager manager(InvalidateFault, &context);
context.manager = &manager;
const auto page_size = manager.GetPageSize();
auto *memory = Allocate(page_size);
const auto address = reinterpret_cast<uint64_t>(memory);
manager.OnGpuMap(address, page_size);
manager.UpdatePageWatchers(true, address, page_size);
Check(manager.HandleFault(PageFaultAccess::Write, address),
"initial write fault was not handled");
DWORD old_protection = 0;
Check(VirtualProtect(memory, page_size, PAGE_READONLY, &old_protection) !=
0 &&
old_protection == PAGE_READWRITE,
"failed to create invalid late-write protection state");
Check(!manager.HandleFault(PageFaultAccess::Write, address) &&
!manager.HandleFault(PageFaultAccess::Write, address),
"invalid late-write token was accepted or retained");
Check(VirtualProtect(memory, page_size, PAGE_READWRITE, &old_protection) != 0,
"failed to restore test protection");
manager.OnGpuUnmap(address, page_size);
Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
}
void TestCrossRegionRange() {
FaultContext context;
PageManager manager(InvalidateFault, &context);
@@ -627,9 +614,7 @@ void TestBatchedWatcherRanges() {
manager->UpdatePageWatchers(false, 0x1000, page_size);
} else {
const bool two_pages = std::strcmp(name, "cross-reentrant") == 0;
auto *memory = Allocate(
two_pages ? page_size * 2 : page_size,
std::strcmp(name, "protection") == 0 ? PAGE_READONLY : PAGE_READWRITE);
auto *memory = Allocate(two_pages ? page_size * 2 : page_size);
const auto address = reinterpret_cast<uint64_t>(memory);
manager->OnGpuMap(address, two_pages ? page_size * 2 : page_size);
manager->UpdatePageWatchers(true, address, page_size);
@@ -659,9 +644,7 @@ void TestBatchedWatcherRanges() {
}
(void)manager->HandleFault(PageFaultAccess::Read, address);
first.join();
} else if (std::strcmp(name, "watched-unmap") == 0) {
manager->OnGpuUnmap(address, page_size);
} else if (std::strcmp(name, "protection") != 0) {
} else {
std::_Exit(0x7f);
}
}
@@ -712,7 +695,7 @@ void TestFatalPaths() {
for (const char *name :
{"invalid-range", "unknown-untrack", "destructor-watch", "non-write",
"callback-false", "reentrant", "cross-reentrant",
"concurrent-non-write", "watched-unmap", "protection"}) {
"concurrent-non-write"}) {
CheckDeathCase(name);
}
}
@@ -773,51 +756,18 @@ void TestExternalDirtyTransferDuringResolution() {
Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
}
void TestMappingDoesNotRequireCpuWriteAccess() {
FaultContext context;
PageManager manager(InvalidateFault, &context);
context.manager = &manager;
const auto page_size = manager.GetPageSize();
auto *memory = Allocate(page_size);
DWORD old_protection = 0;
Check(VirtualProtect(memory, page_size, PAGE_NOACCESS, &old_protection) != 0 &&
old_protection == PAGE_READWRITE,
"failed to prepare CPU-inaccessible mapping");
const auto address = reinterpret_cast<uint64_t>(memory);
manager.OnGpuMap(address, page_size);
Check(manager.IsMapped(address, page_size),
"CPU-inaccessible committed range was not GPU mapped");
manager.OnGpuUnmap(address, page_size);
Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
}
void TestGpuAccessPermissions() {
FaultContext context;
PageManager manager(InvalidateFault, &context);
context.manager = &manager;
const auto page_size = manager.GetPageSize();
auto *memory = Allocate(page_size);
const auto address = reinterpret_cast<uint64_t>(memory);
manager.OnGpuMap(address, page_size, GpuAccess::Read);
Check(manager.HasGpuAccess(address, page_size, GpuAccess::Read) &&
!manager.HasGpuAccess(address, page_size, GpuAccess::Write),
"read-only GPU mapping granted write access");
manager.OnGpuMap(address, page_size, GpuAccess::Write);
Check(manager.HasGpuAccess(address, page_size, GpuAccess::ReadWrite),
"overlapping GPU mappings did not combine permissions");
manager.OnGpuUnmap(address, page_size, GpuAccess::Read);
Check(!manager.HasGpuAccess(address, page_size, GpuAccess::Read) &&
manager.HasGpuAccess(address, page_size, GpuAccess::Write),
"GPU read unmap removed the wrong permission");
manager.OnGpuUnmap(address, page_size, GpuAccess::Write);
Check(!manager.IsMapped(address, page_size),
"GPU permission mappings were not fully balanced");
Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
}
#endif
} // namespace
namespace Libs::LibKernel::Memory {
bool ProtectGuestHostMemory(uint64_t vaddr, uint64_t size, Common::VirtualMemory::Mode mode) {
return ProtectAddressSpace(vaddr, size, mode);
}
} // namespace Libs::LibKernel::Memory
int main(int argc, char **argv) {
#if 1
if (argc == 3 && std::strcmp(argv[1], "--death") == 0) {
@@ -831,13 +781,10 @@ int main(int argc, char **argv) {
TestNativeDelayedReadAfterModeDowngrade();
TestDelayedFaultAfterExplicitUnwatch();
TestNativeAccessViolation();
TestInvalidLateWriteTokenIsConsumed();
TestCrossRegionRange();
TestBatchedWatcherRanges();
TestConcurrentFault();
TestExternalDirtyTransferDuringResolution();
TestMappingDoesNotRequireCpuWriteAccess();
TestGpuAccessPermissions();
TestFatalPaths();
std::puts("PageManagerTests: all cases passed");
return 0;
tests/ShaderRecompilerComputeTests.cpp
+78 -21
View File
@@ -1274,6 +1274,59 @@ public:
std::printf("[host] %-32s ok\n", "SchedulerTimeline");
}
void CheckGpuMappedRangeLifecycle() {
EnsureRuntimeContext();
CommandScheduler scheduler(Renderer(), m_runtime_context);
HW::Context registers{};
HW::UserConfig user_config{};
HW::Shader shaders{};
scheduler.Begin(registers, user_config, shaders);
Gpu gpu(Renderer());
GpuResourceManager resources(m_runtime_context, scheduler);
resources.SetGpu(&gpu);
constexpr uint64_t base = 0x0000000200000000ull;
constexpr uint64_t page = 0x4000;
resources.MapMemory(base, page * 4);
resources.MapMemory(base + page * 2, page * 4);
Require("GpuMappedRangeLifecycle", "union",
resources.IsMapped(base, page * 6) &&
!resources.IsMapped(base, page * 7),
"overlapping maps did not form one interval union");
resources.UnmapMemory(base + page * 2, page * 2);
Require("GpuMappedRangeLifecycle", "subtract",
resources.IsMapped(base, page * 2) &&
resources.IsMapped(base + page * 4, page * 2) &&
!resources.IsMapped(base, page * 6),
"partial unmap did not punch the expected interval hole");
resources.UnmapMemory(base + page * 2, page * 2);
Require("GpuMappedRangeLifecycle", "idempotent unmap",
resources.IsMapped(base, page * 2) &&
resources.IsMapped(base + page * 4, page * 2),
"unmapping an absent interval changed neighboring mappings");
resources.UnmapMemory(base, page * 6);
Require("GpuMappedRangeLifecycle", "clear",
!resources.IsMapped(base, page * 6),
"full unmap did not clear the interval union");
constexpr uint64_t old_prt = base + page * 8;
constexpr uint64_t new_prt = base + page * 16;
resources.MapMemory(old_prt, page * 4);
resources.UnmapMemory(old_prt, page * 4);
resources.MapMemory(new_prt, page * 6);
Require("GpuMappedRangeLifecycle", "PRT replacement",
!resources.IsMapped(old_prt, page * 4) &&
resources.IsMapped(new_prt, page * 6),
"old-unmap/new-map did not replace full PRT coverage");
resources.SetGpu(nullptr);
scheduler.Finish();
std::printf("[host] %-32s ok\n", "GpuMappedRangeLifecycle");
}
void CheckStreamBufferRing() {
EnsureRuntimeContext();
CommandScheduler scheduler(Renderer(), m_runtime_context);
@@ -1526,7 +1579,7 @@ public:
fault_memory == reinterpret_cast<void *>(fault_base),
"fixed processor-fault allocation failed");
auto &resources = context.GetGpuResources();
resources.MapMemory(fault_base, fault_size, GpuAccess::ReadWrite);
resources.MapMemory(fault_base, fault_size);
constexpr uint64_t immediate_dst = fault_base + 0x1000;
constexpr uint64_t immediate_memory_dst = fault_base + 0x2000;
@@ -1683,7 +1736,7 @@ public:
resources.InvalidateMemory(fault_base, sizeof(uint32_t)),
"processor memory invalidation did not find its mapped range");
});
resources.UnmapMemory(fault_base, fault_size, GpuAccess::ReadWrite);
resources.UnmapMemory(fault_base, fault_size);
Require("GpuCommandLane", "processor fault unmap",
Libs::LibKernel::Memory::KernelMunmap(fault_base, fault_size) == 0,
"processor-fault direct-memory mapping release failed");
@@ -1953,7 +2006,7 @@ public:
GpuResourceManager resources(m_runtime_context, scheduler);
resources.SetGpu(&gpu);
auto &cache = resources.GetBufferCache();
resources.MapMemory(base, allocation_size, GpuAccess::ReadWrite);
resources.MapMemory(base, allocation_size);
const auto MarkGpuWrite = [&](uint64_t address, uint64_t size) {
auto allocation =
@@ -2376,7 +2429,7 @@ public:
sizeof(reacquire_value));
resources.SetGpu(nullptr);
resources.UnmapMemory(base, allocation_size, GpuAccess::ReadWrite);
resources.UnmapMemory(base, allocation_size);
scheduler.Finish();
}
gpu.Shutdown();
@@ -2431,7 +2484,7 @@ public:
narrow_download != nullptr && narrow_download_offset % 4 == 0 &&
wide_download != nullptr && wide_download_offset % 16 == 0,
"wide/block image readback was not aligned to its texel block");
resources.MapMemory(base, allocation_size, GpuAccess::ReadWrite);
resources.MapMemory(base, allocation_size);
ImageDesc sampled{};
sampled.type = BindingType::Texture;
@@ -4834,7 +4887,7 @@ public:
m_device.destroyShaderModule(ms_depth_module, nullptr);
resources.SetGpu(nullptr);
resources.UnmapMemory(base, allocation_size, GpuAccess::ReadWrite);
resources.UnmapMemory(base, allocation_size);
scheduler.Finish();
}
gpu.Shutdown();
@@ -4879,7 +4932,7 @@ public:
scheduler.Begin(registers, user_config, shaders);
{
GpuResourceManager resources(m_runtime_context, scheduler);
resources.MapMemory(base, allocation_size, GpuAccess::ReadWrite);
resources.MapMemory(base, allocation_size);
const uint32_t pitch = TileGetTexturePitch(format, 1, 1, tile);
TileSizeAlign total{};
TileSizeOffset mip{};
@@ -4978,7 +5031,7 @@ public:
std::vector<u32>{0x40004200u, 0x44003c00u},
"tiled BGRA16 Buffer mirror changed guest component order");
DestroyBuffer(&mirror_readback);
resources.UnmapMemory(base, allocation_size, GpuAccess::ReadWrite);
resources.UnmapMemory(base, allocation_size);
scheduler.Finish();
}
Require(name, "unmap",
@@ -5025,7 +5078,7 @@ public:
auto &resources = context.GetGpuResources();
auto &texture_cache = resources.GetTextureCache();
auto &executor = context.GetRenderExecutor();
resources.MapMemory(base, allocation_size, GpuAccess::ReadWrite);
resources.MapMemory(base, allocation_size);
constexpr auto stencil_format =
Prospero::GpuEnumValue(Prospero::BufferFormat::k8UInt);
@@ -6011,10 +6064,8 @@ public:
const auto stale_ordered_color =
texture_cache.FindImage(ordered_color_desc);
RenderExecutorTestAccess::BindRenderTarget(executor, stale_ordered_color);
resources.UnmapMemory(ordered_color_address, target_mip_size,
GpuAccess::ReadWrite);
resources.MapMemory(ordered_color_address, target_mip_size,
GpuAccess::ReadWrite);
resources.UnmapMemory(ordered_color_address, target_mip_size);
resources.MapMemory(ordered_color_address, target_mip_size);
auto ordered_depth_desc = depth;
ordered_depth_desc.info.stencil = {ordered_color_address,
@@ -6167,7 +6218,7 @@ public:
texture_cache.GetImage(depth_id).usage.storage,
"storage stencil binding did not acquire the associated depth owner");
RenderExecutorTestAccess::ResetBindings(executor);
resources.UnmapMemory(base, allocation_size, GpuAccess::ReadWrite);
resources.UnmapMemory(base, allocation_size);
scheduler.Finish();
}
@@ -17198,12 +17249,12 @@ void CheckStandard64RenderTargetTileRoundTrip() {
void CheckStorageTextureGpuOwnedRebindState() {
constexpr uintptr_t base = 0x0000000200200000ull;
constexpr uint64_t size = 0x10000;
auto *memory = static_cast<uint8_t *>(
VirtualAlloc(reinterpret_cast<void *>(base), size,
MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE));
const auto guest_memory = Libs::LibKernel::Memory::AllocateRuntimeMemory(
base, size, Common::VirtualMemory::Mode::ReadWrite,
"storage_texture_gpu_owned_rebind", true);
auto *memory = reinterpret_cast<uint8_t *>(guest_memory);
Require("StorageTextureGpuOwnedRebind", "allocation",
memory == reinterpret_cast<void *>(base),
"fixed VirtualAlloc failed");
guest_memory == base, "fixed guest-owner allocation failed");
PageManager page_manager(CacheFault, nullptr);
MemoryTracker tracker(page_manager);
page_manager.OnGpuMap(base, size);
@@ -17215,7 +17266,6 @@ void CheckStorageTextureGpuOwnedRebindState() {
Require(
"StorageTextureGpuOwnedRebind", "owned",
tracker.IsRegionGpuModified(base, size) &&
page_manager.IsMapped(base, size) &&
(!HostMemoryQueryReadable(base, size, readable) || readable < size) &&
HostMemoryQueryRange(base, size, HostMemoryAccess::Mapped, mapped) &&
mapped == size &&
@@ -17261,7 +17311,8 @@ void CheckStorageTextureGpuOwnedRebindState() {
tracker.UntrackMemory(base, size);
page_manager.OnGpuUnmap(base, size);
Require("StorageTextureGpuOwnedRebind", "free",
VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
Libs::LibKernel::Memory::FreeGuestMemory(base, size),
"guest-owner free failed");
std::printf("[host] %-32s ok\n", "StorageTextureGpuOwnedRebind");
}
#endif
@@ -18029,6 +18080,11 @@ int main(int argc, char **argv) {
vulkan.CheckSchedulerTimeline();
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--mapped-range-only") == 0) {
VulkanHarness vulkan;
vulkan.CheckGpuMappedRangeLifecycle();
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--stream-buffer-only") == 0) {
VulkanHarness vulkan;
vulkan.CheckStreamBufferRing();
@@ -18184,6 +18240,7 @@ int main(int argc, char **argv) {
CheckEmbeddedFetchLaneSpill();
CheckPs5GameExampleImageClearRuntimeShape();
vulkan.CheckSchedulerTimeline();
vulkan.CheckGpuMappedRangeLifecycle();
vulkan.CheckStreamBufferRing();
vulkan.CheckCommandPoolGrowth();
vulkan.CheckGpuTilerCpuParity();
tests/VirtualMemoryAllocationTests.cpp
+716 -66
View File
@@ -1,15 +1,21 @@
#include "common/commonSubsystem.h"
#include "common/emulatorConfig.h"
#include "common/file.h"
#include "common/logging/log.h"
#include "common/subsystems.h"
#include "common/threads.h"
#include "common/virtualMemory.h"
#include "kernel/memory.h"
#include "kernel/pthread.h"
#include "libs/errno.h"
#include "loader/runtimeLinker.h"
#include "loader/systemContent.h"
#include <cinttypes>
#include <cstdint>
#include <cstdio>
#include <cstring>
#include <filesystem>
#include <string>
namespace {
@@ -18,11 +24,16 @@ using Libs::LibKernel::Memory::VirtualQueryInfo;
// Prospero ABI?
constexpr uint64_t SceKernelPageSize = 0x4000;
constexpr uint64_t SceKernelTotalPhysicalSize = 13824ull * 1024ull * 1024ull;
constexpr uint64_t TestFlexibleMemorySize = 3072ull * 1024ull * 1024ull;
constexpr int SceKernelProtCpuRead = 0x01;
constexpr int SceKernelProtCpuRw = 0x02;
constexpr int SceKernelProtCpuExec = 0x04;
constexpr int SceKernelMapFixed = 0x10;
constexpr int SceKernelMapNoOverwrite = 0x80;
constexpr int SceKernelMapDmemCompat = 0x400;
constexpr int SceKernelMapNoCoalesce = 0x400000;
constexpr int SceKernelMapAligned64Kb = 16 << 24;
constexpr int SceKernelVqFindNext = 1;
constexpr int SceKernelMtypeC = 11;
constexpr uint64_t SceKernelDirectMemoryStart = 0;
@@ -94,6 +105,29 @@ void InitSubsystems() {
Config::Load(options);
slist->Add(log, {core, config});
Check("InitSubsystems", slist->InitAll(false), "failed to initialize logging subsystem");
const auto param_json =
std::filesystem::temp_directory_path() /
("kyty_virtual_memory_" +
std::to_string(reinterpret_cast<uintptr_t>(&initialized)) + ".json");
constexpr char json[] = R"({"kernel":{"flexibleMemorySize":3221225472}})";
Common::File param_file;
Check("InitSubsystems", param_file.Create(param_json), "failed to create temporary param.json");
uint32_t bytes_written = 0;
param_file.Write(json, sizeof(json) - 1, &bytes_written);
param_file.Close();
Check("InitSubsystems", bytes_written == sizeof(json) - 1,
"failed to write temporary param.json");
Loader::SystemContentLoadParamSfo(param_json);
const auto flexible_memory_size = Loader::SystemContentGetFlexibleMemorySize();
Check("InitSubsystems", Common::File::DeleteFile(param_json),
"failed to remove temporary param.json");
Check("InitSubsystems", flexible_memory_size == TestFlexibleMemorySize,
"failed to read flexible memory size from param.json");
Libs::LibKernel::Memory::SetFlexibleMemorySize(flexible_memory_size);
slist->Add(memory, {core, log, thread});
Check("InitSubsystems", slist->InitAll(false), "failed to initialize memory subsystem");
@@ -131,6 +165,13 @@ size_t AvailableFlexibleMemory(const char* test) {
return size;
}
size_t ConfiguredFlexibleMemory(const char* test) {
size_t size = 0;
CheckOk(test, Libs::LibKernel::Memory::KernelConfiguredFlexibleMemorySize(&size),
"KernelConfiguredFlexibleMemorySize");
return size;
}
uint64_t MapNamedFlexible(const char* test, uint64_t size, int prot, const char* name) {
void* addr = nullptr;
const int ret =
@@ -190,6 +231,365 @@ void TestProsperoArgumentAndInfoSizeContracts() {
std::printf("[host] %-48s ok\n", test);
}
void TestGuestAddressSpaceOwnsReservationsBeforeBacking() {
const char* test = "GuestAddressSpaceOwnsReservationsBeforeBacking";
void* addr = nullptr;
Check(test, Libs::LibKernel::Memory::TestGuestBackingOutsideAddressSpace(),
"boot-time shared backing alias overlaps an owned guest interval");
CheckOk(test,
Libs::LibKernel::Memory::KernelReserveVirtualRange(&addr, SceKernelPageSize, 0,
SceKernelPageSize),
"KernelReserveVirtualRange");
const auto base = reinterpret_cast<uint64_t>(addr);
Check(test, Libs::LibKernel::Memory::TestGuestAddressRangeIsOwned(base, SceKernelPageSize),
"guest reservation was allocated outside the early owner");
Check(test, Libs::LibKernel::Memory::TestPlaceholderRangeIsFree(base, SceKernelPageSize),
"semantic reservation replaced the owner's placeholder");
Check(test,
Libs::LibKernel::Memory::ProtectGuestHostMemory(
base, SceKernelPageSize, Common::VirtualMemory::Mode::NoAccess),
"owner rejected a sparse placeholder protection no-op");
CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(base, SceKernelPageSize), "KernelMunmap");
Check(test, Libs::LibKernel::Memory::TestPlaceholderRangeIsFree(base, SceKernelPageSize),
"released semantic reservation escaped owner control");
std::printf("[host] %-48s ok\n", test);
}
void TestGuestAddressSpaceHasNoFixedFallback() {
const char* test = "GuestAddressSpaceHasNoFixedFallback";
const auto unowned_address = reinterpret_cast<void*>(0x10000);
void* addr = unowned_address;
CheckFailed(test,
Libs::LibKernel::Memory::KernelReserveVirtualRange(
&addr, SceKernelPageSize, SceKernelMapFixed | SceKernelMapNoOverwrite,
SceKernelPageSize),
"KernelReserveVirtualRange(unowned fixed address)");
Check(test, reinterpret_cast<uint64_t>(addr) == 0x10000,
"failed fixed reservation unexpectedly moved");
addr = unowned_address;
CheckFailed(test,
Libs::LibKernel::Memory::KernelMapNamedFlexibleMemory(
&addr, SceKernelPageSize, SceKernelProtCpuRw,
SceKernelMapFixed | SceKernelMapNoOverwrite, "unowned_flexible"),
"KernelMapNamedFlexibleMemory(unowned fixed address)");
int64_t phys_addr = -1;
CheckOk(test,
Libs::LibKernel::Memory::KernelAllocateDirectMemory(
0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(), SceKernelPageSize,
SceKernelPageSize, SceKernelMtypeC, &phys_addr),
"KernelAllocateDirectMemory");
addr = unowned_address;
CheckFailed(test,
Libs::LibKernel::Memory::KernelMapNamedDirectMemory(
&addr, SceKernelPageSize, SceKernelProtCpuRw,
SceKernelMapFixed | SceKernelMapNoOverwrite, phys_addr, SceKernelPageSize,
"unowned_direct"),
"KernelMapNamedDirectMemory(unowned fixed address)");
CheckOk(test,
Libs::LibKernel::Memory::KernelCheckedReleaseDirectMemory(phys_addr, SceKernelPageSize),
"KernelCheckedReleaseDirectMemory");
std::printf("[host] %-48s ok\n", test);
}
void TestGuestFreeRangeSearchDoesNotUnderflow() {
const char* test = "GuestFreeRangeSearchDoesNotUnderflow";
Check(test, Libs::LibKernel::Memory::TestGuestFreeRangeBounds(),
"free-range containment accepted a candidate beyond the range end");
std::printf("[host] %-48s ok\n", test);
}
void TestFlexibleMemoryCapacityIsBootFixed() {
const char* test = "FlexibleMemoryCapacityIsBootFixed";
const auto configured = ConfiguredFlexibleMemory(test);
const auto baseline = AvailableFlexibleMemory(test);
const auto backing = Libs::LibKernel::Memory::TestGuestBackingSize();
Check(test, configured == TestFlexibleMemorySize,
"boot flexible pool did not use the param.json value");
Check(test, configured == baseline, "boot flexible pool did not start at configured capacity");
Check(test, backing == SceKernelTotalPhysicalSize,
"boot backing is not the single 13.5 GiB physical file");
Check(test, backing == Libs::LibKernel::Memory::KernelGetDirectMemorySize() + configured,
"direct and flexible regions do not partition the boot backing");
const auto address =
MapNamedFlexible(test, SceKernelPageSize, SceKernelProtCpuRw, "boot_fixed_flexible");
Check(test, ConfiguredFlexibleMemory(test) == configured,
"configured flexible capacity changed after allocation");
Check(test, Libs::LibKernel::Memory::TestGuestBackingSize() == backing,
"shared backing size changed after allocation");
Check(test, AvailableFlexibleMemory(test) == baseline - SceKernelPageSize,
"flexible allocation did not consume the boot-time pool");
CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(address, SceKernelPageSize),
"KernelMunmap");
Check(test, ConfiguredFlexibleMemory(test) == configured,
"configured flexible capacity changed after release");
Check(test, AvailableFlexibleMemory(test) == baseline,
"flexible release did not restore the boot-time pool");
std::printf("[host] %-48s ok\n", test);
}
void TestFlexibleMemoryUsesSharedBacking() {
const char* test = "FlexibleMemoryUsesSharedBacking";
const auto baseline = AvailableFlexibleMemory(test);
void* address = nullptr;
CheckOk(test,
Libs::LibKernel::Memory::KernelMapNamedFlexibleMemory(
&address, SceKernelPageSize * 2, SceKernelProtCpuRw, 0, "shared_flexible"),
"KernelMapNamedFlexibleMemory");
const auto base = reinterpret_cast<uint64_t>(address);
Check(test, Libs::LibKernel::Memory::TestGuestAddressRangeIsOwned(base, SceKernelPageSize * 2),
"flexible mapping escaped the guest owner");
constexpr uint64_t first_value = 0x464c45584241434bull; // "FLEXBACK"
constexpr uint64_t second_value = 0x534841524544464cull; // "SHAREDFL"
*reinterpret_cast<uint64_t*>(base) = first_value;
uint64_t value = 0;
Check(test, Libs::LibKernel::Memory::TryReadBacking(base, &value, sizeof(value)),
"TryReadBacking did not resolve flexible memory");
Check(test, value == first_value, "backing did not observe a flexible-memory CPU write");
Check(test,
Libs::LibKernel::Memory::TryWriteBacking(base + SceKernelPageSize, &second_value,
sizeof(second_value)),
"TryWriteBacking did not resolve flexible memory");
Check(test, *reinterpret_cast<uint64_t*>(base + SceKernelPageSize) == second_value,
"flexible-memory view did not observe a backing write");
CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(base, SceKernelPageSize * 2),
"KernelMunmap");
Check(test, AvailableFlexibleMemory(test) == baseline,
"flexible backing offsets were not returned to the boot-time pool");
Check(test, !Libs::LibKernel::Memory::TryReadBacking(base, &value, sizeof(value)),
"unmapped flexible memory remained registered in the backing owner");
std::printf("[host] %-48s ok\n", test);
}
void TestFlexibleDmemCompatAndAlignmentFlags() {
const char* test = "FlexibleDmemCompatAndAlignmentFlags";
const auto baseline = AvailableFlexibleMemory(test);
void* address = nullptr;
CheckOk(test,
Libs::LibKernel::Memory::KernelMapNamedFlexibleMemory(
&address, SceKernelPageSize, SceKernelProtCpuRw,
SceKernelMapDmemCompat | SceKernelMapAligned64Kb, "dmem_compat"),
"KernelMapNamedFlexibleMemory(DMEM_COMPAT|ALIGNED_64KB)");
const auto base = reinterpret_cast<uint64_t>(address);
Check(test, (base & (0x10000 - 1u)) == 0, "SDK alignment flag was not honored");
const auto info = Query(test, base);
Check(test, info.is_flexible == 1 && info.is_stack == 0,
"SCE_KERNEL_MAP_DMEM_COMPAT was misclassified as MAP_STACK");
Check(test, AvailableFlexibleMemory(test) + SceKernelPageSize == baseline,
"DMEM_COMPAT mapping did not consume boot-time flexible backing");
void* stack_start = reinterpret_cast<void*>(UINT64_MAX);
void* stack_end = reinterpret_cast<void*>(UINT64_MAX);
CheckOk(test,
Libs::LibKernel::Memory::KernelIsStack(reinterpret_cast<void*>(base), &stack_start,
&stack_end),
"KernelIsStack");
Check(test, stack_start == nullptr && stack_end == nullptr,
"DMEM_COMPAT flexible mapping was reported as a stack");
CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(base, SceKernelPageSize),
"KernelMunmap");
Check(test, AvailableFlexibleMemory(test) == baseline,
"DMEM_COMPAT cleanup did not restore flexible capacity");
void* invalid = nullptr;
CheckFailed(test,
Libs::LibKernel::Memory::KernelMapNamedFlexibleMemory(
&invalid, SceKernelPageSize, SceKernelProtCpuRw, 0x8000, "non_sdk_flag"),
"KernelMapNamedFlexibleMemory(non-SDK flag)");
std::printf("[host] %-48s ok\n", test);
}
void TestFlexibleNoCoalescePreservesBoundaries() {
const char* test = "FlexibleNoCoalescePreservesBoundaries";
const auto baseline = AvailableFlexibleMemory(test);
void* reserve = nullptr;
CheckOk(test,
Libs::LibKernel::Memory::KernelReserveVirtualRange(
&reserve, SceKernelPageSize * 2, 0, SceKernelPageSize),
"KernelReserveVirtualRange");
const auto base = reinterpret_cast<uint64_t>(reserve);
void* left = reinterpret_cast<void*>(base);
CheckOk(test,
Libs::LibKernel::Memory::KernelMapNamedFlexibleMemory(
&left, SceKernelPageSize, SceKernelProtCpuRw,
SceKernelMapFixed | SceKernelMapNoCoalesce, "no_coalesce"),
"KernelMapNamedFlexibleMemory(left)");
void* right = reinterpret_cast<void*>(base + SceKernelPageSize);
CheckOk(test,
Libs::LibKernel::Memory::KernelMapNamedFlexibleMemory(
&right, SceKernelPageSize, SceKernelProtCpuRw,
SceKernelMapFixed | SceKernelMapNoCoalesce, "no_coalesce"),
"KernelMapNamedFlexibleMemory(right)");
ExpectRange(test, Query(test, base), base, base + SceKernelPageSize, SceKernelProtCpuRw, 1, 0,
0, 1, "no_coalesce");
ExpectRange(test, Query(test, base + SceKernelPageSize), base + SceKernelPageSize,
base + SceKernelPageSize * 2, SceKernelProtCpuRw, 1, 0, 0, 1, "no_coalesce");
CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(base, SceKernelPageSize * 2),
"KernelMunmap");
Check(test, AvailableFlexibleMemory(test) == baseline,
"NO_COALESCE cleanup did not restore flexible capacity");
std::printf("[host] %-48s ok\n", test);
}
void TestFlexibleMemoryReuseIsZeroFilled() {
const char* test = "FlexibleMemoryReuseIsZeroFilled";
const auto baseline = AvailableFlexibleMemory(test);
const auto first =
MapNamedFlexible(test, SceKernelPageSize, SceKernelProtCpuRw, "flexible_zero_source");
std::memset(reinterpret_cast<void*>(first), 0xa5, SceKernelPageSize);
CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(first, SceKernelPageSize),
"KernelMunmap(source)");
const auto reused =
MapNamedFlexible(test, SceKernelPageSize, SceKernelProtCpuRw, "flexible_zero_reuse");
const auto* bytes = reinterpret_cast<const uint8_t*>(reused);
Check(test,
std::all_of(bytes, bytes + SceKernelPageSize, [](uint8_t value) { return value == 0; }),
"reused flexible backing exposed stale bytes");
CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(reused, SceKernelPageSize),
"KernelMunmap(reuse)");
Check(test, AvailableFlexibleMemory(test) == baseline,
"zero-fill test leaked flexible backing capacity");
std::printf("[host] %-48s ok\n", test);
}
void TestGuestStackUsesPrivateOwnerMemoryAndCache() {
const char* test = "GuestStackUsesPrivateOwnerMemoryAndCache";
const auto baseline = AvailableFlexibleMemory(test);
uint64_t first = 0;
uint64_t second = 0;
uint64_t map_size = 0;
Check(test, Libs::LibKernel::TestGuestStackOwnerLifecycle(&first, &second, &map_size),
"guest stack owner lifecycle failed");
Check(test, first != 0 && first == second, "guest stack cache did not reuse its owner mapping");
Check(test, map_size != 0 && (map_size & (SceKernelPageSize - 1u)) == 0,
"guest stack mapping is not 16 KiB aligned");
Check(test, AvailableFlexibleMemory(test) == baseline,
"private guest stack changed flexible backing capacity");
std::printf("[host] %-48s ok\n", test);
}
void TestMainEntryUsesGuestStackAndDisablesHostChecks() {
const char* test = "MainEntryUsesGuestStackAndDisablesHostChecks";
Check(test, Loader::TestMainEntryUsesGuestStack(),
"main-entry stack switch did not preserve the guest/host stack invariants");
std::printf("[host] %-48s ok\n", test);
}
void TestFragmentedBackingUnmapRollback() {
const char* test = "FragmentedBackingUnmapRollback";
const auto baseline = AvailableFlexibleMemory(test);
const auto left =
MapNamedFlexible(test, SceKernelPageSize, SceKernelProtCpuRw, "backing_hole_left");
const auto blocker =
MapNamedFlexible(test, SceKernelPageSize, SceKernelProtCpuRw, "backing_blocker");
const auto right =
MapNamedFlexible(test, SceKernelPageSize, SceKernelProtCpuRw, "backing_hole_right");
CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(left, SceKernelPageSize),
"KernelMunmap(left hole)");
CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(right, SceKernelPageSize),
"KernelMunmap(right hole)");
const auto fragmented =
MapNamedFlexible(test, SceKernelPageSize * 2, SceKernelProtCpuRw, "fragmented_backing");
auto* first_word = reinterpret_cast<uint64_t*>(fragmented);
auto* last_word =
reinterpret_cast<uint64_t*>(fragmented + SceKernelPageSize * 2 - sizeof(uint64_t));
*first_word = 0x465241474c454654ull; // "FRAGLEFT"
*last_word = 0x4652414752474854ull; // "FRAGRGHT"
Libs::LibKernel::Memory::TestFailGuestBackingStoreUnmapAfter(1);
CheckFailed(test, Libs::LibKernel::Memory::KernelMunmap(fragmented, SceKernelPageSize * 2),
"KernelMunmap(injected second-view failure)");
ExpectRange(test, Query(test, fragmented), fragmented, fragmented + SceKernelPageSize * 2,
SceKernelProtCpuRw, 1, 0, 0, 1, "fragmented_backing");
Check(test, *first_word == 0x465241474c454654ull && *last_word == 0x4652414752474854ull,
"transactional backing-unmap rollback lost mapped contents");
CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(fragmented, SceKernelPageSize * 2),
"KernelMunmap(retry)");
CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(blocker, SceKernelPageSize),
"KernelMunmap(blocker)");
Check(test, AvailableFlexibleMemory(test) == baseline,
"fragmented backing rollback test leaked flexible capacity");
std::printf("[host] %-48s ok\n", test);
}
void TestRuntimeMemoryOwnerLifecycle() {
const char* test = "RuntimeMemoryOwnerLifecycle";
Check(test,
Libs::LibKernel::Memory::AllocateRuntimeMemory(0x10000, SceKernelPageSize,
Common::VirtualMemory::Mode::ReadWrite,
"runtime_outside_owner", true) == 0,
"fixed runtime allocation escaped the guest owner");
const auto base = Libs::LibKernel::Memory::AllocateRuntimeMemory(
0, SceKernelPageSize * 2, Common::VirtualMemory::Mode::ReadWrite, "runtime_lifecycle");
Check(test, base != 0, "runtime allocation failed");
Check(test, Libs::LibKernel::Memory::TestGuestAddressRangeIsOwned(base, SceKernelPageSize * 2),
"runtime allocation is outside the owner");
*reinterpret_cast<uint64_t*>(base) = 0x52554e54494d454full; // "RUNTIMEO"
Check(test,
Libs::LibKernel::Memory::ProtectGuestMemory(base, SceKernelPageSize,
Common::VirtualMemory::Mode::Read),
"runtime protection failed");
Check(test, Libs::LibKernel::Memory::FreeGuestMemory(base, SceKernelPageSize * 2),
"runtime free failed");
Check(test, Libs::LibKernel::Memory::TestPlaceholderRangeIsFree(base, SceKernelPageSize * 2),
"runtime free did not restore the owner placeholder");
const auto reused = Libs::LibKernel::Memory::AllocateRuntimeMemory(
base, SceKernelPageSize * 2, Common::VirtualMemory::Mode::ReadWrite, "runtime_reuse", true);
Check(test, reused == base, "fixed runtime allocation did not reuse the owner placeholder");
Check(test, Libs::LibKernel::Memory::FreeGuestMemory(reused, SceKernelPageSize * 2),
"reused runtime free failed");
const auto adjacent_first = Libs::LibKernel::Memory::AllocateRuntimeMemory(
0, SceKernelPageSize, Common::VirtualMemory::Mode::ReadWrite, "runtime_adjacent_first");
Check(test, adjacent_first != 0, "first adjacent runtime allocation failed");
const auto adjacent_second = Libs::LibKernel::Memory::AllocateRuntimeMemory(
adjacent_first + SceKernelPageSize, SceKernelPageSize,
Common::VirtualMemory::Mode::ReadWrite, "runtime_adjacent_second", true);
Check(test, adjacent_second == adjacent_first + SceKernelPageSize,
"second adjacent runtime allocation failed");
Check(test,
Libs::LibKernel::Memory::FreeGuestMemory(adjacent_first, SceKernelPageSize * 2),
"combined adjacent runtime free failed");
Check(test,
Libs::LibKernel::Memory::TestPlaceholderRangeIsFree(adjacent_first,
SceKernelPageSize * 2),
"combined adjacent runtime free did not restore one owner placeholder");
std::printf("[host] %-48s ok\n", test);
}
void TestFlexibleMapQueryAndWholeMunmap() {
const char* test = "FlexibleMapQueryAndWholeMunmap";
const auto baseline = AvailableFlexibleMemory(test);
@@ -341,9 +741,11 @@ void TestDirectMapQueryOffsetAndPartialMunmap() {
&addr, SceKernelPageSize * 4, SceKernelProtCpuRw, 0, phys_addr, SceKernelPageSize,
"prospero_direct"),
"KernelMapNamedDirectMemory");
const auto base = reinterpret_cast<uint64_t>(addr);
const auto phys = static_cast<uint64_t>(phys_addr);
void* alias = nullptr;
const auto base = reinterpret_cast<uint64_t>(addr);
const auto phys = static_cast<uint64_t>(phys_addr);
Check(test, Libs::LibKernel::Memory::TestGuestAddressRangeIsOwned(base, SceKernelPageSize * 4),
"direct mapping escaped the guest owner");
void* alias = nullptr;
CheckOk(test,
Libs::LibKernel::Memory::KernelMapNamedDirectMemory(
&alias, SceKernelPageSize * 4, SceKernelProtCpuRw, 0, phys_addr, SceKernelPageSize,
@@ -430,6 +832,192 @@ void TestDirectMapQueryOffsetAndPartialMunmap() {
std::printf("[host] %-48s ok\n", test);
}
void TestDirectPartialProtectUnmapPreservesNeighbors() {
const char* test = "DirectPartialProtectUnmapPreservesNeighbors";
const auto size = SceKernelPageSize * 3;
int64_t phys_addr = 0;
CheckOk(test,
Libs::LibKernel::Memory::KernelAllocateDirectMemory(
0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(), size, SceKernelPageSize,
SceKernelMtypeC, &phys_addr),
"KernelAllocateDirectMemory");
void* address = nullptr;
CheckOk(test,
Libs::LibKernel::Memory::KernelMapNamedDirectMemory(&address, size, SceKernelProtCpuRw,
0, phys_addr, SceKernelPageSize,
"partial_protect_direct"),
"KernelMapNamedDirectMemory");
const auto base = reinterpret_cast<uint64_t>(address);
CheckOk(
test,
Libs::LibKernel::Memory::KernelMprotect(reinterpret_cast<void*>(base + SceKernelPageSize),
SceKernelPageSize, SceKernelProtCpuRead),
"KernelMprotect(middle)");
Check(test,
Libs::LibKernel::Memory::ProtectGuestHostMemory(
base, size, Common::VirtualMemory::Mode::Read),
"owner could not protect fragmented backing views");
Check(test,
Libs::LibKernel::Memory::ProtectGuestHostMemory(
base, size, Common::VirtualMemory::Mode::ReadWrite),
"owner could not restore fragmented backing views");
CheckOk(test,
Libs::LibKernel::Memory::KernelMunmap(base + SceKernelPageSize, SceKernelPageSize),
"KernelMunmap(middle)");
Common::VirtualMemory::Mode old_left {};
Common::VirtualMemory::Mode old_right {};
Check(test,
Common::VirtualMemory::Protect(base, SceKernelPageSize,
Common::VirtualMemory::Mode::ReadWrite, &old_left),
"could not inspect left-page protection");
Check(test,
Common::VirtualMemory::Protect(base + SceKernelPageSize * 2, SceKernelPageSize,
Common::VirtualMemory::Mode::ReadWrite, &old_right),
"could not inspect right-page protection");
Check(test, old_left == Common::VirtualMemory::Mode::ReadWrite,
"partial unmap changed the left neighbor protection");
Check(test, old_right == Common::VirtualMemory::Mode::ReadWrite,
"partial unmap changed the right neighbor protection");
*reinterpret_cast<uint64_t*>(base) = 0x4c45465450524f54ull; // "LEFTPROT"
*reinterpret_cast<uint64_t*>(base + SceKernelPageSize * 2) =
0x5247485450524f54ull; // "RGHTPROT"
CheckOk(test, Libs::LibKernel::Memory::KernelReleaseDirectMemory(phys_addr, size),
"KernelReleaseDirectMemory");
ExpectUnmapped(test, base);
ExpectUnmapped(test, base + SceKernelPageSize * 2);
std::printf("[host] %-48s ok\n", test);
}
void TestDirectMapValidationBeforeOwnerMutation() {
const char* test = "DirectMapValidationBeforeOwnerMutation";
int64_t invalid = -1;
CheckFailed(test,
Libs::LibKernel::Memory::KernelAllocateDirectMemory(
0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(), SceKernelPageSize + 1,
SceKernelPageSize, SceKernelMtypeC, &invalid),
"KernelAllocateDirectMemory(unaligned size)");
Check(test, invalid == -1, "invalid direct allocation changed the output address");
CheckFailed(test,
Libs::LibKernel::Memory::KernelAllocateDirectMemory(
0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(), SceKernelPageSize,
0x1000, SceKernelMtypeC, &invalid),
"KernelAllocateDirectMemory(sub-page alignment)");
Check(test, invalid == -1, "invalid alignment changed the output address");
int64_t phys_addr = 0;
CheckOk(test,
Libs::LibKernel::Memory::KernelAllocateDirectMemory(
0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(), SceKernelPageSize * 2,
SceKernelPageSize, SceKernelMtypeC, &phys_addr),
"KernelAllocateDirectMemory");
auto expect_invalid = [&](size_t len, int prot, int flags, int64_t phys, size_t alignment,
const char* action) {
void* address = nullptr;
CheckFailed(test,
Libs::LibKernel::Memory::KernelMapDirectMemory(&address, len, prot, flags, phys,
alignment),
action);
Check(test, address == nullptr, "invalid direct map changed the output address");
};
expect_invalid(SceKernelPageSize + 1, SceKernelProtCpuRw, 0, phys_addr, SceKernelPageSize,
"KernelMapDirectMemory(unaligned size)");
expect_invalid(SceKernelPageSize, SceKernelProtCpuRw, 0, phys_addr + 1, SceKernelPageSize,
"KernelMapDirectMemory(unaligned physical address)");
expect_invalid(SceKernelPageSize, SceKernelProtCpuExec, 0, phys_addr, SceKernelPageSize,
"KernelMapDirectMemory(executable)");
void* aligned = nullptr;
CheckOk(test,
Libs::LibKernel::Memory::KernelMapDirectMemory(
&aligned, SceKernelPageSize, SceKernelProtCpuRw, 0, phys_addr, 0xc000),
"KernelMapDirectMemory(16K-multiple alignment)");
Check(test, reinterpret_cast<uint64_t>(aligned) % 0xc000 == 0,
"non-power-of-two 16K alignment was not honored");
CheckOk(test,
Libs::LibKernel::Memory::KernelMunmap(reinterpret_cast<uint64_t>(aligned),
SceKernelPageSize),
"KernelMunmap(16K-multiple alignment)");
void* ignored_flag = nullptr;
CheckOk(test,
Libs::LibKernel::Memory::KernelMapDirectMemory(&ignored_flag, SceKernelPageSize,
SceKernelProtCpuRw, 0x08, phys_addr,
SceKernelPageSize),
"KernelMapDirectMemory(ignored flag)");
CheckOk(test,
Libs::LibKernel::Memory::KernelMunmap(reinterpret_cast<uint64_t>(ignored_flag),
SceKernelPageSize),
"KernelMunmap(ignored flag)");
CheckOk(test,
Libs::LibKernel::Memory::KernelReleaseDirectMemory(phys_addr, SceKernelPageSize * 2),
"KernelReleaseDirectMemory");
std::printf("[host] %-48s ok\n", test);
}
void TestDirectReleaseRollbackRestoresOwnerMapping() {
const char* test = "DirectReleaseRollbackRestoresOwnerMapping";
int64_t phys_addr = 0;
CheckOk(test,
Libs::LibKernel::Memory::KernelAllocateDirectMemory(
0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(), SceKernelPageSize,
SceKernelPageSize, SceKernelMtypeC, &phys_addr),
"KernelAllocateDirectMemory");
void* address = nullptr;
CheckOk(test,
Libs::LibKernel::Memory::KernelMapNamedDirectMemory(
&address, SceKernelPageSize, SceKernelProtCpuRw, 0, phys_addr, SceKernelPageSize,
"release_rollback"),
"KernelMapNamedDirectMemory");
const auto base = reinterpret_cast<uint64_t>(address);
*reinterpret_cast<uint64_t*>(base) = 0x52454c524f4c4c42ull; // "RELROLLB"
Libs::LibKernel::Memory::TestFailNextPhysicalMemoryUnmap();
CheckFailed(
test,
Libs::LibKernel::Memory::KernelCheckedReleaseDirectMemory(phys_addr, SceKernelPageSize),
"KernelCheckedReleaseDirectMemory(injected failure)");
ExpectRange(test, Query(test, base), base, base + SceKernelPageSize, SceKernelProtCpuRw, 0, 1,
0, 1, "release_rollback", static_cast<uint64_t>(phys_addr));
Check(test, *reinterpret_cast<uint64_t*>(base) == 0x52454c524f4c4c42ull,
"release rollback lost the shared-backing contents");
CheckOk(test,
Libs::LibKernel::Memory::KernelCheckedReleaseDirectMemory(phys_addr, SceKernelPageSize),
"KernelCheckedReleaseDirectMemory(retry)");
ExpectUnmapped(test, base);
std::printf("[host] %-48s ok\n", test);
}
void TestDirectReleaseContracts() {
const char* test = "DirectReleaseContracts";
CheckOk(test, Libs::LibKernel::Memory::KernelReleaseDirectMemory(0, 0),
"KernelReleaseDirectMemory(zero length)");
CheckOk(test, Libs::LibKernel::Memory::KernelCheckedReleaseDirectMemory(0, 0),
"KernelCheckedReleaseDirectMemory(zero length)");
CheckFailed(test, Libs::LibKernel::Memory::KernelReleaseDirectMemory(1, SceKernelPageSize),
"KernelReleaseDirectMemory(unaligned start)");
CheckFailed(test, Libs::LibKernel::Memory::KernelReleaseDirectMemory(0, SceKernelPageSize + 1),
"KernelReleaseDirectMemory(unaligned size)");
const auto free_offset = static_cast<int64_t>(
Libs::LibKernel::Memory::KernelGetDirectMemorySize() - SceKernelPageSize);
CheckOk(test,
Libs::LibKernel::Memory::KernelReleaseDirectMemory(free_offset, SceKernelPageSize),
"KernelReleaseDirectMemory(unallocated range)");
Check(test,
Libs::LibKernel::Memory::KernelCheckedReleaseDirectMemory(
free_offset, SceKernelPageSize) == Libs::LibKernel::KERNEL_ERROR_ENOENT,
"checked release did not report an unallocated range");
std::printf("[host] %-48s ok\n", test);
}
void TestReleasedReserveCanBeReused() {
const char* test = "ReleasedReserveCanBeReused";
void* addr = nullptr;
@@ -478,21 +1066,23 @@ void TestMunmapAcrossAdjacentFlexibleMappings() {
"KernelMapNamedFlexibleMemory(right)");
Check(test,
Libs::LibKernel::Memory::ClampRangeSize(base + SceKernelPageSize - 0x100, 0x200) ==
0x200,
Libs::LibKernel::Memory::ClampRangeSize(base + SceKernelPageSize - 0x100, 0x200) == 0x200,
"ClampRangeSize did not cross adjacent committed mappings");
Check(test,
Libs::LibKernel::Memory::ProtectGuestHostMemory(
base, SceKernelPageSize * 2, Common::VirtualMemory::Mode::Read),
"owner could not protect adjacent backing mappings");
Check(test,
Libs::LibKernel::Memory::ProtectGuestHostMemory(
base, SceKernelPageSize * 2, Common::VirtualMemory::Mode::ReadWrite),
"owner could not restore adjacent backing mappings");
CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(base, SceKernelPageSize * 2),
"KernelMunmap(adjacent mappings)");
Check(test, AvailableFlexibleMemory(test) == baseline,
"multi-range unmap leaked flexible-memory budget");
ExpectRange(test, Query(test, base), base, base + SceKernelPageSize, 0, 0, 0, 0, 0,
"adjacent_left");
ExpectRange(test, Query(test, base + SceKernelPageSize), base + SceKernelPageSize,
base + SceKernelPageSize * 2, 0, 0, 0, 0, 0, "adjacent_right");
CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(base, SceKernelPageSize * 2),
"KernelMunmap(restored reserve)");
ExpectUnmapped(test, base);
ExpectUnmapped(test, base + SceKernelPageSize);
std::printf("[host] %-48s ok\n", test);
}
@@ -548,6 +1138,52 @@ void TestNonzeroDirectOffsetAliasesSharedBacking() {
std::printf("[host] %-48s ok\n", test);
}
void TestDirectMapAcrossContiguousAllocations() {
const char* test = "DirectMapAcrossContiguousAllocations";
const auto end = Libs::LibKernel::Memory::KernelGetDirectMemorySize();
int64_t first = 0;
int64_t second = 0;
CheckOk(test,
Libs::LibKernel::Memory::KernelAllocateDirectMemory(
0, end, SceKernelPageSize, SceKernelPageSize, SceKernelMtypeC, &first),
"KernelAllocateDirectMemory(first)");
CheckOk(test,
Libs::LibKernel::Memory::KernelAllocateDirectMemory(
0, end, SceKernelPageSize, SceKernelPageSize, SceKernelMtypeC, &second),
"KernelAllocateDirectMemory(second)");
Check(test, second == first + static_cast<int64_t>(SceKernelPageSize),
"test allocations are not physically contiguous");
void* mapping = nullptr;
CheckOk(test,
Libs::LibKernel::Memory::KernelMapNamedDirectMemory(
&mapping, SceKernelPageSize * 2, SceKernelProtCpuRw, 0, first, SceKernelPageSize,
"contiguous_allocations"),
"KernelMapNamedDirectMemory");
auto* words = reinterpret_cast<uint64_t*>(mapping);
words[0] = 0x434f4e5449474c46ull; // "CONTIGLF"
*reinterpret_cast<uint64_t*>(reinterpret_cast<uint64_t>(mapping) + SceKernelPageSize) =
0x434f4e5449475254ull; // "CONTIGRT"
CheckOk(test,
Libs::LibKernel::Memory::KernelCheckedReleaseDirectMemory(first, SceKernelPageSize * 2),
"KernelCheckedReleaseDirectMemory(contiguous span)");
ExpectUnmapped(test, reinterpret_cast<uint64_t>(mapping));
int64_t reclaimed = -1;
CheckOk(test,
Libs::LibKernel::Memory::KernelAllocateDirectMemory(
0, end, SceKernelPageSize * 2, SceKernelPageSize, SceKernelMtypeC, &reclaimed),
"KernelAllocateDirectMemory(reclaimed)");
Check(test, reclaimed == first, "released contiguous span was not coalesced");
CheckOk(
test,
Libs::LibKernel::Memory::KernelCheckedReleaseDirectMemory(reclaimed, SceKernelPageSize * 2),
"KernelCheckedReleaseDirectMemory(reclaimed)");
std::printf("[host] %-48s ok\n", test);
}
void TestDirectPhysicalFreeRangeReuseAndCoalescing() {
const char* test = "DirectPhysicalFreeRangeReuseAndCoalescing";
const auto end = Libs::LibKernel::Memory::KernelGetDirectMemorySize();
@@ -933,38 +1569,6 @@ void TestFixedReserveRollbackConsumesRestoredPlaceholder() {
std::printf("[host] %-48s ok\n", test);
}
void TestFixedReserveRollbackRestoresDecommittedHostPages() {
const char* test = "FixedReserveRollbackRestoresDecommittedHostPages";
constexpr uint64_t size = SceKernelPageSize * 3;
void* mapped = nullptr;
CheckOk(test,
Libs::LibKernel::Memory::KernelMapNamedFlexibleMemory(&mapped, size, SceKernelProtCpuRw,
0, "host_reserve_rollback"),
"KernelMapNamedFlexibleMemory");
const auto base = reinterpret_cast<uint64_t>(mapped);
*reinterpret_cast<uint64_t*>(base) = 0x4b595459484f5354ull; // "KYTYHOST"
*reinterpret_cast<uint64_t*>(base + SceKernelPageSize * 2) =
0x4b5954595441494cull; // "KYTYTAIL"
Libs::LibKernel::Memory::TestFailHostReservationAfter(1);
void* replacement = mapped;
CheckFailed(
test,
Libs::LibKernel::Memory::KernelReserveVirtualRange(
&replacement, size, SceKernelMapFixed | SceKernelMapNoCoalesce, SceKernelPageSize),
"KernelReserveVirtualRange(partial host reservation)");
Check(test, *reinterpret_cast<uint64_t*>(base) == 0x4b595459484f5354ull,
"rollback did not restore the first flexible page");
Check(test, *reinterpret_cast<uint64_t*>(base + SceKernelPageSize * 2) == 0x4b5954595441494cull,
"rollback damaged the flexible tail page");
ExpectRange(test, Query(test, base), base, base + size, SceKernelProtCpuRw, 1, 0, 0, 1,
"host_reserve_rollback");
CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(base, size), "KernelMunmap");
std::printf("[host] %-48s ok\n", test);
}
void TestFixedReserveRangeAddRollbackKeepsPlaceholder() {
const char* test = "FixedReserveRangeAddRollbackKeepsPlaceholder";
constexpr uint64_t size = SceKernelPageSize * 4;
@@ -1055,8 +1659,10 @@ void TestLargeHintedReserveHostsSmallDirectMap() {
CheckOk(test, Libs::LibKernel::Memory::KernelReleaseDirectMemory(phys, SceKernelPageSize * 2),
"KernelReleaseDirectMemory");
CheckOk(test,
Libs::LibKernel::Memory::KernelMunmap(reinterpret_cast<uint64_t>(window), window_size),
"KernelMunmap(window reserve)");
Libs::LibKernel::Memory::KernelMunmap(reinterpret_cast<uint64_t>(window) +
SceKernelPageSize * 2,
window_size - SceKernelPageSize * 2),
"KernelMunmap(window reserve remainder)");
CheckOk(test,
Libs::LibKernel::Memory::KernelMunmap(reinterpret_cast<uint64_t>(arena), arena_size),
"KernelMunmap(arena reserve)");
@@ -1153,18 +1759,25 @@ void TestProsperoSampleMemoryPoolExpandCommit() {
SceKernelProtCpuRw, 0),
"KernelMemoryPoolCommit");
ExpectRange(test, Query(test, base), base, base + commit_len, SceKernelProtCpuRw, 0, 0, 1, 1);
Check(test, Libs::LibKernel::Memory::TestGuestAddressRangeIsOwned(base, commit_len),
"pooled commit escaped the guest owner");
Check(test, AvailableFlexibleMemory(test) == flexible_baseline,
"pooled commit consumed flexible memory instead of expanded direct "
"backing");
CheckFailed(test,
Libs::LibKernel::Memory::KernelReleaseDirectMemory(pool_offset,
SceKernelMemoryPoolExpandLen),
"KernelReleaseDirectMemory(committed pool expansion)");
Libs::LibKernel::Memory::KernelCheckedReleaseDirectMemory(
pool_offset, SceKernelMemoryPoolExpandLen),
"KernelCheckedReleaseDirectMemory(committed pool expansion)");
constexpr uint64_t first_value = 0x504f4f4c4241434bull; // "POOLBACK"
constexpr uint64_t second_value = 0x5348415245444d45ull; // "SHAREDME"
*reinterpret_cast<uint64_t*>(base) = first_value;
*reinterpret_cast<uint64_t*>(base + SceKernelMemoryPoolCommitLen) = second_value;
uint64_t backing_read = 0;
Check(test, Libs::LibKernel::Memory::TryReadBacking(base, &backing_read, sizeof(backing_read)),
"TryReadBacking did not resolve pooled memory");
Check(test, backing_read == first_value,
"shared backing did not observe a pooled-memory CPU write");
CheckOk(
test,
@@ -1276,8 +1889,10 @@ void TestFragmentedMemoryPoolBacking() {
"KernelMemoryPoolCommit(fragmented recommit)");
CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(base, commit_len),
"KernelMunmap(fragmented commit)");
CheckOk(test, Libs::LibKernel::Memory::KernelMunmap(base, SceKernelMemoryPoolReserveLen),
"KernelMunmap(fragmented reserve cleanup)");
CheckOk(test,
Libs::LibKernel::Memory::KernelMunmap(base + commit_len,
SceKernelMemoryPoolReserveLen - commit_len),
"KernelMunmap(fragmented reserve remainder)");
CheckOk(test,
Libs::LibKernel::Memory::KernelReleaseDirectMemory(first_pool,
@@ -1427,22 +2042,32 @@ void TestMemoryPoolCommitDecommitQueryFlags() {
std::printf("[host] %-48s ok\n", test);
}
void TestProgramMemoryRegistrationAndProtection() {
const char* test = "ProgramMemoryRegistrationAndProtection";
void TestProgramMemoryAllocationAndProtection() {
const char* test = "ProgramMemoryAllocationAndProtection";
const auto size = SceKernelPageSize * 3;
const auto base = Common::VirtualMemory::Alloc(0, size, Common::VirtualMemory::Mode::ReadWrite);
Check(test, base != 0, "program host allocation failed");
Libs::LibKernel::Memory::RegisterProgramMemory(
base, size, Common::VirtualMemory::Mode::ReadWrite, "program_test");
const auto base = Libs::LibKernel::Memory::AllocateProgramMemory(
0x900000000, size, Common::VirtualMemory::Mode::ReadWrite, "program_test");
Check(test, base != 0, "program guest allocation failed");
Check(test, Libs::LibKernel::Memory::TestGuestAddressRangeIsOwned(base, size),
"program allocation escaped the guest owner");
ExpectRange(test, Query(test, base), base, base + size,
SceKernelProtCpuRead | SceKernelProtCpuRw, 0, 0, 0, 1, "program_test");
Libs::LibKernel::Memory::UpdateProgramMemoryProtection(base, SceKernelPageSize,
Common::VirtualMemory::Mode::Read);
Check(test,
Libs::LibKernel::Memory::ProtectGuestMemory(base, SceKernelPageSize,
Common::VirtualMemory::Mode::Read),
"ProtectGuestMemory(first page) failed");
ExpectRange(test, Query(test, base), base, base + SceKernelPageSize, SceKernelProtCpuRead, 0, 0,
0, 1, "program_test");
Common::VirtualMemory::Mode previous_mode = Common::VirtualMemory::Mode::NoAccess;
Check(test,
Libs::LibKernel::Memory::ProtectGuestMemory(
base, SceKernelPageSize, Common::VirtualMemory::Mode::ReadWrite, &previous_mode),
"ProtectGuestMemory(tracked restore) failed");
Check(test, previous_mode == Common::VirtualMemory::Mode::Read,
"semantic guest protection did not preserve its tracked old mode");
CheckOk(test,
Libs::LibKernel::Memory::KernelMprotect(
reinterpret_cast<void*>(base + SceKernelPageSize - 0x10), 0x20,
@@ -1451,24 +2076,44 @@ void TestProgramMemoryRegistrationAndProtection() {
ExpectRange(test, Query(test, base), base, base + size,
SceKernelProtCpuRead | SceKernelProtCpuRw, 0, 0, 0, 1, "program_test");
Libs::LibKernel::Memory::UpdateProgramMemoryProtection(
base + SceKernelPageSize * 2, SceKernelPageSize, Common::VirtualMemory::Mode::Read);
Check(test,
Libs::LibKernel::Memory::ProtectGuestMemory(
base + SceKernelPageSize * 2, SceKernelPageSize, Common::VirtualMemory::Mode::Read),
"ProtectGuestMemory(last page) failed");
ExpectRange(test, Query(test, base + SceKernelPageSize * 2), base + SceKernelPageSize * 2,
base + size, SceKernelProtCpuRead, 0, 0, 0, 1, "program_test");
Libs::LibKernel::Memory::UnregisterProgramMemory(base, size);
Check(test, Libs::LibKernel::Memory::FreeGuestMemory(base, size), "program guest free failed");
ExpectUnmapped(test, base);
Check(test, Common::VirtualMemory::Free(base), "program host free failed");
std::printf("[host] %-48s ok\n", test);
}
void TestModuleRelocationUsesWritableHostMapping() {
const char* test = "ModuleRelocationUsesWritableHostMapping";
Check(test, Loader::TestModuleRelocationUsesWritableHostMapping(),
"module relocation did not retain writable host memory and semantic guest protection");
std::printf("[host] %-48s ok\n", test);
}
} // namespace
int main() {
InitSubsystems();
RunTest(TestProsperoArgumentAndInfoSizeContracts);
RunTest(TestGuestAddressSpaceOwnsReservationsBeforeBacking);
RunTest(TestGuestAddressSpaceHasNoFixedFallback);
RunTest(TestGuestFreeRangeSearchDoesNotUnderflow);
RunTest(TestFlexibleMemoryCapacityIsBootFixed);
RunTest(TestFlexibleMemoryUsesSharedBacking);
RunTest(TestFlexibleDmemCompatAndAlignmentFlags);
RunTest(TestFlexibleNoCoalescePreservesBoundaries);
RunTest(TestFlexibleMemoryReuseIsZeroFilled);
RunTest(TestGuestStackUsesPrivateOwnerMemoryAndCache);
RunTest(TestMainEntryUsesGuestStackAndDisablesHostChecks);
RunTest(TestFragmentedBackingUnmapRollback);
RunTest(TestRuntimeMemoryOwnerLifecycle);
RunTest(TestFlexibleMapQueryAndWholeMunmap);
RunTest(TestPartialFlexibleMunmapAndFindNext);
RunTest(TestReserveMapFixedAndNoOverwrite);
@@ -1476,7 +2121,12 @@ int main() {
RunTest(TestReleasedReserveCanBeReused);
RunTest(TestMunmapAcrossAdjacentFlexibleMappings);
RunTest(TestDirectMapQueryOffsetAndPartialMunmap);
RunTest(TestDirectPartialProtectUnmapPreservesNeighbors);
RunTest(TestDirectMapValidationBeforeOwnerMutation);
RunTest(TestDirectReleaseRollbackRestoresOwnerMapping);
RunTest(TestDirectReleaseContracts);
RunTest(TestNonzeroDirectOffsetAliasesSharedBacking);
RunTest(TestDirectMapAcrossContiguousAllocations);
RunTest(TestDirectPhysicalFreeRangeReuseAndCoalescing);
RunTest(TestDirectAlignmentStaysWithinSearchRange);
RunTest(TestDefaultDirectMapUsesSystemAddressRange);
@@ -1485,7 +2135,6 @@ int main() {
RunTest(TestFixedReserveReplacesPartialDirectMapping);
RunTest(TestFixedReserveRollbackConsumesRestoredPlaceholder);
RunTest(TestFixedReserveRollbackSkipsUntouchedChunks);
RunTest(TestFixedReserveRollbackRestoresDecommittedHostPages);
RunTest(TestFixedReserveRangeAddRollbackKeepsPlaceholder);
RunTest(TestLargeHintedReserveHostsSmallDirectMap);
RunTest(TestMemoryPoolAlignmentContracts);
@@ -1493,7 +2142,8 @@ int main() {
RunTest(TestFragmentedMemoryPoolBacking);
RunTest(TestMemoryPoolMultiRangeDecommit);
RunTest(TestMemoryPoolCommitDecommitQueryFlags);
RunTest(TestProgramMemoryRegistrationAndProtection);
RunTest(TestProgramMemoryAllocationAndProtection);
RunTest(TestModuleRelocationUsesWritableHostMapping);
if (g_failed_tests != 0) {
std::printf("VirtualMemoryAllocationTests: %d case(s) failed\n", g_failed_tests);