diff --git a/src/graphics/host_gpu/memoryTracker.cpp b/src/graphics/host_gpu/memoryTracker.cpp index 0898f8a..430a863 100644 --- a/src/graphics/host_gpu/memoryTracker.cpp +++ b/src/graphics/host_gpu/memoryTracker.cpp @@ -157,7 +157,6 @@ void MemoryTracker::UntrackMemoryLocked(uint64_t vaddr, uint64_t size) { const auto changed = manager->ChangeState(manager->GetCpuAddr() + offset, bytes); manager->ApplyProtection(changed, false); - manager->Untrack(manager->GetCpuAddr() + offset, bytes); }); locks.clear(); } @@ -168,93 +167,4 @@ void MemoryTracker::UntrackMemory(uint64_t vaddr, uint64_t size) { UntrackMemoryLocked(vaddr, size); } -bool MemoryTracker::InvalidateRegion(uint64_t vaddr, uint64_t size, PageFaultPhase phase) noexcept { - switch (phase) { - case PageFaultPhase::Release: return true; - case PageFaultPhase::Invalidate: { - const auto action = BeginCpuFault(vaddr, size); - switch (action) { - case CpuFaultAction::Untracked: return false; - case CpuFaultAction::Continue: return true; - case CpuFaultAction::Download: - EXIT("generic region invalidation cannot download GPU-dirty memory\n"); - } - } - case PageFaultPhase::Complete: - return CompleteCpuFault(vaddr, size, PageFaultAccess::Write, false); - } - EXIT("unsupported region invalidation phase\n"); -} - -bool MemoryTracker::InvalidateVirtualGpuWrite(PageFaultAccess access, uint64_t vaddr, uint64_t size, - PageFaultPhase phase) noexcept { - switch (phase) { - case PageFaultPhase::Release: return true; - case PageFaultPhase::Invalidate: { - const bool gpu_modified = Iterate( - vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) { - std::scoped_lock lock(manager->lock); - return manager->IsModified(offset, bytes); - }); - if (!gpu_modified) { - return false; - } - const auto action = BeginCpuFault(vaddr, size); - if (access != PageFaultAccess::Write || action != CpuFaultAction::Download) { - EXIT("virtual GPU write fault requires write access to GPU-dirty memory\n"); - } - return true; - } - case PageFaultPhase::Complete: { - if (access != PageFaultAccess::Write) { - EXIT("virtual GPU write completion requires write access\n"); - } - bool completed = false; - Iterate( - vaddr, size, [&completed](RegionManager* manager, uint64_t offset, uint64_t bytes) { - std::scoped_lock lock(manager->lock); - if (completed) { - EXIT("virtual GPU write fault spans multiple tracked regions\n"); - } - completed = - manager->CompleteVirtualGpuWrite(manager->GetCpuAddr() + offset, bytes); - }); - return completed; - } - } - EXIT("unsupported virtual GPU write invalidation phase\n"); -} - -CpuFaultAction MemoryTracker::BeginCpuFault(uint64_t vaddr, uint64_t size, - PageFaultAccess access) noexcept { - CheckNotInUploadCallback(); - CpuFaultAction action = CpuFaultAction::Untracked; - Iterate( - vaddr, size, [&action, access](RegionManager* manager, uint64_t offset, uint64_t bytes) { - std::scoped_lock lock(manager->lock); - if (action != CpuFaultAction::Untracked) { - EXIT("CPU fault spans multiple tracked regions\n"); - } - action = manager->BeginCpuFault(manager->GetCpuAddr() + offset, bytes, access); - }); - return action; -} - -bool MemoryTracker::CompleteCpuFault(uint64_t vaddr, uint64_t size, PageFaultAccess access, - bool downloaded) noexcept { - CheckNotInUploadCallback(); - bool found = false; - Iterate( - vaddr, size, - [&found, access, downloaded](RegionManager* manager, uint64_t offset, uint64_t bytes) { - std::scoped_lock lock(manager->lock); - if (found) { - EXIT("CPU fault completion spans multiple tracked regions\n"); - } - found = manager->CompleteCpuFault(manager->GetCpuAddr() + offset, bytes, access, - downloaded); - }); - return found; -} - } // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/memoryTracker.h b/src/graphics/host_gpu/memoryTracker.h index f814da9..90715e7 100644 --- a/src/graphics/host_gpu/memoryTracker.h +++ b/src/graphics/host_gpu/memoryTracker.h @@ -30,13 +30,6 @@ public: void MarkRegionAsGpuModified(uint64_t vaddr, uint64_t size); void UnmarkRegionAsGpuModified(uint64_t vaddr, uint64_t size); void UntrackMemory(uint64_t vaddr, uint64_t size); - [[nodiscard]] CpuFaultAction - BeginCpuFault(uint64_t vaddr, uint64_t size, - PageFaultAccess access = PageFaultAccess::Write) noexcept; - [[nodiscard]] bool CompleteCpuFault(uint64_t vaddr, uint64_t size, PageFaultAccess access, - bool downloaded) noexcept; - [[nodiscard]] bool InvalidateRegion(uint64_t vaddr, uint64_t size, - PageFaultPhase phase) noexcept; template void InvalidateRegion(uint64_t vaddr, uint64_t size, Flush&& on_flush) { static_assert(std::is_invocable_v); @@ -78,10 +71,8 @@ public: EXIT("memory invalidation retained GPU-owned pages\n"); } } - [[nodiscard]] bool InvalidateVirtualGpuWrite(PageFaultAccess access, uint64_t vaddr, - uint64_t size, PageFaultPhase phase) noexcept; - void ValidateGpuDirtyPages(const RangeSet& dirty, uint64_t vaddr, uint64_t size, - const char* operation) const noexcept; + void ValidateGpuDirtyPages(const RangeSet& dirty, uint64_t vaddr, uint64_t size, + const char* operation) const noexcept; void ValidateGpuDirtyOwnership(const RangeSet& dirty, uint64_t vaddr, uint64_t size, const char* operation); @@ -102,9 +93,6 @@ public: } Iterate(vaddr, size, [&](RegionManager* manager, uint64_t offset, uint64_t bytes) { const auto address = manager->GetCpuAddr() + offset; - if (manager->HasPendingFault(address, bytes)) { - EXIT("GPU download synchronization raced a pending CPU fault\n"); - } manager->template ForEachModifiedRange(address, bytes, preflight); }); @@ -141,7 +129,6 @@ public: const auto* previous_upload_owner = std::exchange(s_upload_owner, this); Iterate(vaddr, size, [&](RegionManager* manager, uint64_t offset, uint64_t bytes) { manager->lock.lock(); - manager->Track(manager->GetCpuAddr() + offset, bytes); manager->ForEachModifiedRange(manager->GetCpuAddr() + offset, bytes, range_func); if (!is_written) { diff --git a/src/graphics/host_gpu/pageManager.cpp b/src/graphics/host_gpu/pageManager.cpp index 88eb637..367abb6 100644 --- a/src/graphics/host_gpu/pageManager.cpp +++ b/src/graphics/host_gpu/pageManager.cpp @@ -56,8 +56,6 @@ constexpr uint32_t READ_WRITE_PROTECTION = PAGE_READWRITE; // Zero is the unknown protection sentinel. constexpr uint32_t UNKNOWN_PROTECTION = 0; -thread_local bool g_in_fault_resolution = false; - [[noreturn]] void FailFast(const char* reason = nullptr) noexcept { std::fputs("PageManager fail-fast: ", stderr); std::fputs(reason != nullptr ? reason : "invalid page state", stderr); @@ -162,11 +160,8 @@ struct PageManager::Impl { uint32_t original_protection = 0; uint32_t backing_writer = 0; // Shadow the protection applied through Protect(). - uint32_t current_protection = UNKNOWN_PROTECTION; - bool resolving = false; - bool resolving_read_write = false; - bool late_read_pending = false; - bool late_write_pending = false; + uint32_t current_protection = UNKNOWN_PROTECTION; + bool resolving = false; }; struct Region { @@ -193,10 +188,7 @@ struct PageManager::Impl { std::span m_pages; }; - Impl(PageFaultHandler handler, void* context): fault_handler(handler), fault_context(context) { - if (fault_handler == nullptr) { - Fatal("null page-manager fault callback"); - } + Impl() { #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS SYSTEM_INFO info {}; GetSystemInfo(&info); @@ -268,17 +260,6 @@ struct PageManager::Impl { return page.original_protection; } - static void PublishDelayedFaults(PageState& page, uint32_t old_protection, - uint32_t new_protection) { - if (old_protection == NO_ACCESS_PROTECTION && new_protection != NO_ACCESS_PROTECTION) { - page.late_read_pending = true; - } - if ((old_protection == NO_ACCESS_PROTECTION || old_protection == READ_ONLY_PROTECTION) && - new_protection == READ_WRITE_PROTECTION) { - page.late_write_pending = true; - } - } - static void InitializeProtection(std::span pages) { for (auto* page: pages) { page->original_protection = READ_WRITE_PROTECTION; @@ -286,19 +267,8 @@ struct PageManager::Impl { } } - static bool AllowsAccess(const PageState& page, [[maybe_unused]] uint64_t vaddr, - PageFaultAccess access) noexcept { - switch (access) { - case PageFaultAccess::Read: - return page.current_protection == READ_ONLY_PROTECTION || - page.current_protection == READ_WRITE_PROTECTION; - case PageFaultAccess::Write: return page.current_protection == READ_WRITE_PROTECTION; - default: return false; - } - } - void ProtectRange(std::span pages, uint64_t vaddr, uint32_t protection, - std::span expected_old, bool fault_path) noexcept { + std::span expected_old) noexcept { const auto size = pages.size() * PAGE_SIZE; if (pages.size() != expected_old.size()) { FailFast("protection range state size mismatch"); @@ -306,9 +276,6 @@ struct PageManager::Impl { for (size_t i = 0; i < pages.size(); i++) { const auto actual = pages[i]->current_protection; if (actual != UNKNOWN_PROTECTION && actual != expected_old[i]) { - if (fault_path) { - FailFast("mprotect fault transition did not match expected protection"); - } Fatal("invalid protection transition at 0x%016" PRIx64 ", old=0x%08" PRIx32 ", expected=0x%08" PRIx32 ", new=0x%08" PRIx32, vaddr + i * PAGE_SIZE, actual, expected_old[i], protection); @@ -316,9 +283,6 @@ struct PageManager::Impl { } if (!Libs::LibKernel::Memory::ProtectGuestHostMemory(vaddr, size, ToMemoryMode(protection))) { - if (fault_path) { - FailFast("address-space fault protection transition failed"); - } Fatal("address-space protection failed at 0x%016" PRIx64 ", new=0x%08" PRIx32, vaddr, protection); } @@ -327,47 +291,28 @@ struct PageManager::Impl { } } - void Protect(PageState& page, uint64_t vaddr, uint32_t protection, uint32_t expected_old, - bool fault_path) noexcept { + void Protect(PageState& page, uint64_t vaddr, uint32_t protection, + uint32_t expected_old) noexcept { PageState* pages[] = {&page}; uint32_t expected[] = {expected_old}; - ProtectRange(pages, vaddr, protection, expected, fault_path); + ProtectRange(pages, vaddr, protection, expected); } std::unique_ptr[]> regions; std::vector> region_storage; std::mutex region_mutex; - PageFaultHandler fault_handler = nullptr; - void* fault_context = nullptr; }; static_assert(std::atomic::is_always_lock_free); -PageManager::PageManager(PageFaultHandler fault_handler, void* fault_context) - : m_impl(std::make_unique(fault_handler, fault_context)) {} +PageManager::PageManager(): m_impl(std::make_unique()) {} PageManager::~PageManager() = default; uint64_t PageManager::GetPageSize() const { - if (g_in_fault_resolution) { - FailFast("nested page fault while resolving a watched page"); - } return PAGE_SIZE; } -bool PageManager::IsTracked(uint64_t vaddr) const noexcept { - if (g_in_fault_resolution) { - FailFast("IsTracked called during fault resolution"); - } - auto* region = m_impl->FindRegion(vaddr); - if (region == nullptr) { - return false; - } - auto& page = m_impl->GetPage(*region, vaddr); - SpinGuard lock(page.lock); - return page.write_watchers != 0 || page.access_watchers != 0; -} - void PageManager::UpdatePageWatchers(bool track, uint64_t vaddr, uint64_t size, PageWatchMode mode) { if (mode != PageWatchMode::Write && mode != PageWatchMode::ReadWrite) { @@ -473,27 +418,14 @@ void PageManager::UpdatePageWatchers(bool track, uint64_t vaddr, uint64_t size, } m_impl->ProtectRange(std::span {pages}.subspan(first, last - first), chunk_begin + first * PAGE_SIZE, protection, - std::span {old_protections}.subspan(first, last - first), false); + std::span {old_protections}.subspan(first, last - first)); first = current; } - for (size_t i = 0; i < page_count; i++) { - auto& page = *pages[i]; - const auto protection = new_protections[i]; - if (track) { - switch (protection) { - case NO_ACCESS_PROTECTION: - page.late_read_pending = false; - page.late_write_pending = false; - break; - case READ_ONLY_PROTECTION: page.late_write_pending = false; break; - default: break; - } - } else if (page.backing_writer == 0) { - Impl::PublishDelayedFaults(page, old_protections[i], protection); - if (page.write_watchers == 0 && page.access_watchers == 0) { - page.original_protection = 0; - } + for (auto* page: pages) { + if (!track && page->backing_writer == 0 && page->write_watchers == 0 && + page->access_watchers == 0) { + page->original_protection = 0; } } chunk_begin = chunk_end; @@ -546,9 +478,6 @@ PageManager::ReserveBackingWrites(std::span ranges) { } void PageManager::BeginBackingWrite(uint64_t vaddr, uint64_t size) noexcept { - if (g_in_fault_resolution) { - FailFast("backing write began during fault resolution"); - } const auto end = PageEnd(vaddr, size); const auto writer = CurrentThread(); for (auto address = PageStart(vaddr); address < end; address += PAGE_SIZE) { @@ -561,16 +490,12 @@ void PageManager::BeginBackingWrite(uint64_t vaddr, uint64_t size) noexcept { if (page.resolving || page.backing_writer != 0 || page.access_watchers == 0) { Fatal("backing write races page resolution at 0x%016" PRIx64, address); } - page.resolving = true; - page.resolving_read_write = true; - page.backing_writer = writer; + page.resolving = true; + page.backing_writer = writer; } } void PageManager::EndBackingWrite(uint64_t vaddr, uint64_t size) noexcept { - if (g_in_fault_resolution) { - FailFast("backing write ended during fault resolution"); - } const auto end = PageEnd(vaddr, size); const auto writer = CurrentThread(); for (auto address = PageStart(vaddr); address < end; address += PAGE_SIZE) { @@ -586,127 +511,14 @@ void PageManager::EndBackingWrite(uint64_t vaddr, uint64_t size) noexcept { const auto old_protection = NO_ACCESS_PROTECTION; const auto new_protection = Impl::WatcherProtection(page); if (new_protection != old_protection) { - m_impl->Protect(page, address, new_protection, old_protection, false); + m_impl->Protect(page, address, new_protection, old_protection); } - Impl::PublishDelayedFaults(page, old_protection, new_protection); if (page.write_watchers == 0 && page.access_watchers == 0) { page.original_protection = 0; } - page.backing_writer = 0; - page.resolving = false; - page.resolving_read_write = false; + page.backing_writer = 0; + page.resolving = false; } } -bool PageManager::HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noexcept { - if (g_in_fault_resolution) { - FailFast("nested HandleFault call"); - } - auto* region = m_impl->FindRegion(fault_vaddr); - if (region == nullptr) { - return false; - } - auto& page = m_impl->GetPage(*region, fault_vaddr); - bool waited = false; - while (true) { - SpinGuard lock(page.lock); - if (access == PageFaultAccess::Read && page.late_read_pending && - Impl::AllowsAccess(page, fault_vaddr, access)) { - page.late_read_pending = false; - return true; - } - if (access == PageFaultAccess::Write && page.late_write_pending && - Impl::AllowsAccess(page, fault_vaddr, access)) { - page.late_write_pending = false; - return true; - } - if (page.resolving) { - if (page.backing_writer == CurrentThread()) { - FailFast("backing writer faulted on its own reserved page"); - } - if ((!page.resolving_read_write && access != PageFaultAccess::Write) || - (page.resolving_read_write && access != PageFaultAccess::Read && - access != PageFaultAccess::Write)) { - FailFast("fault access is incompatible with the active resolver"); - } - waited = true; - continue; - } - if (page.write_watchers == 0 && page.access_watchers == 0) { - if (access != PageFaultAccess::Read && access != PageFaultAccess::Write) { - return false; - } - bool& pending = (access == PageFaultAccess::Read ? page.late_read_pending - : page.late_write_pending); - const bool allowed = Impl::AllowsAccess(page, fault_vaddr, access); - pending = false; - if (waited && !allowed) { - FailFast("page remained inaccessible after waiting for its resolver"); - } - // More than one CPU can fault before a protection transition becomes visible. The first - // delayed fault consumes the hint bit; later faults must also resume once the mapped - // page already permits the requested access. A genuinely read-only/no-access page still - // falls through to the guest exception path. - return allowed; - } - if ((access != PageFaultAccess::Read && access != PageFaultAccess::Write) || - (access == PageFaultAccess::Read && page.access_watchers == 0)) { - FailFast("fault access is incompatible with active page watchers"); - } - page.resolving = true; - page.resolving_read_write = page.access_watchers != 0; - break; - } - g_in_fault_resolution = true; - const bool handled = m_impl->fault_handler(m_impl->fault_context, access, fault_vaddr, 1, - PageFaultPhase::Invalidate); - g_in_fault_resolution = false; - { - SpinGuard lock(page.lock); - if (!handled || !page.resolving) { - FailFast("fault invalidation did not preserve the resolving state"); - } - } - g_in_fault_resolution = true; - const bool completed = m_impl->fault_handler(m_impl->fault_context, access, fault_vaddr, 1, - PageFaultPhase::Complete); - g_in_fault_resolution = false; - { - SpinGuard lock(page.lock); - if (!completed || !page.resolving) { - FailFast("fault completion did not preserve the resolving state"); - } - if (page.write_watchers != 0 || page.access_watchers != 0) { - const auto old_protection = Impl::WatcherProtection(page); - const bool read_only_fault = access == PageFaultAccess::Read; - if (read_only_fault && page.access_watchers == 0) { - FailFast("read fault completed without a read/write watcher"); - } - page.access_watchers = 0; - if (!read_only_fault) { - page.write_watchers = 0; - } - const auto restored_protection = Impl::WatcherProtection(page); - m_impl->Protect(page, PageStart(fault_vaddr), restored_protection, old_protection, - true); - if (page.write_watchers == 0) { - page.original_protection = 0; - } - Impl::PublishDelayedFaults(page, old_protection, restored_protection); - } else if (!Impl::AllowsAccess(page, fault_vaddr, access)) { - FailFast("fault completion left the page inaccessible"); - } - page.resolving = false; - page.resolving_read_write = false; - } - g_in_fault_resolution = true; - const bool released = m_impl->fault_handler(m_impl->fault_context, access, fault_vaddr, 1, - PageFaultPhase::Release); - g_in_fault_resolution = false; - if (!released) { - FailFast("fault release callback failed"); - } - return true; -} - } // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/pageManager.h b/src/graphics/host_gpu/pageManager.h index 6e98e86..619602e 100644 --- a/src/graphics/host_gpu/pageManager.h +++ b/src/graphics/host_gpu/pageManager.h @@ -11,11 +11,8 @@ namespace Libs::Graphics { enum class PageFaultAccess { Read, Write, Execute, Unknown }; -enum class PageFaultPhase { Invalidate, Complete, Release }; enum class PageWatchMode { Write, ReadWrite }; -using PageFaultHandler = bool (*)(void* context, PageFaultAccess access, uint64_t vaddr, - uint64_t size, PageFaultPhase phase) noexcept; class PageManager final { public: class BackingWrite final { @@ -30,21 +27,19 @@ public: uint64_t m_size = 0; }; - PageManager(PageFaultHandler fault_handler, void* fault_context); + PageManager(); // The owner must stop all PageManager callers before destruction. ~PageManager(); KYTY_CLASS_NO_COPY(PageManager); [[nodiscard]] uint64_t GetPageSize() const; - [[nodiscard]] bool IsTracked(uint64_t vaddr) const noexcept; void UpdatePageWatchers(bool track, uint64_t vaddr, uint64_t size, PageWatchMode mode = PageWatchMode::Write); void OnGpuMap(uint64_t vaddr, uint64_t size); void OnGpuUnmap(uint64_t vaddr, uint64_t size); - [[nodiscard]] bool HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noexcept; [[nodiscard]] std::vector> ReserveBackingWrites(std::span ranges); diff --git a/src/graphics/host_gpu/regionManager.h b/src/graphics/host_gpu/regionManager.h index f3371bb..8b053e5 100644 --- a/src/graphics/host_gpu/regionManager.h +++ b/src/graphics/host_gpu/regionManager.h @@ -25,8 +25,6 @@ namespace Libs::Graphics { -enum class CpuFaultAction { Untracked, Continue, Download }; - class TrackingSpinLock final { public: void lock() noexcept { @@ -85,18 +83,6 @@ public: KYTY_CLASS_NO_COPY(RegionManager); [[nodiscard]] uint64_t GetCpuAddr() const { return m_cpu_addr; } - void Track(uint64_t vaddr, uint64_t size) { - const auto [start, end] = GetPageRange(vaddr, size); - for (auto page = start; page < end; page++) { - m_tracked.set(page); - } - } - void Untrack(uint64_t vaddr, uint64_t size) { - const auto [start, end] = GetPageRange(vaddr, size); - for (auto page = start; page < end; page++) { - m_tracked.reset(page); - } - } template [[nodiscard]] bool IsModified(uint64_t offset, uint64_t size) const { const auto [start, end] = GetPageRange(m_cpu_addr + offset, size); @@ -126,15 +112,15 @@ public: const auto [start, end] = GetPageRange(vaddr, size); if constexpr (source == DirtySource::Cpu && enable) { for (auto page = start; page < end; page++) { - if (m_gpu_dirty.test(page) || m_fault_pending.test(page)) { - EXIT("CPU dirty state conflicts with GPU dirty or pending fault state\n"); + if (m_gpu_dirty.test(page)) { + EXIT("CPU dirty state conflicts with GPU dirty state\n"); } } } if constexpr (source == DirtySource::Gpu && enable) { for (auto page = start; page < end; page++) { - if (m_cpu_dirty.test(page) || m_fault_pending.test(page)) { - EXIT("GPU dirty state conflicts with CPU dirty or pending fault state\n"); + if (m_cpu_dirty.test(page)) { + EXIT("GPU dirty state conflicts with CPU dirty state\n"); } } } @@ -151,110 +137,10 @@ public: return changed; } - [[nodiscard]] CpuFaultAction BeginCpuFault(uint64_t vaddr, uint64_t size, - PageFaultAccess access = PageFaultAccess::Write) { - if (access != PageFaultAccess::Read && access != PageFaultAccess::Write) { - EXIT("unsupported CPU fault access while beginning ownership transfer\n"); - } - const auto [start, end] = GetPageRange(vaddr, size); - const bool tracked = m_tracked.test(start); - for (auto page = start; page < end; page++) { - if (m_tracked.test(page) != tracked) { - EXIT("CPU fault spans mixed tracked and untracked pages\n"); - } - if (m_fault_pending.test(page)) { - return CpuFaultAction::Untracked; - } - if (m_cpu_dirty.test(page) != m_writable.test(page) || - (m_gpu_dirty.test(page) && (m_cpu_dirty.test(page) || m_writable.test(page)))) { - EXIT("inconsistent CPU fault page state\n"); - } - } - if (!tracked) { - return CpuFaultAction::Untracked; - } - bool gpu_dirty = m_gpu_dirty.test(start); - bool writable = m_writable.test(start); - for (auto page = start + 1; page < end; page++) { - if (m_gpu_dirty.test(page) != gpu_dirty || m_writable.test(page) != writable) { - EXIT("CPU fault spans pages with incompatible dirty or writable state\n"); - } - } - for (auto page = start; page < end; page++) { - if (!gpu_dirty && access == PageFaultAccess::Write) { - m_cpu_dirty.set(page); - m_writable.set(page); - } - m_fault_pending.set(page); - } - return gpu_dirty ? CpuFaultAction::Download : CpuFaultAction::Continue; - } - - [[nodiscard]] bool CompleteCpuFault(uint64_t vaddr, uint64_t size, PageFaultAccess access, - bool downloaded) { - const auto [start, end] = GetPageRange(vaddr, size); - for (auto page = start; page < end; page++) { - if (!m_fault_pending.test(page)) { - return false; - } - } - for (auto page = start; page < end; page++) { - const bool gpu_dirty = m_gpu_dirty.test(page); - if (gpu_dirty != downloaded) { - EXIT("CPU fault download result disagrees with GPU dirty state\n"); - } - if (gpu_dirty) { - m_gpu_dirty.reset(page); - switch (access) { - case PageFaultAccess::Read: break; - case PageFaultAccess::Write: - m_cpu_dirty.set(page); - m_writable.set(page); - break; - default: EXIT("unsupported CPU fault access after GPU download\n"); - } - } - m_fault_pending.reset(page); - } - return true; - } - - [[nodiscard]] bool HasPendingFault(uint64_t vaddr, uint64_t size) const { - const auto [start, end] = GetPageRange(vaddr, size); - for (auto page = start; page < end; page++) { - if (m_fault_pending.test(page)) { - return true; - } - } - return false; - } - - [[nodiscard]] bool CompleteVirtualGpuWrite(uint64_t vaddr, uint64_t size) { - const auto [start, end] = GetPageRange(vaddr, size); - for (auto page = start; page < end; page++) { - if (!m_fault_pending.test(page)) { - return false; - } - if (!m_gpu_dirty.test(page)) { - EXIT("virtual GPU write completion found a non-GPU-dirty page\n"); - } - } - for (auto page = start; page < end; page++) { - m_gpu_dirty.reset(page); - m_cpu_dirty.set(page); - m_writable.set(page); - m_fault_pending.reset(page); - } - return true; - } - template RegionBits ForEachModifiedRange(uint64_t vaddr, uint64_t size, Func&& func) { const auto [start, end] = GetPageRange(vaddr, size); auto mask = GetBits(); - if constexpr (source == DirtySource::Cpu) { - mask &= ~m_fault_pending; - } for (auto page = 0u; page < start; page++) { mask.reset(page); } @@ -351,8 +237,6 @@ private: RegionBits m_cpu_dirty; RegionBits m_gpu_dirty; RegionBits m_writable; - RegionBits m_fault_pending; - RegionBits m_tracked; }; } // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/cache/bufferCache.cpp b/src/graphics/host_gpu/renderer/cache/bufferCache.cpp index 2d234f5..ae88571 100644 --- a/src/graphics/host_gpu/renderer/cache/bufferCache.cpp +++ b/src/graphics/host_gpu/renderer/cache/bufferCache.cpp @@ -123,24 +123,6 @@ struct BufferCache::RetiredBuffer { std::shared_ptr owner; }; -struct BufferCache::FaultReadback { - PageFaultAccess access = PageFaultAccess::Unknown; - uint64_t vaddr = 0; - uint64_t size = 0; - std::vector ranges; - bool installed = false; - - [[nodiscard]] bool Active() const noexcept { return !ranges.empty(); } - - void Reset() { - access = PageFaultAccess::Unknown; - vaddr = 0; - size = 0; - installed = false; - ranges.clear(); - } -}; - struct BufferCache::PendingBackingPublication { uint64_t address = 0; uint64_t size = 0; @@ -253,7 +235,7 @@ BufferCache::BufferCache(GraphicContext& graphics, CommandScheduler& scheduler, ResourceMutex& resource_mutex) : m_graphics(graphics), m_scheduler(scheduler), m_gds_buffer(graphics, scheduler, MemoryUsage::Stream, 0, AllFlags, GdsBufferSize), - m_fault_readback(std::make_unique()), m_memory_tracker(page_manager), + m_memory_tracker(page_manager), m_staging_buffer(graphics, scheduler, MemoryUsage::Upload, 512 * MiB), m_stream_buffer(graphics, scheduler, MemoryUsage::Stream, 64 * MiB), m_download_buffer(graphics, scheduler, MemoryUsage::Download, 32 * MiB), @@ -277,9 +259,6 @@ BufferCache::BufferCache(GraphicContext& graphics, CommandScheduler& scheduler, } BufferCache::~BufferCache() { - if (m_fault_readback->Active()) { - EXIT("BufferCache: destroyed with an active fault readback\n"); - } if (!m_gpu_modified_ranges.Empty()) { EXIT("BufferCache: destroyed with pending GPU-modified ranges\n"); } @@ -447,93 +426,6 @@ void BufferCache::ReadMemory(uint64_t vaddr, uint64_t size) { } } -bool BufferCache::InvalidateMemory(PageFaultAccess access, uint64_t vaddr, uint64_t size, - PageFaultPhase phase) noexcept { - const auto page = vaddr & ~(TRACKER_PAGE_SIZE - 1); - if (size == 0 || size > page + TRACKER_PAGE_SIZE - vaddr) { - EXIT("BufferCache: invalid page-fault range\n"); - } - - if (phase == PageFaultPhase::Complete) { - FaultSafeCacheLock lock(this, m_mutex); - auto& fault = *m_fault_readback; - if (!fault.Active()) { - return m_memory_tracker.CompleteCpuFault(vaddr, size, access, false); - } - if (fault.access != access || fault.vaddr != vaddr || fault.size != size || - fault.installed) { - EXIT("BufferCache: mismatched fault readback completion\n"); - } - PublishDownloads(fault.ranges); - if (!m_memory_tracker.CompleteCpuFault(vaddr, size, access, true)) { - EXIT("BufferCache: failed to complete downloaded CPU fault\n"); - } - fault.installed = true; - return true; - } - - if (phase == PageFaultPhase::Release) { - FaultSafeCacheLock lock(this, m_mutex); - auto& fault = *m_fault_readback; - if (fault.Active()) { - if (fault.access != access || fault.vaddr != vaddr || fault.size != size || - !fault.installed) { - EXIT("BufferCache: mismatched fault readback release\n"); - } - for (const auto& range: fault.ranges) { - m_gpu_modified_ranges.Subtract(range.address, range.size); - } - fault.Reset(); - } - return true; - } - - if (phase != PageFaultPhase::Invalidate) { - EXIT("BufferCache: unsupported page-fault phase\n"); - } - - const auto action = m_memory_tracker.BeginCpuFault(vaddr, size, access); - if (action != CpuFaultAction::Download) { - return action == CpuFaultAction::Continue; - } - - auto& fault = *m_fault_readback; - std::vector copies; - { - FaultSafeCacheLock lock(this, m_mutex); - if (fault.Active()) { - EXIT("BufferCache: nested fault readback\n"); - } - fault.access = access; - fault.vaddr = vaddr; - fault.size = size; - - m_gpu_modified_ranges.ForEachIntersection( - page, TRACKER_PAGE_SIZE, [&](RangeSet::Range range) { - auto owner = m_buffers.upper_bound(range.address); - if (owner == m_buffers.begin()) { - EXIT("BufferCache: fault readback has no buffer owner\n"); - } - --owner; - auto& cached = *owner->second; - if (!cached.buffer->IsInBounds(range.address, range.size)) { - EXIT("BufferCache: fault readback is outside its buffer owner\n"); - } - copies.push_back({cached.buffer, cached.buffer->Offset(range.address), - range.address, range.size}); - }); - if (copies.empty()) { - EXIT("BufferCache: GPU-dirty fault page has no dirty byte ranges\n"); - } - } - fault.ranges = RecordDownloads(copies); - if (!fault.Active()) { - EXIT("BufferCache: GPU-dirty fault page has no dirty byte ranges\n"); - } - m_scheduler.FinishCurrent(); - return true; -} - void BufferCache::UnmapMemory(uint64_t vaddr, uint64_t size) { if (vaddr == 0 || size == 0 || size > UINT64_MAX - vaddr) { EXIT("BufferCache: invalid unmap range\n"); @@ -1201,7 +1093,7 @@ void BufferCache::RunGarbageCollector() { if (m_graphics.CanReportMemoryUsage()) { m_total_used_memory = m_graphics.GetDeviceMemoryUsage(); } - if (m_total_used_memory < m_trigger_gc_memory || m_fault_readback->Active()) { + if (m_total_used_memory < m_trigger_gc_memory) { return; } diff --git a/src/graphics/host_gpu/renderer/cache/bufferCache.h b/src/graphics/host_gpu/renderer/cache/bufferCache.h index aaefec5..4091859 100644 --- a/src/graphics/host_gpu/renderer/cache/bufferCache.h +++ b/src/graphics/host_gpu/renderer/cache/bufferCache.h @@ -47,11 +47,9 @@ public: ~BufferCache(); KYTY_CLASS_NO_COPY(BufferCache); - [[nodiscard]] bool InvalidateMemory(PageFaultAccess access, uint64_t vaddr, uint64_t size, - PageFaultPhase phase) noexcept; - void InvalidateMemory(uint64_t vaddr, uint64_t size); - void ReadMemory(uint64_t vaddr, uint64_t size); - void UnmapMemory(uint64_t vaddr, uint64_t size); + void InvalidateMemory(uint64_t vaddr, uint64_t size); + void ReadMemory(uint64_t vaddr, uint64_t size); + void UnmapMemory(uint64_t vaddr, uint64_t size); [[nodiscard]] BufferBinding ObtainBuffer(CommandBuffer& command, uint64_t vaddr, uint64_t size, bool is_written = false, bool is_read = true, bool is_formatted = false); @@ -90,7 +88,6 @@ private: struct DownloadCopy; struct DownloadRange; struct RetiredBuffer; - struct FaultReadback; struct PendingBackingPublication; static constexpr uint64_t DOWNLOAD_ALIGNMENT = 64; [[nodiscard]] static uint64_t AlignDown(uint64_t value) noexcept; @@ -120,7 +117,6 @@ private: Common::Mutex m_mutex; std::shared_ptr m_null_buffer; std::map> m_buffers; - std::unique_ptr m_fault_readback; RangeSet m_gpu_modified_ranges; RangeSet m_image_invalidated_ranges; std::mutex m_publication_mutex; diff --git a/src/graphics/host_gpu/renderer/cache/gpuResourceManager.cpp b/src/graphics/host_gpu/renderer/cache/gpuResourceManager.cpp index f72de2a..efbe250 100644 --- a/src/graphics/host_gpu/renderer/cache/gpuResourceManager.cpp +++ b/src/graphics/host_gpu/renderer/cache/gpuResourceManager.cpp @@ -7,33 +7,11 @@ namespace Libs::Graphics { GpuResourceManager::GpuResourceManager(GraphicContext& graphics, CommandScheduler& scheduler) - : m_page_manager(FaultThunk, this), - m_buffer_cache(graphics, scheduler, m_page_manager, m_texture_cache, m_resource_mutex), + : m_buffer_cache(graphics, scheduler, m_page_manager, m_texture_cache, m_resource_mutex), m_texture_cache(graphics, scheduler, m_page_manager, m_buffer_cache, m_resource_mutex) {} GpuResourceManager::~GpuResourceManager() = default; -bool GpuResourceManager::FaultThunk(void* context, PageFaultAccess access, uint64_t vaddr, - uint64_t size, PageFaultPhase phase) noexcept { - return static_cast(context)->InvalidateMemory(access, vaddr, size, phase); -} - -bool GpuResourceManager::InvalidateMemory(PageFaultAccess access, uint64_t vaddr, uint64_t size, - PageFaultPhase phase) noexcept { - // Let the authoritative image materialize first. A clean overlapping buffer marks a write - // fault CPU-dirty when it begins ownership transfer; doing that before image preflight would - // make the image appear to race a real CPU write. Completion and release retain buffer-first - // ordering so its pending fault is gone before TextureCache publishes the downloaded backing. - if (phase == PageFaultPhase::Invalidate) { - const bool image_handled = m_texture_cache.InvalidateMemory(access, vaddr, size, phase); - const bool buffer_handled = m_buffer_cache.InvalidateMemory(access, vaddr, size, phase); - return buffer_handled || image_handled; - } - const bool buffer_handled = m_buffer_cache.InvalidateMemory(access, vaddr, size, phase); - const bool image_handled = m_texture_cache.InvalidateMemory(access, vaddr, size, phase); - return buffer_handled || image_handled; -} - bool GpuResourceManager::HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noexcept { constexpr uint64_t fault_size = 8; if (!IsMapped(fault_vaddr, fault_size)) { diff --git a/src/graphics/host_gpu/renderer/cache/gpuResourceManager.h b/src/graphics/host_gpu/renderer/cache/gpuResourceManager.h index db2d822..8a3098b 100644 --- a/src/graphics/host_gpu/renderer/cache/gpuResourceManager.h +++ b/src/graphics/host_gpu/renderer/cache/gpuResourceManager.h @@ -34,11 +34,6 @@ public: void RunGarbageCollector(); private: - static bool FaultThunk(void* context, PageFaultAccess access, uint64_t vaddr, uint64_t size, - PageFaultPhase phase) noexcept; - [[nodiscard]] bool InvalidateMemory(PageFaultAccess access, uint64_t vaddr, uint64_t size, - PageFaultPhase phase) noexcept; - PageManager m_page_manager; ResourceMutex m_resource_mutex; BufferCache m_buffer_cache; diff --git a/src/graphics/host_gpu/renderer/cache/textureCache.cpp b/src/graphics/host_gpu/renderer/cache/textureCache.cpp index f95bf54..b3a8ed1 100644 --- a/src/graphics/host_gpu/renderer/cache/textureCache.cpp +++ b/src/graphics/host_gpu/renderer/cache/textureCache.cpp @@ -1826,37 +1826,6 @@ bool TextureCache::TouchMeta(uint64_t address, uint32_t slice, bool is_clear) { return true; } -bool TextureCache::InvalidateMemory(PageFaultAccess access, uint64_t address, uint64_t size, - PageFaultPhase phase) noexcept { - if ((access != PageFaultAccess::Read && access != PageFaultAccess::Write) || - !GuestRange {address, size}.Valid()) { - return false; - } - if (access == PageFaultAccess::Read) { - return false; - } - if (phase == PageFaultPhase::Invalidate) { - CacheLock lock(*this, m_lock); - const bool tracked = - std::ranges::any_of(FindImagesInRegion(address, size, true), [&](ImageId id) { - const auto owner = ResolveOwner(id); - return owner != nullptr && !owner->depth_id && owner->IsTracked(); - }); - if (tracked) { - InvalidateCpuAliases(address, size); - } - return tracked; - } - if (phase != PageFaultPhase::Complete && phase != PageFaultPhase::Release) { - return false; - } - CacheLock lock(*this, m_lock); - return std::ranges::any_of(FindImagesInRegion(address, size, true), [&](ImageId id) { - const auto owner = ResolveOwner(id); - return owner != nullptr && !owner->depth_id; - }); -} - void TextureCache::UnmapMemory(uint64_t address, uint64_t size) { if (!GuestRange {address, size}.Valid()) { EXIT("TextureCache: invalid unmap range\n"); diff --git a/src/graphics/host_gpu/renderer/cache/textureCache.h b/src/graphics/host_gpu/renderer/cache/textureCache.h index 03af6c6..e9964a7 100644 --- a/src/graphics/host_gpu/renderer/cache/textureCache.h +++ b/src/graphics/host_gpu/renderer/cache/textureCache.h @@ -76,11 +76,9 @@ public: [[nodiscard]] bool ClearMeta(uint64_t address); [[nodiscard]] bool TouchMeta(uint64_t address, uint32_t slice, bool is_clear); - [[nodiscard]] bool InvalidateMemory(PageFaultAccess access, uint64_t address, uint64_t size, - PageFaultPhase phase) noexcept; - void UnmapMemory(uint64_t address, uint64_t size); - void ProcessDownloadImages(); - void RunGarbageCollector(); + void UnmapMemory(uint64_t address, uint64_t size); + void ProcessDownloadImages(); + void RunGarbageCollector(); private: enum class TransferDirection { Upload, Download }; diff --git a/src/kernel/memory.cpp b/src/kernel/memory.cpp index bd5ed1f..c16b192 100644 --- a/src/kernel/memory.cpp +++ b/src/kernel/memory.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include @@ -805,15 +806,15 @@ private: Common::Mutex m_mutex; }; -static PhysicalMemory* g_physical_memory = nullptr; -static FlexibleMemory* g_flexible_memory = nullptr; -static PooledMemory* g_pooled_memory = nullptr; -static VirtualRanges* g_virtual_ranges = nullptr; -static GuestAddressSpace* g_guest_address_space = nullptr; -static callback_func_t g_alloc_callback = nullptr; -static callback_func_t g_free_callback = nullptr; -static std::atomic g_memory_pool_committed = 0; -static void MemoryPoolSubtractCommitted(uint64_t len); +static std::unique_ptr g_physical_memory; +static std::unique_ptr g_flexible_memory; +static std::unique_ptr g_pooled_memory; +static std::unique_ptr g_virtual_ranges; +static std::unique_ptr g_guest_address_space; +static callback_func_t g_alloc_callback = nullptr; +static callback_func_t g_free_callback = nullptr; +static std::atomic g_memory_pool_committed = 0; +static void MemoryPoolSubtractCommitted(uint64_t len); // Keep host mappings, physical blocks, placeholders, and virtual ranges in step. static std::recursive_mutex g_memory_operation_mutex; @@ -970,11 +971,11 @@ static bool ReplaceFixedRangeWithReserved(uint64_t start, uint64_t size); KYTY_SUBSYSTEM_INIT(Memory) { g_flexible_memory_size_frozen = true; VirtualMemory::Init(); - g_guest_address_space = new GuestAddressSpace(PhysicalMemory::TotalSize()); - g_physical_memory = new PhysicalMemory; - g_flexible_memory = new FlexibleMemory; - g_pooled_memory = new PooledMemory; - g_virtual_ranges = new VirtualRanges; + g_guest_address_space = std::make_unique(PhysicalMemory::TotalSize()); + g_physical_memory = std::make_unique(); + g_flexible_memory = std::make_unique(); + g_pooled_memory = std::make_unique(); + g_virtual_ranges = std::make_unique(); EXIT_IF(!g_guest_address_space->SelfTest()); EXIT_IF(!SelfTestSub64SharedPlaceholderAlias()); } @@ -982,16 +983,11 @@ KYTY_SUBSYSTEM_INIT(Memory) { KYTY_SUBSYSTEM_UNEXPECTED_SHUTDOWN(Memory) {} KYTY_SUBSYSTEM_DESTROY(Memory) { - delete g_pooled_memory; - g_pooled_memory = nullptr; - delete g_flexible_memory; - g_flexible_memory = nullptr; - delete g_physical_memory; - g_physical_memory = nullptr; - delete g_virtual_ranges; - g_virtual_ranges = nullptr; - delete g_guest_address_space; - g_guest_address_space = nullptr; + g_pooled_memory.reset(); + g_flexible_memory.reset(); + g_physical_memory.reset(); + g_virtual_ranges.reset(); + g_guest_address_space.reset(); } struct AlignedPos { diff --git a/tests/MemoryTrackerTests.cpp b/tests/MemoryTrackerTests.cpp index 818d206..5177a66 100644 --- a/tests/MemoryTrackerTests.cpp +++ b/tests/MemoryTrackerTests.cpp @@ -1,15 +1,12 @@ -#include "common/assert.h" #include "common/virtualMemory.h" #include "graphics/host_gpu/memoryTracker.h" #include "graphics/host_gpu/rangeSet.h" -#include #include #include #include #include #include -#include #include #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS @@ -20,1175 +17,425 @@ #undef min #undef max #else -#include #include #include #include -#include #include #endif namespace { -using Libs::Graphics::CpuFaultAction; -using Libs::Graphics::DirtySource; using Libs::Graphics::MemoryTracker; -using Libs::Graphics::PageFaultAccess; -using Libs::Graphics::PageFaultPhase; using Libs::Graphics::PageManager; using Libs::Graphics::PageWatchMode; using Libs::Graphics::RangeSet; -using Libs::Graphics::RegionManager; -void Check(bool value, const char* text) { - if (!value) { - std::fprintf(stderr, "MemoryTrackerTests: failed: %s\n", text); - std::abort(); - } +void Check(bool value, const char *text) { + if (!value) { + std::fprintf(stderr, "MemoryTrackerTests: failed: %s\n", text); + std::abort(); + } } #if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS -// POSIX shims for the shared test body. -using DWORD = uint32_t; -constexpr uint32_t PAGE_NOACCESS = 1; -constexpr uint32_t PAGE_READONLY = 2; +using DWORD = uint32_t; +constexpr uint32_t PAGE_NOACCESS = 1; +constexpr uint32_t PAGE_READONLY = 2; constexpr uint32_t PAGE_READWRITE = 3; -constexpr uint32_t MEM_RESERVE = 0; -constexpr uint32_t MEM_COMMIT = 0; -constexpr uint32_t MEM_RELEASE = 0; +constexpr uint32_t MEM_RESERVE = 0; +constexpr uint32_t MEM_COMMIT = 0; +constexpr uint32_t MEM_RELEASE = 0; int ToHostProt(uint32_t protection) { - switch (protection) { - case PAGE_NOACCESS: return PROT_NONE; - case PAGE_READONLY: return PROT_READ; - default: return PROT_READ | PROT_WRITE; - } + switch (protection) { + case PAGE_NOACCESS: + return PROT_NONE; + case PAGE_READONLY: + return PROT_READ; + default: + return PROT_READ | PROT_WRITE; + } } -uint32_t Protection(const void* address) { - const auto addr = reinterpret_cast(address); - std::FILE* maps = std::fopen("/proc/self/maps", "r"); - Check(maps != nullptr, "open /proc/self/maps failed"); - char line[512]; - uint32_t result = 0; // 0 => not mapped at all - while (std::fgets(line, sizeof(line), maps) != nullptr) { - unsigned long start = 0; - unsigned long end = 0; - char perms[8] {}; - if (std::sscanf(line, "%lx-%lx %7s", &start, &end, perms) != 3) { - continue; - } - if (addr >= start && addr < end) { - result = perms[1] == 'w' ? PAGE_READWRITE - : perms[0] == 'r' ? PAGE_READONLY - : PAGE_NOACCESS; - break; - } - } - std::fclose(maps); - return result; +uint32_t Protection(const void *address) { + const auto addr = reinterpret_cast(address); + std::FILE *maps = std::fopen("/proc/self/maps", "r"); + Check(maps != nullptr, "open /proc/self/maps failed"); + char line[512]; + uint32_t result = 0; + while (std::fgets(line, sizeof(line), maps) != nullptr) { + unsigned long start = 0; + unsigned long end = 0; + char perms[8]{}; + if (std::sscanf(line, "%lx-%lx %7s", &start, &end, perms) != 3) { + continue; + } + if (addr >= start && addr < end) { + result = perms[1] == 'w' ? PAGE_READWRITE + : perms[0] == 'r' ? PAGE_READONLY + : PAGE_NOACCESS; + break; + } + } + std::fclose(maps); + return result; } -bool IsWritable(const void* address) { - return Protection(address) == PAGE_READWRITE; +std::map &AllocationSizes() { + static std::map sizes; + return sizes; } -// munmap needs a length where VirtualFree's callers pass 0, so sizes are remembered here. -std::map& AllocationSizes() { - static std::map sizes; - return sizes; +void *VirtualAlloc(void *address, size_t size, DWORD, uint32_t protection) { + const int extra = address != nullptr ? MAP_FIXED_NOREPLACE : 0; + void *raw = ::mmap(address, size, ToHostProt(protection), + MAP_PRIVATE | MAP_ANONYMOUS | extra, -1, 0); + if (raw == MAP_FAILED) { + return nullptr; + } + AllocationSizes()[raw] = size; + return raw; } -void* VirtualAlloc(void* address, size_t size, DWORD /*type*/, uint32_t protection) { - const int extra = address != nullptr ? MAP_FIXED_NOREPLACE : 0; - void* raw = - ::mmap(address, size, ToHostProt(protection), MAP_PRIVATE | MAP_ANONYMOUS | extra, -1, 0); - if (raw == MAP_FAILED) { - return nullptr; - } - AllocationSizes()[raw] = size; - return raw; +int VirtualFree(void *address, size_t, DWORD) { + auto &sizes = AllocationSizes(); + auto it = sizes.find(address); + if (it == sizes.end()) { + return 0; + } + const int ok = ::munmap(address, it->second) == 0 ? 1 : 0; + sizes.erase(it); + return ok; } -int VirtualFree(void* address, size_t /*size*/, DWORD /*type*/) { - auto& sizes = AllocationSizes(); - auto it = sizes.find(address); - if (it == sizes.end()) { - return 0; - } - const int ok = ::munmap(address, it->second) == 0 ? 1 : 0; - sizes.erase(it); - return ok; +int VirtualProtect(void *address, size_t size, uint32_t protection, + DWORD *old_protection) { + if (old_protection != nullptr) { + *old_protection = Protection(address); + } + return ::mprotect(address, size, ToHostProt(protection)) == 0 ? 1 : 0; } - -[[maybe_unused]] int VirtualProtect(void* address, size_t size, uint32_t protection, - DWORD* old_protection) { - if (old_protection != nullptr) { - *old_protection = Protection(address); - } - return ::mprotect(address, size, ToHostProt(protection)) == 0 ? 1 : 0; +#else +uint32_t Protection(const void *address) { + MEMORY_BASIC_INFORMATION info{}; + Check(VirtualQuery(address, &info, sizeof(info)) != 0, "VirtualQuery failed"); + return info.Protect; } - -// Two views of one anonymous shared object. -class SharedPage final { -public: - SharedPage(uintptr_t address, uint64_t size): size_(static_cast(size)) { - fd_ = static_cast(::syscall(SYS_memfd_create, "kyty-shared-page", 0u)); - Check(fd_ >= 0, "memfd_create failed"); - Check(::ftruncate(fd_, static_cast(size)) == 0, "ftruncate failed"); - guest = static_cast(::mmap(reinterpret_cast(address), size_, - PROT_READ | PROT_WRITE, - MAP_SHARED | MAP_FIXED_NOREPLACE, fd_, 0)); - Check(guest == reinterpret_cast(address), "fixed shared view failed"); - backing = static_cast( - ::mmap(nullptr, size_, PROT_READ | PROT_WRITE, MAP_SHARED, fd_, 0)); - Check(backing != MAP_FAILED && backing != nullptr, "shared backing view failed"); - } - - ~SharedPage() { - Check(::munmap(backing, size_) == 0, "shared backing unmap failed"); - Check(::munmap(guest, size_) == 0, "shared guest unmap failed"); - Check(::close(fd_) == 0, "shared mapping close failed"); - } - - SharedPage(const SharedPage&) = delete; - SharedPage& operator=(const SharedPage&) = delete; - - uint8_t* guest = nullptr; - uint8_t* backing = nullptr; - -private: - size_t size_ = 0; - int fd_ = -1; -}; #endif -#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS -bool IsWritable(const void* address) { - MEMORY_BASIC_INFORMATION info {}; - Check(VirtualQuery(address, &info, sizeof(info)) != 0, "VirtualQuery failed"); - return info.Protect == PAGE_READWRITE; +bool IsWritable(const void *address) { + return Protection(address) == PAGE_READWRITE; } -uint32_t Protection(const void* address) { - MEMORY_BASIC_INFORMATION info {}; - Check(VirtualQuery(address, &info, sizeof(info)) != 0, "VirtualQuery failed"); - return info.Protect; -} - -class SharedPage final { -public: - SharedPage(uintptr_t address, uint64_t size) { - mapping_ = - CreateFileMappingA(INVALID_HANDLE_VALUE, nullptr, PAGE_READWRITE, - static_cast(size >> 32u), static_cast(size), nullptr); - Check(mapping_ != nullptr, "CreateFileMapping failed"); - guest = static_cast(MapViewOfFileEx(mapping_, FILE_MAP_ALL_ACCESS, 0, 0, size, - reinterpret_cast(address))); - Check(guest == reinterpret_cast(address), "fixed shared view failed"); - backing = static_cast(MapViewOfFile(mapping_, FILE_MAP_ALL_ACCESS, 0, 0, size)); - Check(backing != nullptr, "shared backing view failed"); - } - - ~SharedPage() { - Check(UnmapViewOfFile(backing) != 0, "shared backing unmap failed"); - Check(UnmapViewOfFile(guest) != 0, "shared guest unmap failed"); - Check(CloseHandle(mapping_) != 0, "shared mapping close failed"); - } - - SharedPage(const SharedPage&) = delete; - SharedPage& operator=(const SharedPage&) = delete; - - uint8_t* guest = nullptr; - uint8_t* backing = nullptr; - -private: - HANDLE mapping_ = nullptr; -}; -#endif - -bool ProtectAddressSpace(uint64_t vaddr, uint64_t size, Common::VirtualMemory::Mode mode) { - uint32_t protection = PAGE_NOACCESS; - if (mode == Common::VirtualMemory::Mode::Read) { - protection = PAGE_READONLY; - } else if (mode == Common::VirtualMemory::Mode::ReadWrite) { - protection = PAGE_READWRITE; - } - DWORD old_protection = 0; - return VirtualProtect(reinterpret_cast(vaddr), size, protection, &old_protection) != 0; -} - -#if 1 - -bool DummyFault(void*, PageFaultAccess, uint64_t, uint64_t, PageFaultPhase) noexcept { - return true; +bool ProtectAddressSpace(uint64_t vaddr, uint64_t size, + Common::VirtualMemory::Mode mode) { + uint32_t protection = PAGE_NOACCESS; + if (mode == Common::VirtualMemory::Mode::Read) { + protection = PAGE_READONLY; + } else if (mode == Common::VirtualMemory::Mode::ReadWrite) { + protection = PAGE_READWRITE; + } + DWORD old_protection = 0; + return VirtualProtect(reinterpret_cast(vaddr), size, protection, + &old_protection) != 0; } struct TrackerHarness { - static bool Fault(void* context, PageFaultAccess access, uint64_t vaddr, uint64_t size, - PageFaultPhase phase) noexcept { - auto* self = static_cast(context); - if (self == nullptr || self->target == nullptr) { - EXIT("memory-tracker test fault has no target\n"); - } - return self->discard_virtual - ? self->target->InvalidateVirtualGpuWrite(access, vaddr, size, phase) - : self->target->InvalidateRegion(vaddr, size, phase); - } + explicit TrackerHarness( + PageWatchMode gpu_watch_mode = PageWatchMode::ReadWrite) + : tracker(page_manager, gpu_watch_mode) {} - explicit TrackerHarness(PageWatchMode gpu_watch_mode = PageWatchMode::ReadWrite) - : page_manager(Fault, this), tracker(page_manager, gpu_watch_mode) { - target = &tracker; - } - - MemoryTracker* target = nullptr; - bool discard_virtual = false; - PageManager page_manager; - MemoryTracker tracker; + PageManager page_manager; + MemoryTracker tracker; }; -struct SharedTrackerHarness { - static bool Fault(void* context, PageFaultAccess, uint64_t vaddr, uint64_t size, - PageFaultPhase phase) noexcept { - auto* self = static_cast(context); - if (self == nullptr) { - EXIT("shared memory-tracker test fault has no harness\n"); - } - const bool first = self->first.InvalidateRegion(vaddr, size, phase); - const bool second = self->second.InvalidateRegion(vaddr, size, phase); - return first || second; - } - - SharedTrackerHarness(): page_manager(Fault, this), first(page_manager), second(page_manager) {} - - PageManager page_manager; - MemoryTracker first; - MemoryTracker second; -}; - -struct SharedMetadataImageHarness { - static bool Fault(void* context, PageFaultAccess access, uint64_t vaddr, uint64_t size, - PageFaultPhase phase) noexcept { - auto* self = static_cast(context); - if (self == nullptr || - (access != PageFaultAccess::Read && access != PageFaultAccess::Write)) { - EXIT("shared metadata/image test received an invalid fault\n"); - } - const bool metadata = access == PageFaultAccess::Write && - self->metadata.InvalidateVirtualGpuWrite(access, vaddr, size, phase); - const bool image = self->image.InvalidateRegion(vaddr, size, phase); - return metadata || image; - } - - SharedMetadataImageHarness() - : page_manager(Fault, this), image(page_manager), - metadata(page_manager, PageWatchMode::Write) {} - - PageManager page_manager; - MemoryTracker image; - MemoryTracker metadata; -}; - -struct SplitTrackerHarness { - static bool Fault(void* context, PageFaultAccess access, uint64_t vaddr, uint64_t size, - PageFaultPhase phase) noexcept { - auto* self = static_cast(context); - if (self == nullptr) { - EXIT("split memory-tracker test fault has no harness\n"); - } - if (phase == PageFaultPhase::Release) { - return true; - } - const bool buffer = self->buffer.InvalidateRegion(vaddr, size, phase); - const bool metadata = self->metadata.InvalidateVirtualGpuWrite(access, vaddr, size, phase); - const bool image = self->image.InvalidateRegion(vaddr, size, phase); - if (static_cast(buffer) + static_cast(metadata) + - static_cast(image) != - 1) { - EXIT("split memory-tracker fault matched multiple owners\n"); - } - return true; - } - - SplitTrackerHarness() - : page_manager(Fault, this), buffer(page_manager), image(page_manager), - metadata(page_manager, PageWatchMode::Write) {} - - PageManager page_manager; - MemoryTracker buffer; - MemoryTracker image; - MemoryTracker metadata; -}; - -struct DownloadTrackerHarness { - static bool Fault(void* context, PageFaultAccess access, uint64_t vaddr, uint64_t size, - PageFaultPhase phase) noexcept { - auto* self = static_cast(context); - if (self == nullptr) { - EXIT("download memory-tracker test fault has no harness\n"); - } - if (phase == PageFaultPhase::Invalidate) { - if (self->pending_access != PageFaultAccess::Unknown) { - EXIT("download memory-tracker test has an overlapping request\n"); - } - const auto action = self->tracker.BeginCpuFault(vaddr, size); - if (action == CpuFaultAction::Download) { - self->pending_access = access; - } - return action != CpuFaultAction::Untracked; - } - if (phase == PageFaultPhase::Release) { - return true; - } - const bool downloaded = self->pending_access != PageFaultAccess::Unknown; - if (downloaded) { - if (self->pending_access != access || self->download_data.empty()) { - EXIT("download memory-tracker test has invalid completion state\n"); - } - if (self->backing == nullptr || self->download_address < self->guest_address) { - EXIT("download memory-tracker test has no backing alias\n"); - } - std::memcpy(self->backing + self->download_address - self->guest_address, - self->download_data.data(), self->download_data.size()); - } - const bool completed = self->tracker.CompleteCpuFault(vaddr, size, access, downloaded); - self->pending_access = PageFaultAccess::Unknown; - return completed; - } - - DownloadTrackerHarness(): page_manager(Fault, this), tracker(page_manager) {} - - PageFaultAccess pending_access = PageFaultAccess::Unknown; - uint64_t download_address = 0; - uint64_t guest_address = 0; - uint8_t* backing = nullptr; - std::vector download_data; - PageManager page_manager; - MemoryTracker tracker; -}; - -std::atomic g_native_page_manager {nullptr}; -std::atomic_bool g_native_fault_entered {false}; - -#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS -LONG CALLBACK NativeTrackerFaultHandler(EXCEPTION_POINTERS* exception) { - if (exception == nullptr || exception->ExceptionRecord == nullptr || - exception->ExceptionRecord->ExceptionCode != EXCEPTION_ACCESS_VIOLATION) { - return EXCEPTION_CONTINUE_SEARCH; - } - const auto operation = exception->ExceptionRecord->ExceptionInformation[0]; - const auto access = operation == 0 ? PageFaultAccess::Read - : operation == 1 ? PageFaultAccess::Write - : operation == 8 ? PageFaultAccess::Execute - : PageFaultAccess::Unknown; - auto* page_manager = g_native_page_manager.load(std::memory_order_acquire); - if (page_manager == nullptr) { - return EXCEPTION_CONTINUE_SEARCH; - } - g_native_fault_entered.store(true, std::memory_order_release); - return page_manager->HandleFault(access, exception->ExceptionRecord->ExceptionInformation[1]) - ? EXCEPTION_CONTINUE_EXECUTION - : EXCEPTION_CONTINUE_SEARCH; -} -#else -// SIGSEGV stands in for the vectored exception handler. -void NativeTrackerFaultHandler(int signal_number, siginfo_t* info, void* native_context) { - auto* context = static_cast(native_context); - const auto error_code = static_cast(context->uc_mcontext.gregs[REG_ERR]); - const auto access = (error_code & 0x10u) != 0 ? PageFaultAccess::Execute - : (error_code & 0x02u) != 0 ? PageFaultAccess::Write - : PageFaultAccess::Read; - auto* page_manager = g_native_page_manager.load(std::memory_order_acquire); - if (page_manager != nullptr) { - g_native_fault_entered.store(true, std::memory_order_release); - if (page_manager->HandleFault(access, reinterpret_cast(info->si_addr))) { - return; - } - } - struct sigaction restore {}; - restore.sa_handler = SIG_DFL; - sigemptyset(&restore.sa_mask); - ::sigaction(signal_number, &restore, nullptr); +uint8_t *Allocate(PageManager &manager, uint64_t pages) { + constexpr uintptr_t base = 0x0000000200010000ull; + const auto size = manager.GetPageSize() * pages; + auto *memory = static_cast( + VirtualAlloc(reinterpret_cast(base), size, + MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); + Check(memory == reinterpret_cast(base), "fixed VirtualAlloc failed"); + manager.OnGpuMap(base, size); + return memory; } -struct sigaction g_saved_segv_action {}; - -void* AddVectoredExceptionHandler(unsigned long /*first*/, - void (*handler)(int, siginfo_t*, void*)) { - struct sigaction action {}; - action.sa_sigaction = handler; - sigemptyset(&action.sa_mask); - action.sa_flags = SA_SIGINFO; - if (::sigaction(SIGSEGV, &action, &g_saved_segv_action) != 0) { - return nullptr; - } - return reinterpret_cast(handler); -} - -int RemoveVectoredExceptionHandler(void* /*token*/) { - return ::sigaction(SIGSEGV, &g_saved_segv_action, nullptr) == 0 ? 1 : 0; -} -#endif - -void TestPendingFaultBlocksUploadConsumption() { - constexpr uintptr_t base = 0x0000000200010000ull; - constexpr uint64_t region_size = 4ull * 1024ull * 1024ull; - PageManager page_manager(DummyFault, nullptr); - const auto page_size = page_manager.GetPageSize(); - auto* memory = static_cast(VirtualAlloc(reinterpret_cast(base), page_size, - MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); - Check(memory == reinterpret_cast(base), "fixed VirtualAlloc failed"); - const auto address = reinterpret_cast(memory); - page_manager.OnGpuMap(address, page_size); - RegionManager region(page_manager, address & ~(region_size - 1)); - - Libs::Graphics::RegionBits changed; - { - std::scoped_lock lock(region.lock); - region.Track(address, page_size); - changed = region.ForEachModifiedRange( - address, page_size, [](uint64_t, uint64_t) noexcept {}); - changed = region.ChangeState(address, page_size); - Check(region.BeginCpuFault(address, 1) == CpuFaultAction::Continue, - "fault ownership rejected an already CPU-dirty page"); - } - page_manager.UpdatePageWatchers(false, address, page_size); - - uint32_t ranges = 0; - { - std::scoped_lock lock(region.lock); - const auto pending_change = region.ForEachModifiedRange( - address, page_size, [&](uint64_t, uint64_t) noexcept { ranges++; }); - Check(ranges == 0 && pending_change.none() && - region.IsModified(address - region.GetCpuAddr(), page_size), - "pending fault page was consumed by upload"); - Check(region.CompleteCpuFault(address, 1, PageFaultAccess::Write, false), - "fault completion was not recorded"); - changed = region.ForEachModifiedRange( - address, page_size, [&](uint64_t, uint64_t) noexcept { ranges++; }); - } - Check(ranges == 1, "completed fault page was not available to upload"); - { - std::scoped_lock lock(region.lock); - changed = region.ChangeState(address, page_size); - } - region.ApplyProtection(changed, false); - page_manager.OnGpuUnmap(address, page_size); - Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); -} - -void TestCleanReadFaultPreservesCpuState() { - constexpr uintptr_t base = 0x0000000200010000ull; - PageManager page_manager(DummyFault, nullptr); - MemoryTracker tracker(page_manager); - const auto page_size = page_manager.GetPageSize(); - auto* memory = static_cast(VirtualAlloc(reinterpret_cast(base), page_size, - MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); - Check(memory == reinterpret_cast(base), "fixed VirtualAlloc failed"); - const auto address = reinterpret_cast(memory); - page_manager.OnGpuMap(address, page_size); - tracker.ForEachUploadRange( - address, page_size, false, [](uint64_t, uint64_t) noexcept {}, []() noexcept {}); - Check(!tracker.IsRegionCpuModified(address, page_size) && - !tracker.IsRegionGpuModified(address, page_size), - "clean read-fault setup retained dirty ownership"); - Check(tracker.BeginCpuFault(address, 1, PageFaultAccess::Read) == CpuFaultAction::Continue, - "clean tracked read fault was not accepted"); - Check(tracker.CompleteCpuFault(address, 1, PageFaultAccess::Read, false), - "clean tracked read fault did not complete"); - Check(!tracker.IsRegionCpuModified(address, page_size) && - !tracker.IsRegionGpuModified(address, page_size), - "clean read fault incorrectly transferred write ownership to the CPU"); - tracker.UntrackMemory(address, page_size); - page_manager.OnGpuUnmap(address, page_size); - Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); -} - -void TestGpuDownloadFaultOwnership() { - constexpr uintptr_t base = 0x0000000200010000ull; - DownloadTrackerHarness harness; - const auto page_size = harness.page_manager.GetPageSize(); - SharedPage shared(base, page_size); - auto* memory = shared.guest; - const auto address = reinterpret_cast(memory); - harness.guest_address = address; - harness.backing = shared.backing; - harness.page_manager.OnGpuMap(address, page_size); - harness.tracker.ForEachUploadRange( - address, page_size, true, [](uint64_t, uint64_t) noexcept {}, []() noexcept {}); - harness.download_address = address + 32; - harness.download_data = {0x11, 0x22, 0x33, 0x44}; - Check(harness.page_manager.HandleFault(PageFaultAccess::Read, address + 32), - "GPU-dirty read fault was not handled"); - Check(std::memcmp(memory + 32, harness.download_data.data(), 4) == 0 && - !harness.tracker.IsRegionGpuModified(address, page_size) && - !harness.tracker.IsRegionCpuModified(address, page_size) && - Protection(memory) == PAGE_READONLY && harness.page_manager.IsTracked(address), - "GPU readback did not leave a clean write-watched page"); - - Check(harness.page_manager.HandleFault(PageFaultAccess::Write, address + 32), - "write watcher was not preserved after GPU readback"); - Check(harness.tracker.IsRegionCpuModified(address, page_size) && IsWritable(memory), - "post-read CPU write did not claim CPU ownership"); - - harness.tracker.ForEachUploadRange( - address, page_size, true, [](uint64_t, uint64_t) noexcept {}, []() noexcept {}); - harness.download_data = {0xaa, 0xbb, 0xcc, 0xdd}; - Check(harness.page_manager.HandleFault(PageFaultAccess::Write, address + 33), - "GPU-dirty write fault was not handled"); - Check(std::memcmp(memory + 32, harness.download_data.data(), 4) == 0 && - !harness.tracker.IsRegionGpuModified(address, page_size) && - harness.tracker.IsRegionCpuModified(address, page_size) && IsWritable(memory), - "GPU write fault did not download before granting CPU ownership"); - harness.tracker.UntrackMemory(address, page_size); -} - -void TestVirtualGpuWriteDiscard() { - constexpr uintptr_t base = 0x0000000200010000ull; - TrackerHarness harness(PageWatchMode::Write); - harness.discard_virtual = true; - const auto page_size = harness.page_manager.GetPageSize(); - auto* memory = static_cast(VirtualAlloc(reinterpret_cast(base), page_size, - MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); - Check(memory == reinterpret_cast(base), "fixed VirtualAlloc failed"); - const auto address = reinterpret_cast(memory); - memory[8] = 0x5a; - harness.page_manager.OnGpuMap(address, page_size); - harness.tracker.ForEachUploadRange( - address, page_size, true, [](uint64_t, uint64_t) noexcept {}, []() noexcept {}); - Check(Protection(memory) == PAGE_READONLY && memory[8] == 0x5a && - harness.tracker.IsRegionGpuModified(address, page_size), - "virtual GPU ownership did not preserve authoritative backing reads"); - Check(harness.page_manager.HandleFault(PageFaultAccess::Write, address + 8), - "virtual GPU write fault was not discarded"); - Check(!harness.tracker.IsRegionGpuModified(address, page_size) && - harness.tracker.IsRegionCpuModified(address, page_size) && IsWritable(memory), - "virtual GPU discard did not transfer the page to CPU ownership"); - harness.tracker.UntrackMemory(address, page_size); - Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); -} - -void TestSameSlabTrackerArbitration() { - constexpr uintptr_t base = 0x0000000200010000ull; - SplitTrackerHarness harness; - const auto page_size = harness.page_manager.GetPageSize(); - auto* memory = static_cast(VirtualAlloc(reinterpret_cast(base), page_size * 3, - MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); - Check(memory == reinterpret_cast(base), "fixed VirtualAlloc failed"); - const auto address = reinterpret_cast(memory); - harness.page_manager.OnGpuMap(address, page_size * 3); - - harness.buffer.ForEachUploadRange( - address, page_size, false, [](uint64_t, uint64_t) noexcept {}, []() noexcept {}); - harness.image.ForEachUploadRange( - address + page_size, page_size, false, [](uint64_t, uint64_t) noexcept {}, - []() noexcept {}); - harness.metadata.ForEachUploadRange( - address + page_size * 2, page_size, true, [](uint64_t, uint64_t) noexcept {}, - []() noexcept {}); - - Check(harness.buffer.BeginCpuFault(address + page_size * 2, 1) == CpuFaultAction::Untracked && - harness.image.BeginCpuFault(address + page_size * 2, 1) == CpuFaultAction::Untracked, - "unrelated same-slab trackers claimed the metadata page"); - Check(harness.page_manager.HandleFault(PageFaultAccess::Write, address + page_size * 2 + 8), - "metadata write fault was not exclusively handled"); - Check(harness.page_manager.HandleFault(PageFaultAccess::Write, address + 8), - "buffer write fault was not exclusively handled"); - Check(harness.page_manager.HandleFault(PageFaultAccess::Write, address + page_size + 8), - "image write fault was not exclusively handled"); - Check(harness.buffer.IsRegionCpuModified(address, page_size) && - harness.image.IsRegionCpuModified(address + page_size, page_size) && - harness.metadata.IsRegionCpuModified(address + page_size * 2, page_size) && - IsWritable(memory) && IsWritable(memory + page_size) && - IsWritable(memory + page_size * 2), - "exclusive same-slab faults did not transfer exact CPU ownership"); - - harness.buffer.UntrackMemory(address, page_size); - harness.image.UntrackMemory(address + page_size, page_size); - harness.metadata.UntrackMemory(address + page_size * 2, page_size); - harness.page_manager.OnGpuUnmap(address, page_size * 3); - Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); -} - -void TestSharedMetadataAndImagePageFault() { - constexpr uintptr_t base = 0x0000000200010000ull; - SharedMetadataImageHarness harness; - const auto page_size = harness.page_manager.GetPageSize(); - auto* memory = static_cast(VirtualAlloc(reinterpret_cast(base), page_size, - MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); - Check(memory == reinterpret_cast(base), "fixed VirtualAlloc failed"); - const auto address = reinterpret_cast(memory); - harness.page_manager.OnGpuMap(address, page_size); - harness.metadata.ForEachUploadRange( - address, page_size, true, [](uint64_t, uint64_t) noexcept {}, []() noexcept {}); - harness.image.ForEachUploadRange( - address, page_size, false, [](uint64_t, uint64_t) noexcept {}, []() noexcept {}); - Check(Protection(memory) == PAGE_READONLY && - harness.metadata.IsRegionGpuModified(address, page_size) && - !harness.image.IsRegionCpuModified(address, page_size), - "shared metadata/image page was not write-watched"); - Check(harness.page_manager.HandleFault(PageFaultAccess::Write, address + 8), - "shared metadata/image write fault was not handled"); - Check(!harness.metadata.IsRegionGpuModified(address, page_size) && - harness.metadata.IsRegionCpuModified(address, page_size) && - harness.image.IsRegionCpuModified(address, page_size) && IsWritable(memory), - "shared write fault did not invalidate both native trackers"); - harness.metadata.UntrackMemory(address, page_size); - harness.image.UntrackMemory(address, page_size); - harness.page_manager.OnGpuUnmap(address, page_size); - Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); +void Release(PageManager &manager, uint8_t *memory, uint64_t size) { + const auto address = reinterpret_cast(memory); + manager.OnGpuUnmap(address, size); + Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); } void TestRangeSet() { - RangeSet ranges; - ranges.Add(0x1000, 0x80); - ranges.Add(0x1080, 0x80); - ranges.Add(0x1200, 0x40); - Check(ranges.Contains(0x1010, 0xe0) && !ranges.Contains(0x1010, 0x200), - "range set containment did not require full coverage"); - auto intersections = ranges.Intersections(0x1070, 0x1b0); - Check(intersections.size() == 2 && intersections[0].address == 0x1070 && - intersections[0].size == 0x90 && intersections[1].address == 0x1200 && - intersections[1].size == 0x20, - "range set did not merge and intersect exact byte ranges"); - ranges.Subtract(0x1040, 0x1e0); - intersections = ranges.Intersections(0x1000, 0x300); - Check(intersections.size() == 2 && intersections[0].address == 0x1000 && - intersections[0].size == 0x40 && intersections[1].address == 0x1220 && - intersections[1].size == 0x20, - "range set subtraction did not preserve both exact tails"); + RangeSet ranges; + ranges.Add(0x1000, 0x80); + ranges.Add(0x1080, 0x80); + ranges.Add(0x1200, 0x40); + Check(ranges.Contains(0x1010, 0xe0) && !ranges.Contains(0x1010, 0x200), + "range set containment did not require full coverage"); + auto intersections = ranges.Intersections(0x1070, 0x1b0); + Check(intersections.size() == 2 && intersections[0].address == 0x1070 && + intersections[0].size == 0x90 && + intersections[1].address == 0x1200 && intersections[1].size == 0x20, + "range set did not merge and intersect exact byte ranges"); + ranges.Subtract(0x1040, 0x1e0); + intersections = ranges.Intersections(0x1000, 0x300); + Check(intersections.size() == 2 && intersections[0].address == 0x1000 && + intersections[0].size == 0x40 && + intersections[1].address == 0x1220 && intersections[1].size == 0x20, + "range set subtraction did not preserve both exact tails"); } void TestQueriesDoNotRequireMappedOwnership() { - constexpr uint64_t address = 0x0000000203000000ull; - TrackerHarness harness; - const auto page_size = harness.page_manager.GetPageSize(); - Check(harness.tracker.IsRegionCpuModified(address, page_size) && - !harness.tracker.IsRegionGpuModified(address, page_size), - "unowned tracker range did not expose its initial CPU-dirty state"); + constexpr uint64_t address = 0x0000000203000000ull; + TrackerHarness harness; + const auto page_size = harness.page_manager.GetPageSize(); + Check(harness.tracker.IsRegionCpuModified(address, page_size) && + !harness.tracker.IsRegionGpuModified(address, page_size), + "unowned tracker range did not expose its initial CPU-dirty state"); +} + +void TestCpuDirtyUpload() { + TrackerHarness harness; + auto &tracker = harness.tracker; + auto &page_manager = harness.page_manager; + const auto page_size = page_manager.GetPageSize(); + auto *memory = Allocate(page_manager, 2); + const auto address = reinterpret_cast(memory); + Check(tracker.IsRegionCpuModified(address + 16, 32), + "new region was not CPU dirty"); + + uint32_t ranges = 0; + bool uploaded = false; + tracker.ForEachUploadRange( + address + 16, 32, false, + [&](uint64_t upload_address, uint64_t upload_size) noexcept { + Check(upload_address == address && upload_size == page_size, + "upload range was not page aligned"); + ranges++; + }, + [&]() noexcept { uploaded = true; }); + Check(ranges == 1 && uploaded && + !tracker.IsRegionCpuModified(address, page_size) && + Protection(memory) == PAGE_READONLY, + "upload did not clear CPU dirty state and arm protection"); + + tracker.MarkRegionAsCpuModified(address + 16, 32); + Check(tracker.IsRegionCpuModified(address, page_size) && IsWritable(memory), + "explicit CPU dirtiness did not release write protection"); + tracker.UntrackMemory(address, page_size * 2); + Release(page_manager, memory, page_size * 2); } void TestRangeInvalidation() { - constexpr uintptr_t base = 0x0000000201000000ull; - TrackerHarness harness; - auto& tracker = harness.tracker; - auto& page_manager = harness.page_manager; - constexpr uint64_t size = Libs::Graphics::TRACKER_REGION_SIZE * 2; - auto* memory = static_cast(VirtualAlloc(reinterpret_cast(base), size, - MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); - Check(memory == reinterpret_cast(base), "range invalidation allocation failed"); - const auto address = reinterpret_cast(memory); - page_manager.OnGpuMap(address, size); + constexpr uintptr_t base = 0x0000000201000000ull; + TrackerHarness harness; + auto &tracker = harness.tracker; + auto &page_manager = harness.page_manager; + constexpr uint64_t size = Libs::Graphics::TRACKER_REGION_SIZE * 2; + auto *memory = static_cast( + VirtualAlloc(reinterpret_cast(base), size, + MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); + Check(memory == reinterpret_cast(base), + "range invalidation allocation failed"); + const auto address = reinterpret_cast(memory); + page_manager.OnGpuMap(address, size); - tracker.ForEachUploadRange( - address, size, true, [](uint64_t, uint64_t) noexcept {}, []() noexcept {}); - Check(tracker.IsRegionGpuModified(address, size) && !IsWritable(memory), - "range invalidation setup did not establish GPU ownership"); - - uint32_t flushes = 0; - tracker.InvalidateRegion(address + 16, size - 32, [&] { - flushes++; - tracker.ForEachDownloadRange(address + 16, size - 32, - [](uint64_t, uint64_t) noexcept {}); - }); - Check(flushes == 1 && !tracker.IsRegionGpuModified(address, size) && - tracker.IsRegionCpuModified(address, size) && IsWritable(memory) && - IsWritable(memory + size - 1), - "range invalidation did not batch ownership transfer across regions"); - - tracker.InvalidateRegion(address + 16, size - 32, [&] { flushes++; }); - Check(flushes == 1, "clean range invalidation unnecessarily requested a GPU flush"); - tracker.UntrackMemory(address, size); - page_manager.OnGpuUnmap(address, size); - Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "range invalidation VirtualFree failed"); -} - -void TestCpuDirtyUploadAndFault() { - constexpr uintptr_t base = 0x0000000200010000ull; - TrackerHarness harness; - auto& tracker = harness.tracker; - auto& page_manager = harness.page_manager; - const auto page_size = page_manager.GetPageSize(); - auto* memory = static_cast(VirtualAlloc(reinterpret_cast(base), page_size * 2, - MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); - Check(memory == reinterpret_cast(base), "fixed VirtualAlloc failed"); - const auto address = reinterpret_cast(memory); - page_manager.OnGpuMap(address, page_size * 2); - Check(tracker.IsRegionCpuModified(address + 16, 32), "new region was not CPU dirty"); - - uint32_t ranges = 0; - bool uploaded = false; - tracker.ForEachUploadRange( - address + 16, 32, false, - [&](uint64_t upload_addr, uint64_t upload_size) noexcept { - Check(upload_addr == address && upload_size == page_size, - "upload range was not page aligned"); - ranges++; - }, - [&]() noexcept { uploaded = true; }); - Check(ranges == 1 && uploaded && !tracker.IsRegionCpuModified(address + 16, 32) && - !IsWritable(memory), - "upload did not clear CPU dirty state and arm protection"); - - Check(page_manager.HandleFault(PageFaultAccess::Write, address + 24), - "tracked CPU write fault was not handled"); - Check(tracker.IsRegionCpuModified(address + 16, 32) && IsWritable(memory), - "fault did not restore CPU dirty state and write access"); - tracker.ForEachUploadRange( - address + 16, 32, false, [](uint64_t, uint64_t) noexcept {}, []() noexcept {}); - Check(!tracker.IsRegionCpuModified(address + 16, 32) && !IsWritable(memory), - "fault owner state did not support a balanced rearm"); - tracker.MarkRegionAsCpuModified(address + 16, 32); - Check(IsWritable(memory), "explicit CPU dirty transition did not release the rearmed watch"); - - tracker.UntrackMemory(address, page_size * 2); - Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); -} - -void TestFaultDuringUploadRemainsDirty() { - constexpr uintptr_t base = 0x0000000200010000ull; - TrackerHarness harness; - auto& tracker = harness.tracker; - auto& page_manager = harness.page_manager; - const auto page_size = page_manager.GetPageSize(); - auto* memory = static_cast(VirtualAlloc(reinterpret_cast(base), page_size, - MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); - Check(memory == reinterpret_cast(base), "fixed VirtualAlloc failed"); - const auto address = reinterpret_cast(memory); - page_manager.OnGpuMap(address, page_size); - tracker.ForEachUploadRange( - address, page_size, false, [](uint64_t, uint64_t) noexcept {}, - [&]() noexcept { - bool handled = false; - std::thread fault( - [&] { handled = page_manager.HandleFault(PageFaultAccess::Write, address); }); - fault.join(); - Check(handled, "concurrent write racing upload was not handled"); - }); - Check(tracker.IsRegionCpuModified(address, page_size) && IsWritable(memory), - "upload completion erased a racing CPU dirty transition"); - tracker.UntrackMemory(address, page_size); - Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); -} - -void TestNativeStoreDuringRangeEnumeration() { - constexpr uintptr_t base = 0x0000000200010000ull; - TrackerHarness harness; - auto& tracker = harness.tracker; - auto& page_manager = harness.page_manager; - const auto page_size = page_manager.GetPageSize(); - auto* memory = static_cast(VirtualAlloc(reinterpret_cast(base), page_size, - MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); - Check(memory == reinterpret_cast(base), "fixed VirtualAlloc failed"); - const auto address = reinterpret_cast(memory); - page_manager.OnGpuMap(address, page_size); - - void* handler = AddVectoredExceptionHandler(1, NativeTrackerFaultHandler); - Check(handler != nullptr, "AddVectoredExceptionHandler failed"); - Check(g_native_page_manager.exchange(&page_manager, std::memory_order_acq_rel) == nullptr, - "native page manager already installed"); - g_native_fault_entered.store(false, std::memory_order_release); - - std::thread writer; - tracker.ForEachUploadRange( - address, page_size, false, - [&](uint64_t, uint64_t) noexcept { - writer = std::thread([&] { *static_cast(memory) = 0x6b; }); - while (!g_native_fault_entered.load(std::memory_order_acquire)) { - std::this_thread::yield(); - } - }, - [&]() noexcept { writer.join(); }); - - Check(g_native_page_manager.exchange(nullptr, std::memory_order_acq_rel) == &page_manager, - "native page manager publication changed"); - Check(RemoveVectoredExceptionHandler(handler) != 0, "RemoveVectoredExceptionHandler failed"); - Check(memory[0] == 0x6b && tracker.IsRegionCpuModified(address, page_size) && - IsWritable(memory), - "native store during range enumeration was lost"); - - tracker.UntrackMemory(address, page_size); - Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); -} - -void TestFaultDuringDownloadSynchronization() { - constexpr uintptr_t base = 0x0000000200010000ull; - TrackerHarness harness; - auto& tracker = harness.tracker; - auto& page_manager = harness.page_manager; - const auto page_size = page_manager.GetPageSize(); - SharedPage shared(base, page_size * 3); - auto* memory = shared.guest; - const auto address = reinterpret_cast(memory); - page_manager.OnGpuMap(address, page_size * 3); - tracker.ForEachUploadRange( - address, page_size, true, [](uint64_t, uint64_t) noexcept {}, []() noexcept {}); - tracker.ForEachUploadRange( - address + page_size * 2, page_size, true, [](uint64_t, uint64_t) noexcept {}, - []() noexcept {}); - memory[page_size] = 0x31; - - void* handler = AddVectoredExceptionHandler(1, NativeTrackerFaultHandler); - Check(handler != nullptr, "AddVectoredExceptionHandler failed"); - Check(g_native_page_manager.exchange(&page_manager, std::memory_order_acq_rel) == nullptr, - "native page manager already installed"); - g_native_fault_entered.store(false, std::memory_order_release); - - uint32_t ranges = 0; - std::vector download(page_size, 0x5a); - std::thread writer; - { - PageManager::BackingWrite first(page_manager, address, page_size); - PageManager::BackingWrite third(page_manager, address + page_size * 2, page_size); - tracker.ForEachDownloadRange( - address, page_size * 3, - [&](uint64_t download_address, uint64_t download_size) noexcept { - Check( - (download_address == address || download_address == address + page_size * 2) && - download_size == page_size, - "download transaction reported the wrong range"); - ranges++; - if (download_address == address) { - writer = std::thread([&] { *static_cast(memory) = 0x7c; }); - while (!g_native_fault_entered.load(std::memory_order_acquire)) { - std::this_thread::yield(); - } - } - std::memcpy(shared.backing + download_address - address, download.data(), - download_size); - Check(Protection(reinterpret_cast(download_address)) == PAGE_NOACCESS, - "backing alias exposed the protected guest page"); - Check(Protection(memory + page_size) == PAGE_READWRITE, - "clean middle page was reserved or protected"); - }); - Check(ranges == 2 && Protection(memory) == PAGE_NOACCESS && - Protection(memory + page_size) == PAGE_READWRITE && - Protection(memory + page_size * 2) == PAGE_NOACCESS, - "download completion released protection before reservation"); - } - writer.join(); - - Check(g_native_page_manager.exchange(nullptr, std::memory_order_acq_rel) == &page_manager, - "native page manager publication changed"); - Check(RemoveVectoredExceptionHandler(handler) != 0, "RemoveVectoredExceptionHandler failed"); - Check(ranges == 2, "partial download did not enumerate both dirty ranges"); - Check(memory[0] == 0x7c && memory[1] == 0x5a && memory[page_size] == 0x31 && - memory[page_size * 2] == 0x5a, - "reserved backing write raced or lost downloaded data"); - Check(!tracker.IsRegionGpuModified(address, page_size * 3) && - tracker.IsRegionCpuModified(address, page_size * 3), - "partial download left incorrect tracker ownership"); - Check(IsWritable(memory), "first dirty page did not restore write access"); - Check(IsWritable(memory + page_size), "clean page lost write access"); - Check(Protection(memory + page_size * 2) == PAGE_READONLY && - page_manager.IsTracked(address + page_size * 2), - "uncontended dirty page did not retain its clean write watch"); - - tracker.UntrackMemory(address, page_size * 3); -} - -void TestFaultAndExplicitDirtyRace() { - constexpr uintptr_t base = 0x0000000200010000ull; - TrackerHarness harness; - auto& tracker = harness.tracker; - auto& page_manager = harness.page_manager; - const auto page_size = page_manager.GetPageSize(); - auto* memory = static_cast(VirtualAlloc(reinterpret_cast(base), page_size, - MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); - Check(memory == reinterpret_cast(base), "fixed VirtualAlloc failed"); - const auto address = reinterpret_cast(memory); - page_manager.OnGpuMap(address, page_size); - for (uint32_t iteration = 0; iteration < 64; iteration++) { - tracker.ForEachUploadRange( - address, page_size, false, [](uint64_t, uint64_t) noexcept {}, []() noexcept {}); - std::atomic_bool start {false}; - bool handled = false; - std::thread fault([&] { - while (!start.load(std::memory_order_acquire)) { - std::this_thread::yield(); - } - handled = page_manager.HandleFault(PageFaultAccess::Write, address); - }); - std::thread dirty([&] { - while (!start.load(std::memory_order_acquire)) { - std::this_thread::yield(); - } - tracker.MarkRegionAsCpuModified(address, page_size); - }); - start.store(true, std::memory_order_release); - fault.join(); - dirty.join(); - Check(handled && tracker.IsRegionCpuModified(address, page_size) && IsWritable(memory), - "fault/explicit-dirty race lost dirty state or write access"); - } - tracker.UntrackMemory(address, page_size); - Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); -} - -void TestSharedTrackersAndConcurrentPageFaults() { - constexpr uintptr_t base = 0x0000000200010000ull; - SharedTrackerHarness harness; - const auto page_size = harness.page_manager.GetPageSize(); - auto* memory = static_cast(VirtualAlloc(reinterpret_cast(base), page_size * 2, - MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); - Check(memory == reinterpret_cast(base), "fixed VirtualAlloc failed"); - const auto address = reinterpret_cast(memory); - harness.page_manager.OnGpuMap(address, page_size * 2); - - for (auto* tracker: {&harness.first, &harness.second}) { - tracker->ForEachUploadRange( - address, page_size * 2, false, [](uint64_t, uint64_t) noexcept {}, []() noexcept {}); - } - Check(!IsWritable(memory) && !IsWritable(memory + page_size), - "shared trackers did not arm both pages"); - - std::atomic_bool start {false}; - bool first_handled = false; - bool second_handled = false; - std::thread first_fault([&] { - while (!start.load(std::memory_order_acquire)) { - std::this_thread::yield(); - } - first_handled = harness.page_manager.HandleFault(PageFaultAccess::Write, address + 8); - }); - std::thread second_fault([&] { - while (!start.load(std::memory_order_acquire)) { - std::this_thread::yield(); - } - second_handled = - harness.page_manager.HandleFault(PageFaultAccess::Write, address + page_size + 8); - }); - start.store(true, std::memory_order_release); - first_fault.join(); - second_fault.join(); - - Check(first_handled && second_handled, "concurrent faults were not handled by shared trackers"); - for (auto* tracker: {&harness.first, &harness.second}) { - Check(tracker->IsRegionCpuModified(address, page_size * 2), - "a shared tracker lost concurrent CPU dirtiness"); - } - - harness.first.UntrackMemory(address, page_size * 2); - harness.second.UntrackMemory(address, page_size * 2); - Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); + tracker.ForEachUploadRange( + address, size, true, [](uint64_t, uint64_t) noexcept {}, + []() noexcept {}); + Check(tracker.IsRegionGpuModified(address, size) && !IsWritable(memory), + "range invalidation setup did not establish GPU ownership"); + uint32_t flushes = 0; + tracker.InvalidateRegion(address + 16, size - 32, [&] { + flushes++; + tracker.ForEachDownloadRange(address + 16, size - 32, + [](uint64_t, uint64_t) noexcept {}); + }); + Check(flushes == 1 && !tracker.IsRegionGpuModified(address, size) && + tracker.IsRegionCpuModified(address, size) && IsWritable(memory) && + IsWritable(memory + size - 1), + "range invalidation did not batch ownership transfer across regions"); + tracker.InvalidateRegion(address + 16, size - 32, [&] { flushes++; }); + Check(flushes == 1, + "clean range invalidation unnecessarily requested a GPU flush"); + tracker.UntrackMemory(address, size); + Release(page_manager, memory, size); } void TestGpuDirtyBits() { - constexpr uintptr_t base = 0x0000000200010000ull; - TrackerHarness harness; - auto& tracker = harness.tracker; - auto& page_manager = harness.page_manager; - const auto page_size = page_manager.GetPageSize(); - auto* memory = static_cast(VirtualAlloc(reinterpret_cast(base), page_size * 2, - MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); - Check(memory == reinterpret_cast(base), "fixed VirtualAlloc failed"); - const auto address = reinterpret_cast(memory); - page_manager.OnGpuMap(address, page_size * 2); - tracker.ForEachUploadRange( - address, page_size, true, [](uint64_t, uint64_t) noexcept {}, []() noexcept {}); - Check(tracker.IsRegionGpuModified(address, page_size) && - !tracker.IsRegionGpuModified(address + page_size, page_size) && - Protection(memory) == PAGE_NOACCESS, - "GPU dirty state escaped the requested range"); - tracker.UnmarkRegionAsGpuModified(address, page_size); - Check(!tracker.IsRegionGpuModified(address, page_size) && Protection(memory) == PAGE_READONLY, - "GPU dirty state did not restore write-only tracking"); - tracker.MarkRegionAsGpuModified(address, page_size); - Check(tracker.IsRegionGpuModified(address, page_size) && Protection(memory) == PAGE_NOACCESS, - "explicit GPU dirty transition did not trap CPU access"); - tracker.UnmarkRegionAsGpuModified(address, page_size); - tracker.MarkRegionAsCpuModified(address, page_size); - tracker.UntrackMemory(address, page_size * 2); - Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); + TrackerHarness harness; + auto &tracker = harness.tracker; + auto &page_manager = harness.page_manager; + const auto page_size = page_manager.GetPageSize(); + auto *memory = Allocate(page_manager, 2); + const auto address = reinterpret_cast(memory); + + tracker.ForEachUploadRange( + address, page_size, true, [](uint64_t, uint64_t) noexcept {}, + []() noexcept {}); + Check(tracker.IsRegionGpuModified(address, page_size) && + !tracker.IsRegionGpuModified(address + page_size, page_size) && + Protection(memory) == PAGE_NOACCESS, + "GPU dirty state escaped the requested range"); + tracker.UnmarkRegionAsGpuModified(address, page_size); + Check(!tracker.IsRegionGpuModified(address, page_size) && + Protection(memory) == PAGE_READONLY, + "GPU dirty state did not restore write-only tracking"); + tracker.MarkRegionAsCpuModified(address, page_size); + tracker.UntrackMemory(address, page_size * 2); + Release(page_manager, memory, page_size * 2); } void TestCrossRegionUpload() { - constexpr uintptr_t base = 0x0000000200010000ull; - constexpr uint64_t region_size = 4ull * 1024ull * 1024ull; - TrackerHarness harness; - auto& tracker = harness.tracker; - auto& page_manager = harness.page_manager; - const auto page_size = page_manager.GetPageSize(); - auto* memory = static_cast(VirtualAlloc( - reinterpret_cast(base), region_size * 2, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); - Check(memory == reinterpret_cast(base), "fixed VirtualAlloc failed"); - const auto address = reinterpret_cast(memory); - const auto boundary = (address + region_size - 1) & ~(region_size - 1); - page_manager.OnGpuMap(address, region_size * 2); - uint32_t ranges = 0; - tracker.ForEachUploadRange( - boundary - page_size, page_size * 2, false, [&](uint64_t, uint64_t) noexcept { ranges++; }, - []() noexcept {}); - Check(ranges == 2 && !tracker.IsRegionCpuModified(boundary - page_size, page_size * 2) && - !IsWritable(reinterpret_cast(boundary - page_size)) && - !IsWritable(reinterpret_cast(boundary)), - "cross-region upload did not clear and protect both regions"); - tracker.MarkRegionAsCpuModified(boundary - page_size, page_size * 2); - tracker.ForEachUploadRange( - boundary - page_size, page_size * 2, true, [](uint64_t, uint64_t) noexcept {}, - []() noexcept {}); - Check(tracker.IsRegionGpuModified(boundary - page_size, page_size * 2), - "cross-region written upload did not mark GPU dirty state"); - tracker.UnmarkRegionAsGpuModified(boundary - page_size, page_size * 2); - tracker.MarkRegionAsCpuModified(boundary - page_size, page_size * 2); - tracker.UntrackMemory(address, region_size * 2); - Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); + constexpr uintptr_t base = 0x0000000200010000ull; + constexpr uint64_t region_size = 4ull * 1024ull * 1024ull; + TrackerHarness harness; + auto &tracker = harness.tracker; + auto &page_manager = harness.page_manager; + const auto page_size = page_manager.GetPageSize(); + auto *memory = static_cast( + VirtualAlloc(reinterpret_cast(base), region_size * 2, + MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); + Check(memory == reinterpret_cast(base), "fixed VirtualAlloc failed"); + const auto address = reinterpret_cast(memory); + const auto boundary = (address + region_size - 1) & ~(region_size - 1); + page_manager.OnGpuMap(address, region_size * 2); + uint32_t ranges = 0; + tracker.ForEachUploadRange( + boundary - page_size, page_size * 2, false, + [&](uint64_t, uint64_t) noexcept { ranges++; }, []() noexcept {}); + Check(ranges == 2 && + !tracker.IsRegionCpuModified(boundary - page_size, page_size * 2) && + !IsWritable(reinterpret_cast(boundary - page_size)) && + !IsWritable(reinterpret_cast(boundary)), + "cross-region upload did not clear and protect both regions"); + tracker.MarkRegionAsCpuModified(boundary - page_size, page_size * 2); + tracker.UntrackMemory(address, region_size * 2); + Release(page_manager, memory, region_size * 2); } -[[noreturn]] void RunDeathCase(const char* name) { - constexpr uintptr_t base = 0x0000000200010000ull; - TrackerHarness harness; - auto& tracker = harness.tracker; - auto& page_manager = harness.page_manager; - const auto page_size = page_manager.GetPageSize(); - const auto allocation_size = - std::strcmp(name, "missing-download-bytes") == 0 ? page_size * 2 : page_size; - auto* memory = static_cast(VirtualAlloc( - reinterpret_cast(base), allocation_size, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); - Check(memory == reinterpret_cast(base), "fixed VirtualAlloc failed"); - const auto address = reinterpret_cast(memory); - page_manager.OnGpuMap(address, allocation_size); - if (std::strcmp(name, "gpu-dirty-fault") == 0 || std::strcmp(name, "gpu-dirty-read") == 0 || - std::strcmp(name, "gpu-dirty-explicit-cpu") == 0 || - std::strcmp(name, "virtual-gpu-read") == 0) { - harness.discard_virtual = std::strcmp(name, "virtual-gpu-read") == 0; - tracker.ForEachUploadRange( - address, page_size, true, [](uint64_t, uint64_t) noexcept {}, []() noexcept {}); - if (std::strcmp(name, "gpu-dirty-explicit-cpu") == 0) { - tracker.MarkRegionAsCpuModified(address, page_size); - } else { - (void)page_manager.HandleFault((std::strcmp(name, "gpu-dirty-read") == 0 || - std::strcmp(name, "virtual-gpu-read") == 0) - ? PageFaultAccess::Read - : PageFaultAccess::Write, - address); - } - } else if (std::strcmp(name, "reentrant-upload") == 0) { - tracker.ForEachUploadRange( - address, page_size, true, [](uint64_t, uint64_t) noexcept {}, - [&]() noexcept { (void)tracker.IsRegionCpuModified(address, page_size); }); - } else if (std::strcmp(name, "writable-upload-race") == 0) { - std::atomic_bool start {false}; - std::atomic_bool entered {false}; - std::thread fault([&] { - while (!start.load(std::memory_order_acquire)) { - std::this_thread::yield(); - } - entered.store(true, std::memory_order_release); - (void)page_manager.HandleFault(PageFaultAccess::Write, address); - }); - tracker.ForEachUploadRange( - address, page_size, true, [](uint64_t, uint64_t) noexcept {}, - [&]() noexcept { - start.store(true, std::memory_order_release); - while (!entered.load(std::memory_order_acquire)) { - std::this_thread::yield(); - } - }); - fault.join(); - } else if (std::strcmp(name, "missing-download-bytes") == 0) { - tracker.ForEachUploadRange( - address, allocation_size, true, [](uint64_t, uint64_t) noexcept {}, []() noexcept {}); - RangeSet dirty_bytes; - dirty_bytes.Add(address, 1); - PageManager::BackingWrite backing(page_manager, address, page_size); - tracker.ForEachDownloadRange( - address, allocation_size, - [&](uint64_t dirty_address, uint64_t dirty_size) noexcept { - for (auto page = dirty_address; page < dirty_address + dirty_size; - page += page_size) { - if (dirty_bytes.Intersections(page, page_size).empty()) { - EXIT("GPU-dirty test page has no dirty byte record\n"); - } - } - }, - [](uint64_t, uint64_t) noexcept {}); - } - std::_Exit(0x7f); +void TestBackingWritePublication() { + TrackerHarness harness; + auto &tracker = harness.tracker; + auto &page_manager = harness.page_manager; + const auto page_size = page_manager.GetPageSize(); + auto *memory = Allocate(page_manager, 1); + const auto address = reinterpret_cast(memory); + + tracker.ForEachUploadRange( + address, page_size, true, [](uint64_t, uint64_t) noexcept {}, + []() noexcept {}); + Check(tracker.IsRegionGpuModified(address, page_size) && + Protection(memory) == PAGE_NOACCESS, + "backing publication setup did not establish GPU ownership"); + std::vector dirty{{address, page_size}}; + auto writes = page_manager.ReserveBackingWrites(dirty); + Check(writes.size() == 1 && Protection(memory) == PAGE_NOACCESS, + "backing reservation exposed protected guest memory"); + uint32_t downloads = 0; + tracker.ForEachDownloadRange( + address, page_size, [&](uint64_t, uint64_t) noexcept { downloads++; }); + tracker.MarkRegionAsCpuModified(address, page_size); + writes.clear(); + Check(downloads == 1 && !tracker.IsRegionGpuModified(address, page_size) && + tracker.IsRegionCpuModified(address, page_size) && + IsWritable(memory), + "backing publication did not restore CPU ownership"); + tracker.UntrackMemory(address, page_size); + Release(page_manager, memory, page_size); +} + +[[noreturn]] void RunDeathCase(const char *name) { + TrackerHarness harness; + auto &tracker = harness.tracker; + auto &page_manager = harness.page_manager; + const auto page_size = page_manager.GetPageSize(); + auto *memory = Allocate(page_manager, 1); + const auto address = reinterpret_cast(memory); + if (std::strcmp(name, "gpu-dirty-explicit-cpu") == 0) { + tracker.ForEachUploadRange( + address, page_size, true, [](uint64_t, uint64_t) noexcept {}, + []() noexcept {}); + tracker.MarkRegionAsCpuModified(address, page_size); + } else if (std::strcmp(name, "reentrant-upload") == 0) { + tracker.ForEachUploadRange( + address, page_size, true, [](uint64_t, uint64_t) noexcept {}, + [&]() noexcept { + (void)tracker.IsRegionCpuModified(address, page_size); + }); + } + std::_Exit(0x7f); +} + +void CheckDeathCase(const char *name) { +#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS + char path[MAX_PATH]{}; + Check(GetModuleFileNameA(nullptr, path, MAX_PATH) != 0, + "GetModuleFileName failed"); + std::string command = std::string("\"") + path + "\" --death " + name; + std::vector mutable_command(command.begin(), command.end()); + mutable_command.push_back('\0'); + STARTUPINFOA startup{sizeof(startup)}; + PROCESS_INFORMATION process{}; + Check(CreateProcessA(nullptr, mutable_command.data(), nullptr, nullptr, FALSE, + CREATE_NO_WINDOW, nullptr, nullptr, &startup, + &process) != 0, + "CreateProcess failed"); + Check(WaitForSingleObject(process.hProcess, 10000) == WAIT_OBJECT_0, + "MemoryTracker death test timed out"); + DWORD exit_code = 0; + Check( + GetExitCodeProcess(process.hProcess, &exit_code) != 0 && + (exit_code == 321 || exit_code == EXCEPTION_NONCONTINUABLE_EXCEPTION), + "MemoryTracker death path used the wrong exit"); + CloseHandle(process.hThread); + CloseHandle(process.hProcess); +#else + const pid_t pid = ::fork(); + Check(pid >= 0, "fork failed"); + if (pid == 0) { + ::execl("/proc/self/exe", "MemoryTrackerTests", "--death", name, nullptr); + std::_Exit(0x7e); + } + int status = 0; + Check(::waitpid(pid, &status, 0) == pid, "waitpid failed"); + const bool fatal_exit = + WIFEXITED(status) && WEXITSTATUS(status) == (321 & 0xff); + Check(fatal_exit || WIFSIGNALED(status), + "MemoryTracker death path used the wrong exit"); +#endif } void TestFatalPaths() { -#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS - char path[MAX_PATH] {}; - Check(GetModuleFileNameA(nullptr, path, MAX_PATH) != 0, "GetModuleFileName failed"); -#endif - for (const char* name: - {"gpu-dirty-fault", "gpu-dirty-read", "virtual-gpu-read", "gpu-dirty-explicit-cpu", - "reentrant-upload", "writable-upload-race", "missing-download-bytes"}) { -#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS - std::string command = std::string("\"") + path + "\" --death " + name; - std::vector mutable_command(command.begin(), command.end()); - mutable_command.push_back('\0'); - STARTUPINFOA startup {sizeof(startup)}; - PROCESS_INFORMATION process {}; - Check(CreateProcessA(nullptr, mutable_command.data(), nullptr, nullptr, FALSE, - CREATE_NO_WINDOW, nullptr, nullptr, &startup, &process) != 0, - "CreateProcess failed"); - Check(WaitForSingleObject(process.hProcess, 10000) == WAIT_OBJECT_0, - "MemoryTracker death test timed out"); - DWORD exit_code = 0; - Check(GetExitCodeProcess(process.hProcess, &exit_code) != 0 && - (exit_code == 321 || exit_code == EXCEPTION_NONCONTINUABLE_EXCEPTION), - "MemoryTracker death path used the wrong exit"); - CloseHandle(process.hThread); - CloseHandle(process.hProcess); -#else - const pid_t pid = ::fork(); - Check(pid >= 0, "fork failed"); - if (pid == 0) { - ::execl("/proc/self/exe", "MemoryTrackerTests", "--death", name, nullptr); - std::_Exit(0x7e); - } - int status = 0; - Check(::waitpid(pid, &status, 0) == pid, "waitpid failed"); - // Exit status carries only the low 8 bits. - const bool fatal_exit = WIFEXITED(status) && (WEXITSTATUS(status) == (321 & 0xff) || - WEXITSTATUS(status) == (322 & 0xff)); - Check(fatal_exit || WIFSIGNALED(status), "MemoryTracker death path used the wrong exit"); -#endif - } + for (const char *name : {"gpu-dirty-explicit-cpu", "reentrant-upload"}) { + CheckDeathCase(name); + } } -#endif } // namespace namespace Libs::LibKernel::Memory { -bool ProtectGuestHostMemory(uint64_t vaddr, uint64_t size, Common::VirtualMemory::Mode mode) { - return ProtectAddressSpace(vaddr, size, mode); +bool ProtectGuestHostMemory(uint64_t vaddr, uint64_t size, + Common::VirtualMemory::Mode mode) { + return ProtectAddressSpace(vaddr, size, mode); } } // namespace Libs::LibKernel::Memory -int main(int argc, char** argv) { -#if 1 - if (argc == 3 && std::strcmp(argv[1], "--death") == 0) { - RunDeathCase(argv[2]); - } - TestCpuDirtyUploadAndFault(); - TestPendingFaultBlocksUploadConsumption(); - TestCleanReadFaultPreservesCpuState(); - TestGpuDownloadFaultOwnership(); - TestVirtualGpuWriteDiscard(); - TestSameSlabTrackerArbitration(); - TestSharedMetadataAndImagePageFault(); - TestRangeSet(); - TestQueriesDoNotRequireMappedOwnership(); - TestRangeInvalidation(); - TestGpuDirtyBits(); - TestCrossRegionUpload(); - TestFaultDuringUploadRemainsDirty(); - TestNativeStoreDuringRangeEnumeration(); - TestFaultDuringDownloadSynchronization(); - TestFaultAndExplicitDirtyRace(); - TestSharedTrackersAndConcurrentPageFaults(); - TestFatalPaths(); - std::puts("MemoryTrackerTests: all cases passed"); - return 0; -#else - (void)argc; - (void)argv; - std::fputs("MemoryTrackerTests: unsupported platform\n", stderr); - return 1; -#endif +int main(int argc, char **argv) { + if (argc == 3 && std::strcmp(argv[1], "--death") == 0) { + RunDeathCase(argv[2]); + } + TestRangeSet(); + TestQueriesDoNotRequireMappedOwnership(); + TestCpuDirtyUpload(); + TestRangeInvalidation(); + TestGpuDirtyBits(); + TestCrossRegionUpload(); + TestBackingWritePublication(); + TestFatalPaths(); + std::puts("MemoryTrackerTests: all cases passed"); + return 0; } diff --git a/tests/PageManagerTests.cpp b/tests/PageManagerTests.cpp index 288566e..7c00a11 100644 --- a/tests/PageManagerTests.cpp +++ b/tests/PageManagerTests.cpp @@ -1,14 +1,12 @@ #include "common/virtualMemory.h" #include "graphics/host_gpu/pageManager.h" -#include #include #include #include #include #include #include -#include #include #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS @@ -19,731 +17,343 @@ #undef min #undef max #else -#include #include #include #include -#include #include #endif namespace { -using Libs::Graphics::PageFaultAccess; using Libs::Graphics::PageManager; +using Libs::Graphics::PageWatchMode; -void Check(bool value, const char* text) { - if (!value) { - std::fprintf(stderr, "PageManagerTests: failed: %s\n", text); - std::abort(); - } +void Check(bool value, const char *text) { + if (!value) { + std::fprintf(stderr, "PageManagerTests: failed: %s\n", text); + std::abort(); + } } #if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS -// POSIX shims for the shared test body. -using DWORD = uint32_t; -constexpr uint32_t PAGE_NOACCESS = 1; -constexpr uint32_t PAGE_READONLY = 2; +using DWORD = uint32_t; +constexpr uint32_t PAGE_NOACCESS = 1; +constexpr uint32_t PAGE_READONLY = 2; constexpr uint32_t PAGE_READWRITE = 3; -constexpr uint32_t MEM_RELEASE = 0; +constexpr uint32_t MEM_RELEASE = 0; int ToHostProt(uint32_t protection) { - switch (protection) { - case PAGE_NOACCESS: return PROT_NONE; - case PAGE_READONLY: return PROT_READ; - default: return PROT_READ | PROT_WRITE; - } + switch (protection) { + case PAGE_NOACCESS: + return PROT_NONE; + case PAGE_READONLY: + return PROT_READ; + default: + return PROT_READ | PROT_WRITE; + } } -uint32_t Protection(const void* address) { - const auto addr = reinterpret_cast(address); - std::FILE* maps = std::fopen("/proc/self/maps", "r"); - Check(maps != nullptr, "open /proc/self/maps failed"); - char line[512]; - uint32_t result = 0; // 0 => not mapped at all - while (std::fgets(line, sizeof(line), maps) != nullptr) { - unsigned long start = 0; - unsigned long end = 0; - char perms[8] {}; - if (std::sscanf(line, "%lx-%lx %7s", &start, &end, perms) != 3) { - continue; - } - if (addr >= start && addr < end) { - result = perms[1] == 'w' ? PAGE_READWRITE - : perms[0] == 'r' ? PAGE_READONLY - : PAGE_NOACCESS; - break; - } - } - std::fclose(maps); - return result; +uint32_t Protection(const void *address) { + const auto addr = reinterpret_cast(address); + std::FILE *maps = std::fopen("/proc/self/maps", "r"); + Check(maps != nullptr, "open /proc/self/maps failed"); + char line[512]; + uint32_t result = 0; + while (std::fgets(line, sizeof(line), maps) != nullptr) { + unsigned long start = 0; + unsigned long end = 0; + char perms[8]{}; + if (std::sscanf(line, "%lx-%lx %7s", &start, &end, perms) != 3) { + continue; + } + if (addr >= start && addr < end) { + result = perms[1] == 'w' ? PAGE_READWRITE + : perms[0] == 'r' ? PAGE_READONLY + : PAGE_NOACCESS; + break; + } + } + std::fclose(maps); + return result; } -bool IsWritable(const void* address) { - return Protection(address) == PAGE_READWRITE; +std::map &AllocationSizes() { + static std::map sizes; + return sizes; } -// munmap needs the length that VirtualFree's callers pass as 0, so sizes are remembered here. -std::map& AllocationSizes() { - static std::map sizes; - return sizes; +int VirtualFree(void *address, size_t, DWORD) { + auto &sizes = AllocationSizes(); + auto it = sizes.find(address); + if (it == sizes.end()) { + return 0; + } + const int ok = ::munmap(address, it->second) == 0 ? 1 : 0; + sizes.erase(it); + return ok; } -int VirtualFree(void* address, size_t /*size*/, DWORD /*type*/) { - auto& sizes = AllocationSizes(); - auto it = sizes.find(address); - if (it == sizes.end()) { - return 0; - } - const int ok = ::munmap(address, it->second) == 0 ? 1 : 0; - sizes.erase(it); - return ok; -} - -int VirtualProtect(void* address, size_t size, uint32_t protection, DWORD* old_protection) { - if (old_protection != nullptr) { - *old_protection = Protection(address); - } - return ::mprotect(address, size, ToHostProt(protection)) == 0 ? 1 : 0; -} -#endif - -#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS -bool IsWritable(const void* address) { - MEMORY_BASIC_INFORMATION info {}; - Check(VirtualQuery(address, &info, sizeof(info)) != 0, "VirtualQuery failed"); - return info.Protect == PAGE_READWRITE; -} - -uint32_t Protection(const void* address) { - MEMORY_BASIC_INFORMATION info {}; - Check(VirtualQuery(address, &info, sizeof(info)) != 0, "VirtualQuery failed"); - return info.Protect; -} -#endif - -std::atomic_uint64_t g_protection_calls {0}; - -bool ProtectAddressSpace(uint64_t vaddr, uint64_t size, Common::VirtualMemory::Mode mode) { - uint32_t protection = PAGE_NOACCESS; - if (mode == Common::VirtualMemory::Mode::Read) { - protection = PAGE_READONLY; - } else if (mode == Common::VirtualMemory::Mode::ReadWrite) { - protection = PAGE_READWRITE; - } - DWORD old_protection = 0; - g_protection_calls.fetch_add(1, std::memory_order_relaxed); - return VirtualProtect(reinterpret_cast(vaddr), size, protection, &old_protection) != 0; -} - -#if 1 - -struct FaultContext { - PageManager* manager = nullptr; - bool result = true; - bool reenter = false; - uint64_t reenter_address = 0; - bool block = false; - std::atomic_uint32_t calls {0}; - std::atomic_bool entered {false}; - std::atomic_bool release {false}; -}; - -std::atomic g_native_fault_manager {nullptr}; -std::atomic_bool g_delay_native_fault {false}; -std::atomic_bool g_native_fault_entered {false}; -std::atomic_bool g_release_native_fault {false}; - -#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS -LONG CALLBACK NativeFaultHandler(EXCEPTION_POINTERS* exception) { - if (exception == nullptr || exception->ExceptionRecord == nullptr || - exception->ExceptionRecord->ExceptionCode != EXCEPTION_ACCESS_VIOLATION) { - return EXCEPTION_CONTINUE_SEARCH; - } - const auto operation = exception->ExceptionRecord->ExceptionInformation[0]; - const auto access = operation == 0 ? PageFaultAccess::Read - : operation == 1 ? PageFaultAccess::Write - : operation == 8 ? PageFaultAccess::Execute - : PageFaultAccess::Unknown; - auto* manager = g_native_fault_manager.load(std::memory_order_acquire); - if (manager == nullptr) { - return EXCEPTION_CONTINUE_SEARCH; - } - if (g_delay_native_fault.load(std::memory_order_acquire)) { - g_native_fault_entered.store(true, std::memory_order_release); - while (!g_release_native_fault.load(std::memory_order_acquire)) { - std::this_thread::yield(); - } - } - return manager->HandleFault(access, exception->ExceptionRecord->ExceptionInformation[1]) - ? EXCEPTION_CONTINUE_EXECUTION - : EXCEPTION_CONTINUE_SEARCH; +int VirtualProtect(void *address, size_t size, uint32_t protection, + DWORD *old_protection) { + if (old_protection != nullptr) { + *old_protection = Protection(address); + } + return ::mprotect(address, size, ToHostProt(protection)) == 0 ? 1 : 0; } #else -// SIGSEGV stands in for the vectored exception handler. -void NativeFaultHandler(int signal_number, siginfo_t* info, void* native_context) { - auto* context = static_cast(native_context); - const auto error_code = static_cast(context->uc_mcontext.gregs[REG_ERR]); - const auto access = (error_code & 0x10u) != 0 ? PageFaultAccess::Execute - : (error_code & 0x02u) != 0 ? PageFaultAccess::Write - : PageFaultAccess::Read; - auto* manager = g_native_fault_manager.load(std::memory_order_acquire); - if (manager != nullptr) { - if (g_delay_native_fault.load(std::memory_order_acquire)) { - g_native_fault_entered.store(true, std::memory_order_release); - while (!g_release_native_fault.load(std::memory_order_acquire)) { - std::this_thread::yield(); - } - } - if (manager->HandleFault(access, reinterpret_cast(info->si_addr))) { - return; - } - } - struct sigaction restore {}; - restore.sa_handler = SIG_DFL; - sigemptyset(&restore.sa_mask); - ::sigaction(signal_number, &restore, nullptr); -} - -struct sigaction g_saved_segv_action {}; - -void* AddVectoredExceptionHandler(unsigned long /*first*/, - void (*handler)(int, siginfo_t*, void*)) { - struct sigaction action {}; - action.sa_sigaction = handler; - sigemptyset(&action.sa_mask); - action.sa_flags = SA_SIGINFO; - if (::sigaction(SIGSEGV, &action, &g_saved_segv_action) != 0) { - return nullptr; - } - return reinterpret_cast(handler); -} - -int RemoveVectoredExceptionHandler(void* /*token*/) { - return ::sigaction(SIGSEGV, &g_saved_segv_action, nullptr) == 0 ? 1 : 0; +uint32_t Protection(const void *address) { + MEMORY_BASIC_INFORMATION info{}; + Check(VirtualQuery(address, &info, sizeof(info)) != 0, "VirtualQuery failed"); + return info.Protect; } #endif -bool InvalidateFault(void* context, Libs::Graphics::PageFaultAccess, uint64_t vaddr, uint64_t size, - Libs::Graphics::PageFaultPhase phase) noexcept { - auto* fault = static_cast(context); - Check(fault != nullptr && fault->manager != nullptr, "invalid fault context"); - if (phase != Libs::Graphics::PageFaultPhase::Invalidate) { - return true; - } - fault->calls.fetch_add(1, std::memory_order_relaxed); - if (fault->reenter) { - const auto address = fault->reenter_address != 0 ? fault->reenter_address : vaddr; - (void)fault->manager->HandleFault(PageFaultAccess::Write, address); - } - if (fault->block) { - fault->entered.store(true, std::memory_order_release); - while (!fault->release.load(std::memory_order_acquire)) { - std::this_thread::yield(); - } - } - (void)size; - return fault->result; +bool IsWritable(const void *address) { + return Protection(address) == PAGE_READWRITE; } -uint8_t* Allocate(uint64_t size, uint32_t protection = PAGE_READWRITE) { - constexpr uintptr_t test_address = 0x0000000200010000ull; +uint64_t g_protection_calls = 0; + +bool ProtectAddressSpace(uint64_t vaddr, uint64_t size, + Common::VirtualMemory::Mode mode) { + uint32_t protection = PAGE_NOACCESS; + if (mode == Common::VirtualMemory::Mode::Read) { + protection = PAGE_READONLY; + } else if (mode == Common::VirtualMemory::Mode::ReadWrite) { + protection = PAGE_READWRITE; + } + DWORD old_protection = 0; + g_protection_calls++; + return VirtualProtect(reinterpret_cast(vaddr), size, protection, + &old_protection) != 0; +} + +uint8_t *Allocate(uint64_t size, uint32_t protection = PAGE_READWRITE) { + constexpr uintptr_t test_address = 0x0000000200010000ull; #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS - auto* memory = static_cast(VirtualAlloc(reinterpret_cast(test_address), size, - MEM_RESERVE | MEM_COMMIT, protection)); - Check(memory == reinterpret_cast(test_address), "fixed low VirtualAlloc failed"); + auto *memory = static_cast( + VirtualAlloc(reinterpret_cast(test_address), size, + MEM_RESERVE | MEM_COMMIT, protection)); + Check(memory == reinterpret_cast(test_address), + "fixed low VirtualAlloc failed"); #else - // Do not overwrite a leaked mapping from an earlier case. - void* raw = ::mmap(reinterpret_cast(test_address), size, ToHostProt(protection), - MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); - Check(raw == reinterpret_cast(test_address), "fixed low mmap failed"); - auto* memory = static_cast(raw); - AllocationSizes()[raw] = static_cast(size); + void *raw = ::mmap(reinterpret_cast(test_address), size, + ToHostProt(protection), + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); + Check(raw == reinterpret_cast(test_address), "fixed low mmap failed"); + auto *memory = static_cast(raw); + AllocationSizes()[raw] = static_cast(size); #endif - return memory; + return memory; } -void TestWatchFaultAndUnwatch() { - g_protection_calls.store(0, std::memory_order_relaxed); - FaultContext context; - PageManager manager(InvalidateFault, &context); - context.manager = &manager; - const auto page_size = manager.GetPageSize(); - auto* memory = Allocate(page_size * 2); +void TestWatchAndUnwatch() { + g_protection_calls = 0; + PageManager manager; + const auto page_size = manager.GetPageSize(); + auto *memory = Allocate(page_size * 2); + const auto address = reinterpret_cast(memory); - manager.OnGpuMap(reinterpret_cast(memory), page_size * 2); - manager.UpdatePageWatchers(true, reinterpret_cast(memory), page_size); - Check(manager.IsTracked(reinterpret_cast(memory)) && !IsWritable(memory), - "watch did not protect the page"); - Check(g_protection_calls.load(std::memory_order_relaxed) != 0, - "watch protection bypassed the address-space owner callback"); - Check(manager.HandleFault(PageFaultAccess::Write, reinterpret_cast(memory + 32)), - "tracked write fault was not handled"); - Check(!manager.IsTracked(reinterpret_cast(memory)) && IsWritable(memory), - "fault invalidation did not remove the watcher"); - Check(manager.HandleFault(PageFaultAccess::Write, reinterpret_cast(memory)), - "single delayed write fault was not coalesced"); - Check(manager.HandleFault(PageFaultAccess::Write, reinterpret_cast(memory)), - "second delayed write fault was not coalesced"); - manager.OnGpuUnmap(reinterpret_cast(memory), page_size * 2); - Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); + manager.OnGpuMap(address, page_size * 2); + manager.UpdatePageWatchers(true, address, page_size); + Check(Protection(memory) == PAGE_READONLY && IsWritable(memory + page_size), + "write watch installed incorrect protections"); + Check(g_protection_calls != 0, + "watch protection bypassed the address-space owner callback"); + manager.UpdatePageWatchers(false, address, page_size); + Check(IsWritable(memory), "write unwatch did not restore access"); + manager.OnGpuUnmap(address, page_size * 2); + Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); } -void TestSharedWatcherFault() { - FaultContext context; - PageManager manager(InvalidateFault, &context); - context.manager = &manager; - const auto page_size = manager.GetPageSize(); - auto* memory = Allocate(page_size); - const auto address = reinterpret_cast(memory); +void TestSharedWatcherCounts() { + PageManager manager; + const auto page_size = manager.GetPageSize(); + auto *memory = Allocate(page_size); + const auto address = reinterpret_cast(memory); - manager.OnGpuMap(address, page_size); - manager.UpdatePageWatchers(true, address + 8, 32); - manager.UpdatePageWatchers(true, address + 128, 64); - Check(manager.HandleFault(PageFaultAccess::Write, address + 16), - "shared-watcher fault was not handled"); - Check(!manager.IsTracked(address) && IsWritable(memory), - "fault callback did not clear every shared watcher"); - manager.OnGpuUnmap(address, page_size); - Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); + manager.OnGpuMap(address, page_size); + manager.UpdatePageWatchers(true, address + 8, 32); + manager.UpdatePageWatchers(true, address + 128, 64); + manager.UpdatePageWatchers(false, address + 8, 32); + Check(Protection(memory) == PAGE_READONLY, + "first unwatch released a shared watcher"); + manager.UpdatePageWatchers(false, address + 128, 64); + Check(IsWritable(memory), "last unwatch did not restore access"); + manager.OnGpuUnmap(address, page_size); + Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); } -void TestReadWriteWatcherFault() { - FaultContext context; - PageManager manager(InvalidateFault, &context); - context.manager = &manager; - const auto page_size = manager.GetPageSize(); - auto* memory = Allocate(page_size); - const auto address = reinterpret_cast(memory); +void TestMixedWatcherModes() { + PageManager manager; + const auto page_size = manager.GetPageSize(); + auto *memory = Allocate(page_size); + const auto address = reinterpret_cast(memory); - manager.OnGpuMap(address, page_size); - manager.UpdatePageWatchers(true, address, page_size, Libs::Graphics::PageWatchMode::Write); - manager.UpdatePageWatchers(true, address, page_size, Libs::Graphics::PageWatchMode::ReadWrite); - manager.UpdatePageWatchers(false, address, page_size, Libs::Graphics::PageWatchMode::Write); - Check(Protection(memory) == PAGE_NOACCESS, - "read/write watcher did not install no-access protection"); - Check(manager.HandleFault(PageFaultAccess::Read, address + 8), - "tracked read fault was not handled"); - Check(Protection(memory) == PAGE_READWRITE && !manager.IsTracked(address), - "read fault did not release every watcher"); - manager.OnGpuUnmap(address, page_size); - Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); -} - -void TestPermittedMappedLateFaultsResume() { - FaultContext context; - PageManager manager(InvalidateFault, &context); - context.manager = &manager; - const auto page_size = manager.GetPageSize(); - auto* memory = Allocate(page_size); - const auto address = reinterpret_cast(memory); - - manager.OnGpuMap(address, page_size); - manager.UpdatePageWatchers(true, address, page_size); - manager.UpdatePageWatchers(false, address, page_size); - Check(manager.HandleFault(PageFaultAccess::Write, address), - "first delayed mapped write was not accepted"); - Check(manager.HandleFault(PageFaultAccess::Write, address), - "second delayed mapped write was not accepted"); - Check(manager.HandleFault(PageFaultAccess::Read, address), - "delayed mapped read was not accepted on readable backing"); - manager.OnGpuUnmap(address, page_size); - Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); -} - -void TestPartialMappingUnmapPreservesTokens() { - FaultContext context; - PageManager manager(InvalidateFault, &context); - context.manager = &manager; - const auto page_size = manager.GetPageSize(); - auto* memory = Allocate(page_size); - const auto address = reinterpret_cast(memory); - - manager.OnGpuMap(address, page_size); - manager.OnGpuMap(address + 8, 16); - manager.UpdatePageWatchers(true, address, page_size, Libs::Graphics::PageWatchMode::ReadWrite); - manager.UpdatePageWatchers(false, address, page_size, Libs::Graphics::PageWatchMode::ReadWrite); - manager.OnGpuUnmap(address + 8, 16); - Check(manager.HandleFault(PageFaultAccess::Read, address), - "partial mapping unmap erased delayed read ownership"); - Check(manager.HandleFault(PageFaultAccess::Write, address), - "partial mapping unmap erased delayed write ownership"); - manager.OnGpuUnmap(address, page_size); - Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); -} - -void TestNativeDelayedReadAfterModeDowngrade() { - FaultContext context; - PageManager manager(InvalidateFault, &context); - context.manager = &manager; - const auto page_size = manager.GetPageSize(); - auto* memory = Allocate(page_size); - memory[0] = 0x6d; - const auto address = reinterpret_cast(memory); - - manager.OnGpuMap(address, page_size); - manager.UpdatePageWatchers(true, address, page_size); - manager.UpdatePageWatchers(true, address, page_size, Libs::Graphics::PageWatchMode::ReadWrite); - manager.UpdatePageWatchers(false, address, page_size); - Check(Protection(memory) == PAGE_NOACCESS, - "read/write ownership did not install no-access protection"); - - void* handler = AddVectoredExceptionHandler(1, NativeFaultHandler); - Check(handler != nullptr, "AddVectoredExceptionHandler failed"); - Check(g_native_fault_manager.exchange(&manager, std::memory_order_acq_rel) == nullptr, - "native fault manager already installed"); - g_native_fault_entered.store(false, std::memory_order_release); - g_release_native_fault.store(false, std::memory_order_release); - g_delay_native_fault.store(true, std::memory_order_release); - - uint8_t value = 0; - std::thread reader([&] { value = *static_cast(memory); }); - while (!g_native_fault_entered.load(std::memory_order_acquire)) { - std::this_thread::yield(); - } - manager.UpdatePageWatchers(true, address, page_size); - manager.UpdatePageWatchers(false, address, page_size, Libs::Graphics::PageWatchMode::ReadWrite); - Check(Protection(memory) == PAGE_READONLY, - "mode downgrade did not restore readable protection"); - manager.UpdatePageWatchers(true, address, page_size); - - context.block = true; - bool write_handled = false; - std::thread writer( - [&] { write_handled = manager.HandleFault(PageFaultAccess::Write, address); }); - while (!context.entered.load(std::memory_order_acquire)) { - std::this_thread::yield(); - } - g_release_native_fault.store(true, std::memory_order_release); - reader.join(); - context.release.store(true, std::memory_order_release); - writer.join(); - - g_delay_native_fault.store(false, std::memory_order_release); - Check(g_native_fault_manager.exchange(nullptr, std::memory_order_acq_rel) == &manager, - "native fault manager publication changed"); - Check(RemoveVectoredExceptionHandler(handler) != 0, "RemoveVectoredExceptionHandler failed"); - Check(value == 0x6d && write_handled && context.calls.load(std::memory_order_relaxed) == 1, - "delayed read was not coalesced across write ownership/resolution"); - - manager.OnGpuUnmap(address, page_size); - Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); -} - -void TestDelayedFaultAfterExplicitUnwatch() { - FaultContext context; - PageManager manager(InvalidateFault, &context); - context.manager = &manager; - const auto page_size = manager.GetPageSize(); - auto* memory = Allocate(page_size); - const auto address = reinterpret_cast(memory); - manager.OnGpuMap(address, page_size); - manager.UpdatePageWatchers(true, address, page_size); - - // Models a store that already raised an AV before another thread published - // explicit CPU dirtiness and removed the watcher. - manager.UpdatePageWatchers(false, address, page_size); - Check(manager.HandleFault(PageFaultAccess::Write, address), - "delayed watched write was not accepted after explicit unwatch"); - Check(manager.HandleFault(PageFaultAccess::Write, address) && - context.calls.load(std::memory_order_relaxed) == 0, - "second explicit-unwatch fault was not coalesced or dispatched invalidation"); - - manager.OnGpuUnmap(address, page_size); - Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); -} - -void TestNativeAccessViolation() { - FaultContext context; - PageManager manager(InvalidateFault, &context); - context.manager = &manager; - const auto page_size = manager.GetPageSize(); - auto* memory = Allocate(page_size); - const auto address = reinterpret_cast(memory); - manager.OnGpuMap(address, page_size); - manager.UpdatePageWatchers(true, address, page_size); - - void* handler = AddVectoredExceptionHandler(1, NativeFaultHandler); - Check(handler != nullptr, "AddVectoredExceptionHandler failed"); - Check(g_native_fault_manager.exchange(&manager, std::memory_order_acq_rel) == nullptr, - "native fault manager already installed"); - *static_cast(memory) = 0x5a; - Check(g_native_fault_manager.exchange(nullptr, std::memory_order_acq_rel) == &manager, - "native fault manager publication changed"); - Check(RemoveVectoredExceptionHandler(handler) != 0, "RemoveVectoredExceptionHandler failed"); - - Check(memory[0] == 0x5a && !manager.IsTracked(address) && - context.calls.load(std::memory_order_relaxed) == 1, - "native access violation did not invalidate and resume the store"); - manager.OnGpuUnmap(address, page_size); - Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); + manager.OnGpuMap(address, page_size); + manager.UpdatePageWatchers(true, address, page_size, PageWatchMode::Write); + manager.UpdatePageWatchers(true, address, page_size, + PageWatchMode::ReadWrite); + Check(Protection(memory) == PAGE_NOACCESS, + "read/write watcher did not deny access"); + manager.UpdatePageWatchers(false, address, page_size, PageWatchMode::Write); + Check(Protection(memory) == PAGE_NOACCESS, + "write unwatch released a read/write watcher"); + manager.UpdatePageWatchers(false, address, page_size, + PageWatchMode::ReadWrite); + Check(IsWritable(memory), "read/write unwatch did not restore access"); + manager.OnGpuUnmap(address, page_size); + Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); } void TestCrossRegionRange() { - FaultContext context; - PageManager manager(InvalidateFault, &context); - context.manager = &manager; - const auto page_size = manager.GetPageSize(); - constexpr uint64_t region_size = 4ull * 1024ull * 1024ull; - auto* memory = Allocate(region_size * 2); - const auto base = reinterpret_cast(memory); - const auto boundary = (base + region_size - 1) & ~(region_size - 1); - Check(boundary >= base + page_size && boundary + page_size <= base + region_size * 2, - "test allocation does not contain a region boundary"); + PageManager manager; + const auto page_size = manager.GetPageSize(); + constexpr uint64_t region_size = 4ull * 1024ull * 1024ull; + auto *memory = Allocate(region_size * 2); + const auto base = reinterpret_cast(memory); + const auto boundary = (base + region_size - 1) & ~(region_size - 1); + Check(boundary >= base + page_size && + boundary + page_size <= base + region_size * 2, + "test allocation does not contain a region boundary"); - manager.OnGpuMap(base, region_size * 2); - manager.UpdatePageWatchers(true, boundary - page_size, page_size * 2); - Check(!IsWritable(reinterpret_cast(boundary - page_size)) && - !IsWritable(reinterpret_cast(boundary)), - "cross-region watch did not protect both pages"); - manager.UpdatePageWatchers(false, boundary - page_size, page_size * 2); - Check(IsWritable(reinterpret_cast(boundary - page_size)) && - IsWritable(reinterpret_cast(boundary)), - "cross-region unwatch did not restore both pages"); - manager.OnGpuUnmap(base, region_size * 2); - Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); + manager.OnGpuMap(base, region_size * 2); + manager.UpdatePageWatchers(true, boundary - page_size, page_size * 2); + Check(!IsWritable(reinterpret_cast(boundary - page_size)) && + !IsWritable(reinterpret_cast(boundary)), + "cross-region watch did not protect both pages"); + manager.UpdatePageWatchers(false, boundary - page_size, page_size * 2); + Check(IsWritable(reinterpret_cast(boundary - page_size)) && + IsWritable(reinterpret_cast(boundary)), + "cross-region unwatch did not restore both pages"); + manager.OnGpuUnmap(base, region_size * 2); + Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); } void TestBatchedWatcherRanges() { - FaultContext context; - PageManager manager(InvalidateFault, &context); - context.manager = &manager; - const auto page_size = manager.GetPageSize(); - constexpr uint64_t region_size = 4ull * 1024ull * 1024ull; - constexpr uint64_t allocation_size = region_size * 3; - auto* memory = Allocate(allocation_size); - const auto address = reinterpret_cast(memory); + PageManager manager; + const auto page_size = manager.GetPageSize(); + constexpr uint64_t region_size = 4ull * 1024ull * 1024ull; + constexpr uint64_t allocation_size = region_size * 3; + auto *memory = Allocate(allocation_size); + const auto address = reinterpret_cast(memory); - manager.OnGpuMap(address, allocation_size); + manager.OnGpuMap(address, allocation_size); + manager.UpdatePageWatchers(true, address + page_size, page_size); + manager.UpdatePageWatchers(true, address + page_size * 3, page_size); + manager.UpdatePageWatchers(true, address, page_size * 5); + manager.UpdatePageWatchers(false, address, page_size * 5); + Check(IsWritable(memory) && Protection(memory + page_size) == PAGE_READONLY && + IsWritable(memory + page_size * 2) && + Protection(memory + page_size * 3) == PAGE_READONLY && + IsWritable(memory + page_size * 4), + "fragmented unwatch lost overlapping watcher counts"); + manager.UpdatePageWatchers(false, address + page_size, page_size); + manager.UpdatePageWatchers(false, address + page_size * 3, page_size); - manager.UpdatePageWatchers(true, address + page_size, page_size); - manager.UpdatePageWatchers(true, address + page_size * 3, page_size); - manager.UpdatePageWatchers(true, address, page_size * 5); - for (uint64_t page = 0; page < 5; page++) { - Check(Protection(memory + page * page_size) == PAGE_READONLY, - "fragmented watch did not coalesce to read-only"); - } - manager.UpdatePageWatchers(false, address, page_size * 5); - Check(IsWritable(memory) && Protection(memory + page_size) == PAGE_READONLY && - IsWritable(memory + page_size * 2) && - Protection(memory + page_size * 3) == PAGE_READONLY && - IsWritable(memory + page_size * 4), - "fragmented unwatch lost overlapping watcher counts"); - manager.UpdatePageWatchers(false, address + page_size, page_size); - manager.UpdatePageWatchers(false, address + page_size * 3, page_size); + g_protection_calls = 0; + manager.UpdatePageWatchers(true, address, allocation_size); + Check(g_protection_calls == 4 && Protection(memory) == PAGE_READONLY && + Protection(memory + region_size) == PAGE_READONLY && + Protection(memory + region_size * 2) == PAGE_READONLY && + Protection(memory + allocation_size - page_size) == PAGE_READONLY, + "large watch was not batched and protected by tracking region"); + manager.UpdatePageWatchers(false, address, allocation_size); + Check(IsWritable(memory) && IsWritable(memory + region_size) && + IsWritable(memory + region_size * 2) && + IsWritable(memory + allocation_size - page_size), + "large cross-region unwatch did not restore the full range"); - manager.UpdatePageWatchers(true, address, allocation_size); - Check(!IsWritable(memory) && !IsWritable(memory + region_size) && - !IsWritable(memory + region_size * 2) && - !IsWritable(memory + allocation_size - page_size), - "large cross-region watch did not protect the full range"); - manager.UpdatePageWatchers(false, address, allocation_size); - Check(IsWritable(memory) && IsWritable(memory + region_size) && - IsWritable(memory + region_size * 2) && - IsWritable(memory + allocation_size - page_size), - "large cross-region unwatch did not restore the full range"); - - manager.UpdatePageWatchers(true, address, page_size * 5); - manager.UpdatePageWatchers(true, address + page_size, page_size * 3, - Libs::Graphics::PageWatchMode::ReadWrite); - Check(Protection(memory) == PAGE_READONLY && Protection(memory + page_size) == PAGE_NOACCESS && - Protection(memory + page_size * 2) == PAGE_NOACCESS && - Protection(memory + page_size * 3) == PAGE_NOACCESS && - Protection(memory + page_size * 4) == PAGE_READONLY, - "mixed watcher modes installed incorrect protections"); - manager.UpdatePageWatchers(false, address, page_size * 5); - Check(IsWritable(memory) && Protection(memory + page_size) == PAGE_NOACCESS && - Protection(memory + page_size * 2) == PAGE_NOACCESS && - Protection(memory + page_size * 3) == PAGE_NOACCESS && - IsWritable(memory + page_size * 4), - "write unwatch incorrectly released read/write watchers"); - manager.UpdatePageWatchers(false, address + page_size, page_size * 3, - Libs::Graphics::PageWatchMode::ReadWrite); - Check(IsWritable(memory + page_size) && IsWritable(memory + page_size * 2) && - IsWritable(memory + page_size * 3), - "read/write unwatch did not restore writable protection"); - - manager.OnGpuUnmap(address, allocation_size); - Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); + manager.OnGpuUnmap(address, allocation_size); + Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); } -[[noreturn]] void RunDeathCase(const char* name) { - FaultContext context; - auto manager = std::make_unique(InvalidateFault, &context); - context.manager = manager.get(); - const auto page_size = manager->GetPageSize(); - if (std::strcmp(name, "invalid-range") == 0) { - manager->UpdatePageWatchers(true, (1ull << 40u) - 1, 2); - } else if (std::strcmp(name, "unknown-untrack") == 0) { - manager->UpdatePageWatchers(false, 0x1000, page_size); - } else { - const bool two_pages = std::strcmp(name, "cross-reentrant") == 0; - auto* memory = Allocate(two_pages ? page_size * 2 : page_size); - const auto address = reinterpret_cast(memory); - manager->OnGpuMap(address, two_pages ? page_size * 2 : page_size); - manager->UpdatePageWatchers(true, address, page_size); - if (two_pages) { - manager->UpdatePageWatchers(true, address + page_size, page_size); - } - if (std::strcmp(name, "destructor-watch") == 0) { - manager.reset(); - } else if (std::strcmp(name, "non-write") == 0) { - (void)manager->HandleFault(PageFaultAccess::Read, address); - } else if (std::strcmp(name, "callback-false") == 0) { - context.result = false; - (void)manager->HandleFault(PageFaultAccess::Write, address); - } else if (std::strcmp(name, "reentrant") == 0) { - context.reenter = true; - (void)manager->HandleFault(PageFaultAccess::Write, address); - } else if (std::strcmp(name, "cross-reentrant") == 0) { - context.reenter = true; - context.reenter_address = address + page_size; - (void)manager->HandleFault(PageFaultAccess::Write, address); - } else if (std::strcmp(name, "concurrent-non-write") == 0) { - context.block = true; - std::thread first([&] { (void)manager->HandleFault(PageFaultAccess::Write, address); }); - while (!context.entered.load(std::memory_order_acquire)) { - std::this_thread::yield(); - } - (void)manager->HandleFault(PageFaultAccess::Read, address); - first.join(); - } else { - std::_Exit(0x7f); - } - } - std::_Exit(0x7f); +[[noreturn]] void RunDeathCase(const char *name) { + PageManager manager; + const auto page_size = manager.GetPageSize(); + if (std::strcmp(name, "invalid-range") == 0) { + manager.UpdatePageWatchers(true, (1ull << 40u) - 1, 2); + } else if (std::strcmp(name, "unknown-untrack") == 0) { + manager.UpdatePageWatchers(false, 0x1000, page_size); + } else if (std::strcmp(name, "destructor-watch") == 0) { + auto doomed = std::make_unique(); + auto *memory = Allocate(page_size); + const auto address = reinterpret_cast(memory); + doomed->OnGpuMap(address, page_size); + doomed->UpdatePageWatchers(true, address, page_size); + doomed.reset(); + } + std::_Exit(0x7f); } -void CheckDeathCase(const char* name) { +void CheckDeathCase(const char *name) { #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS - char path[MAX_PATH] {}; - Check(GetModuleFileNameA(nullptr, path, MAX_PATH) != 0, "GetModuleFileName failed"); - std::string command = std::string("\"") + path + "\" --death " + name; - std::vector mutable_command(command.begin(), command.end()); - mutable_command.push_back('\0'); - STARTUPINFOA startup {sizeof(startup)}; - PROCESS_INFORMATION process {}; - Check(CreateProcessA(nullptr, mutable_command.data(), nullptr, nullptr, FALSE, CREATE_NO_WINDOW, - nullptr, nullptr, &startup, &process) != 0, - "CreateProcess failed"); - Check(WaitForSingleObject(process.hProcess, 10000) == WAIT_OBJECT_0, "death test timed out"); - DWORD exit_code = 0; - Check(GetExitCodeProcess(process.hProcess, &exit_code) != 0 && - (exit_code == 322 || exit_code == EXCEPTION_NONCONTINUABLE_EXCEPTION), - "death case did not use the PageManager fatal exit"); - CloseHandle(process.hThread); - CloseHandle(process.hProcess); + char path[MAX_PATH]{}; + Check(GetModuleFileNameA(nullptr, path, MAX_PATH) != 0, + "GetModuleFileName failed"); + std::string command = std::string("\"") + path + "\" --death " + name; + std::vector mutable_command(command.begin(), command.end()); + mutable_command.push_back('\0'); + STARTUPINFOA startup{sizeof(startup)}; + PROCESS_INFORMATION process{}; + Check(CreateProcessA(nullptr, mutable_command.data(), nullptr, nullptr, FALSE, + CREATE_NO_WINDOW, nullptr, nullptr, &startup, + &process) != 0, + "CreateProcess failed"); + Check(WaitForSingleObject(process.hProcess, 10000) == WAIT_OBJECT_0, + "death test timed out"); + DWORD exit_code = 0; + Check( + GetExitCodeProcess(process.hProcess, &exit_code) != 0 && + (exit_code == 322 || exit_code == EXCEPTION_NONCONTINUABLE_EXCEPTION), + "death case did not use the PageManager fatal exit"); + CloseHandle(process.hThread); + CloseHandle(process.hProcess); #else - const pid_t pid = ::fork(); - Check(pid >= 0, "fork failed"); - if (pid == 0) { - ::execl("/proc/self/exe", "PageManagerTests", "--death", name, nullptr); - std::_Exit(0x7e); - } - int status = 0; - Check(::waitpid(pid, &status, 0) == pid, "waitpid failed"); - // Exit status carries only the low 8 bits. - const bool fatal_exit = WIFEXITED(status) && WEXITSTATUS(status) == (322 & 0xff); - const bool fatal_signal = WIFSIGNALED(status); - Check(fatal_exit || fatal_signal, "death case did not use the PageManager fatal exit"); + const pid_t pid = ::fork(); + Check(pid >= 0, "fork failed"); + if (pid == 0) { + ::execl("/proc/self/exe", "PageManagerTests", "--death", name, nullptr); + std::_Exit(0x7e); + } + int status = 0; + Check(::waitpid(pid, &status, 0) == pid, "waitpid failed"); + const bool fatal_exit = + WIFEXITED(status) && WEXITSTATUS(status) == (322 & 0xff); + const bool fatal_signal = WIFSIGNALED(status); + Check(fatal_exit || fatal_signal, + "death case did not use the PageManager fatal exit"); #endif } void TestFatalPaths() { - for (const char* name: - {"invalid-range", "unknown-untrack", "destructor-watch", "non-write", "callback-false", - "reentrant", "cross-reentrant", "concurrent-non-write"}) { - CheckDeathCase(name); - } + for (const char *name : + {"invalid-range", "unknown-untrack", "destructor-watch"}) { + CheckDeathCase(name); + } } -void TestConcurrentFault() { - FaultContext context; - PageManager manager(InvalidateFault, &context); - context.manager = &manager; - context.block = true; - const auto page_size = manager.GetPageSize(); - auto* memory = Allocate(page_size); - const auto address = reinterpret_cast(memory); - manager.OnGpuMap(address, page_size); - manager.UpdatePageWatchers(true, address, page_size); - bool first_result = false; - bool second_result = false; - std::thread first([&] { first_result = manager.HandleFault(PageFaultAccess::Write, address); }); - while (!context.entered.load(std::memory_order_acquire)) { - std::this_thread::yield(); - } - std::thread second( - [&] { second_result = manager.HandleFault(PageFaultAccess::Write, address); }); - context.release.store(true, std::memory_order_release); - first.join(); - second.join(); - Check(first_result && second_result && context.calls.load(std::memory_order_relaxed) == 1, - "concurrent faults dispatched invalidation more than once"); - manager.OnGpuUnmap(address, page_size); - Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); -} - -void TestExternalDirtyTransferDuringResolution() { - FaultContext context; - PageManager manager(InvalidateFault, &context); - context.manager = &manager; - context.block = true; - const auto page_size = manager.GetPageSize(); - auto* memory = Allocate(page_size); - const auto address = reinterpret_cast(memory); - manager.OnGpuMap(address, page_size); - manager.UpdatePageWatchers(true, address, page_size); - bool handled = false; - std::thread fault([&] { handled = manager.HandleFault(PageFaultAccess::Write, address); }); - while (!context.entered.load(std::memory_order_acquire)) { - std::this_thread::yield(); - } - manager.UpdatePageWatchers(false, address, page_size); - context.release.store(true, std::memory_order_release); - fault.join(); - Check(handled && !manager.IsTracked(address) && IsWritable(memory), - "external dirty transfer did not satisfy active resolution"); - manager.OnGpuUnmap(address, page_size); - Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); -} - -#endif - } // namespace namespace Libs::LibKernel::Memory { -bool ProtectGuestHostMemory(uint64_t vaddr, uint64_t size, Common::VirtualMemory::Mode mode) { - return ProtectAddressSpace(vaddr, size, mode); +bool ProtectGuestHostMemory(uint64_t vaddr, uint64_t size, + Common::VirtualMemory::Mode mode) { + return ProtectAddressSpace(vaddr, size, mode); } } // namespace Libs::LibKernel::Memory -int main(int argc, char** argv) { -#if 1 - if (argc == 3 && std::strcmp(argv[1], "--death") == 0) { - RunDeathCase(argv[2]); - } - TestWatchFaultAndUnwatch(); - TestSharedWatcherFault(); - TestReadWriteWatcherFault(); - TestPermittedMappedLateFaultsResume(); - TestPartialMappingUnmapPreservesTokens(); - TestNativeDelayedReadAfterModeDowngrade(); - TestDelayedFaultAfterExplicitUnwatch(); - TestNativeAccessViolation(); - TestCrossRegionRange(); - TestBatchedWatcherRanges(); - TestConcurrentFault(); - TestExternalDirtyTransferDuringResolution(); - TestFatalPaths(); - std::puts("PageManagerTests: all cases passed"); - return 0; -#else - (void)argc; - (void)argv; - std::fputs("PageManagerTests: unsupported platform\n", stderr); - return 1; -#endif +int main(int argc, char **argv) { + if (argc == 3 && std::strcmp(argv[1], "--death") == 0) { + RunDeathCase(argv[2]); + } + TestWatchAndUnwatch(); + TestSharedWatcherCounts(); + TestMixedWatcherModes(); + TestCrossRegionRange(); + TestBatchedWatcherRanges(); + TestFatalPaths(); + std::puts("PageManagerTests: all cases passed"); + return 0; } diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index eaaf06f..d0f27a3 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -3135,10 +3135,7 @@ public: Require(name, "partial image unmap tracking", partial_unmap_image_id && !texture_cache.FindImageFromRange(partial_unmap_image.info.data.address, - 0x2000, false) && - !texture_cache.InvalidateMemory( - PageFaultAccess::Write, partial_unmap_image.info.data.address + 0x1000, - 0x1000, PageFaultPhase::Invalidate), + 0x2000, false), "partial unmap left the deleted image's mapped tail tracked"); constexpr uint64_t unformatted_alias_offset = 0x2500000; @@ -14459,17 +14456,6 @@ void CheckEmbeddedFetchVertexOffset() { std::printf("[host] %-32s ok\n", "EmbeddedFetchVertexOffset"); } -struct CacheFaultContext { - TextureCache* texture = nullptr; -}; - -bool CacheFault(void* opaque, PageFaultAccess access, uint64_t vaddr, uint64_t size, - PageFaultPhase phase) noexcept { - auto* context = static_cast(opaque); - return context != nullptr && context->texture != nullptr && - context->texture->InvalidateMemory(access, vaddr, size, phase); -} - [[noreturn]] void RunReverseRenderTargetDeathCase() { (void)TextureGetRenderTargetFormat(12u, 7u, 3u); std::_Exit(0x7f); @@ -16110,7 +16096,7 @@ void CheckStorageTextureGpuOwnedRebindState() { auto* memory = reinterpret_cast(guest_memory); Require("StorageTextureGpuOwnedRebind", "allocation", guest_memory == base, "fixed guest-owner allocation failed"); - PageManager page_manager(CacheFault, nullptr); + PageManager page_manager; MemoryTracker tracker(page_manager); page_manager.OnGpuMap(base, size); tracker.ForEachUploadRange(