diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index bd60568..9ce9a3c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -345,6 +345,11 @@ add_executable(page_manager_tests EXCLUDE_FROM_ALL ) target_include_directories(page_manager_tests PRIVATE ${inc_headers}) +add_executable(bit_array_tests EXCLUDE_FROM_ALL + ../tests/BitArrayTests.cpp +) +target_include_directories(bit_array_tests PRIVATE ${inc_headers}) + add_executable(memory_tracker_tests EXCLUDE_FROM_ALL ../tests/MemoryTrackerTests.cpp graphics/host_gpu/pageManager.cpp @@ -457,6 +462,7 @@ if(BUILD_TESTING) add_test(NAME image_page_table COMMAND $) add_test(NAME memory_tracker COMMAND $) add_test(NAME page_manager COMMAND $) + add_test(NAME bit_array COMMAND $) add_test(NAME shader_vertex_metadata COMMAND $) add_test(NAME shader_stage_runtime COMMAND $) add_test(NAME resource_tracking COMMAND $) @@ -502,6 +508,7 @@ if(BUILD_TESTING) image_page_table_tests memory_tracker_tests page_manager_tests + bit_array_tests shader_vertex_metadata_tests shader_stage_runtime_tests resource_tracking_tests diff --git a/src/common/bitArray.h b/src/common/bitArray.h new file mode 100644 index 0000000..df9ebdb --- /dev/null +++ b/src/common/bitArray.h @@ -0,0 +1,260 @@ +#ifndef EMULATOR_SRC_COMMON_BITARRAY_H_ +#define EMULATOR_SRC_COMMON_BITARRAY_H_ + +#include +#include +#include +#include +#include +#include + +namespace Common { + +template +class BitArray final { + static_assert(N != 0, "BitArray size must be nonzero"); + static_assert(N % 64 == 0, "BitArray size must be a multiple of 64 bits"); + + static constexpr size_t BITS_PER_WORD = 64; + static constexpr size_t WORD_COUNT = N / BITS_PER_WORD; + +public: + using Range = std::pair; + + class Iterator final { + public: + using iterator_category = std::forward_iterator_tag; + using value_type = Range; + using difference_type = std::ptrdiff_t; + using pointer = const Range*; + using reference = const Range&; + + Iterator(const BitArray& bits, size_t start) + : m_bits(bits), m_range(bits.FirstRangeFrom(start)) {} + + Iterator& operator++() { + m_range = m_bits.FirstRangeFrom(m_range.second); + return *this; + } + + [[nodiscard]] bool operator==(const Iterator& other) const { + return &m_bits == &other.m_bits && m_range == other.m_range; + } + + [[nodiscard]] bool operator!=(const Iterator& other) const { return !(*this == other); } + + [[nodiscard]] reference operator*() const { return m_range; } + [[nodiscard]] pointer operator->() const { return &m_range; } + + private: + const BitArray& m_bits; + Range m_range; + }; + + using const_iterator = Iterator; + + constexpr BitArray() = default; + + constexpr BitArray(const BitArray& other, size_t start, size_t end) { + if (start >= end || end > N) { + return; + } + + const auto first_word = start / BITS_PER_WORD; + const auto last_word = (end - 1) / BITS_PER_WORD; + const auto start_bit = start % BITS_PER_WORD; + const auto end_bit = (end - 1) % BITS_PER_WORD; + const auto start_mask = ~uint64_t {0} << start_bit; + const auto end_mask = + end_bit == BITS_PER_WORD - 1 ? ~uint64_t {0} : (uint64_t {1} << (end_bit + 1)) - 1; + + if (first_word == last_word) { + m_data[first_word] = other.m_data[first_word] & start_mask & end_mask; + return; + } + + m_data[first_word] = other.m_data[first_word] & start_mask; + for (auto word = first_word + 1; word < last_word; word++) { + m_data[word] = other.m_data[word]; + } + m_data[last_word] = other.m_data[last_word] & end_mask; + } + + [[nodiscard]] constexpr bool Get(size_t index) const { + return (m_data[index / BITS_PER_WORD] & (uint64_t {1} << (index % BITS_PER_WORD))) != 0; + } + + constexpr void Set(size_t index) { + m_data[index / BITS_PER_WORD] |= uint64_t {1} << (index % BITS_PER_WORD); + } + + constexpr void Unset(size_t index) { + m_data[index / BITS_PER_WORD] &= ~(uint64_t {1} << (index % BITS_PER_WORD)); + } + + constexpr void SetRange(size_t start, size_t end) { + if (start >= end || end > N) { + return; + } + + const auto first_word = start / BITS_PER_WORD; + const auto last_word = (end - 1) / BITS_PER_WORD; + const auto start_bit = start % BITS_PER_WORD; + const auto end_bit = (end - 1) % BITS_PER_WORD; + const auto start_mask = ~uint64_t {0} << start_bit; + const auto end_mask = + end_bit == BITS_PER_WORD - 1 ? ~uint64_t {0} : (uint64_t {1} << (end_bit + 1)) - 1; + + if (first_word == last_word) { + m_data[first_word] |= start_mask & end_mask; + return; + } + + m_data[first_word] |= start_mask; + for (auto word = first_word + 1; word < last_word; word++) { + m_data[word] = ~uint64_t {0}; + } + m_data[last_word] |= end_mask; + } + + constexpr void UnsetRange(size_t start, size_t end) { + if (start >= end || end > N) { + return; + } + + const auto first_word = start / BITS_PER_WORD; + const auto last_word = (end - 1) / BITS_PER_WORD; + const auto start_bit = start % BITS_PER_WORD; + const auto end_bit = (end - 1) % BITS_PER_WORD; + const auto start_mask = (uint64_t {1} << start_bit) - 1; + const auto end_mask = + end_bit == BITS_PER_WORD - 1 ? uint64_t {0} : ~((uint64_t {1} << (end_bit + 1)) - 1); + + if (first_word == last_word) { + m_data[first_word] &= start_mask | end_mask; + return; + } + + m_data[first_word] &= start_mask; + for (auto word = first_word + 1; word < last_word; word++) { + m_data[word] = 0; + } + m_data[last_word] &= end_mask; + } + + constexpr void Clear() { m_data.fill(0); } + constexpr void Fill() { m_data.fill(~uint64_t {0}); } + + [[nodiscard]] constexpr bool None() const { + uint64_t combined = 0; + for (const auto word: m_data) { + combined |= word; + } + return combined == 0; + } + + [[nodiscard]] constexpr bool Any() const { return !None(); } + + [[nodiscard]] constexpr Range FirstRangeFrom(size_t start) const { + if (start >= N) { + return {N, N}; + } + + auto word_index = start / BITS_PER_WORD; + auto word = m_data[word_index] & (~uint64_t {0} << (start % BITS_PER_WORD)); + while (word == 0) { + word_index++; + if (word_index == WORD_COUNT) { + return {N, N}; + } + word = m_data[word_index]; + } + + const auto first = word_index * BITS_PER_WORD + std::countr_zero(word); + const auto first_bit = first % BITS_PER_WORD; + const auto first_ones = + static_cast(std::countr_one(m_data[word_index] >> first_bit)); + if (first_bit + first_ones < BITS_PER_WORD) { + return {first, first + first_ones}; + } + + for (word_index++; word_index < WORD_COUNT; word_index++) { + word = m_data[word_index]; + if (word != ~uint64_t {0}) { + return {first, word_index * BITS_PER_WORD + std::countr_one(word)}; + } + } + return {first, N}; + } + + [[nodiscard]] constexpr Range FirstRange() const { return FirstRangeFrom(0); } + + [[nodiscard]] constexpr Range LastRangeFrom(size_t end) const { + if (end == 0) { + return {0, 0}; + } + if (end > N) { + end = N; + } + + auto word_index = (end - 1) / BITS_PER_WORD; + const auto end_bit = (end - 1) % BITS_PER_WORD; + const auto end_mask = + end_bit == BITS_PER_WORD - 1 ? ~uint64_t {0} : (uint64_t {1} << (end_bit + 1)) - 1; + auto word = m_data[word_index] & end_mask; + while (word == 0) { + if (word_index == 0) { + return {0, 0}; + } + word = m_data[--word_index]; + } + + const auto empty_bits = static_cast(std::countl_zero(word)); + const auto ones = static_cast(std::countl_one(word << empty_bits)); + const auto last = (word_index + 1) * BITS_PER_WORD - empty_bits; + if (empty_bits + ones < BITS_PER_WORD) { + return {last - ones, last}; + } + + while (word_index != 0) { + word = m_data[--word_index]; + if (word != ~uint64_t {0}) { + return {(word_index + 1) * BITS_PER_WORD - std::countl_one(word), last}; + } + } + return {0, last}; + } + + [[nodiscard]] constexpr Range LastRange() const { return LastRangeFrom(N); } + + [[nodiscard]] const_iterator begin() const { return Iterator(*this, 0); } + [[nodiscard]] const_iterator end() const { return Iterator(*this, N); } + + constexpr BitArray& operator^=(const BitArray& other) { + for (size_t word = 0; word < WORD_COUNT; word++) { + m_data[word] ^= other.m_data[word]; + } + return *this; + } + + [[nodiscard]] constexpr BitArray operator^(const BitArray& other) const { + auto result = *this; + result ^= other; + return result; + } + + [[nodiscard]] constexpr BitArray operator~() const { + auto result = *this; + for (auto& word: result.m_data) { + word = ~word; + } + return result; + } + +private: + std::array m_data {}; +}; + +} // namespace Common + +#endif // EMULATOR_SRC_COMMON_BITARRAY_H_ diff --git a/src/graphics/host_gpu/memoryTracker.cpp b/src/graphics/host_gpu/memoryTracker.cpp index b51b61c..441696e 100644 --- a/src/graphics/host_gpu/memoryTracker.cpp +++ b/src/graphics/host_gpu/memoryTracker.cpp @@ -6,13 +6,7 @@ namespace Libs::Graphics { static_assert(std::atomic::is_always_lock_free); -MemoryTracker::MemoryTracker(PageManager& page_manager, PageWatchMode gpu_watch_mode) - : m_page_manager(page_manager), m_gpu_watch_mode(gpu_watch_mode) { - switch (m_gpu_watch_mode) { - case PageWatchMode::Write: - case PageWatchMode::ReadWrite: break; - default: EXIT("unsupported memory tracker GPU page-watch mode\n"); - } +MemoryTracker::MemoryTracker(PageManager& page_manager): m_page_manager(page_manager) { m_regions = std::make_unique[]>(REGION_COUNT); for (size_t i = 0; i < REGION_COUNT; i++) { m_regions[i].store(nullptr, std::memory_order_relaxed); @@ -106,34 +100,25 @@ void MemoryTracker::MarkRegionAsCpuModified(uint64_t vaddr, uint64_t size) { std::lock_guard access(m_access_mutex); Iterate(vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) { std::scoped_lock lock(manager->lock); - const auto changed = - manager->ChangeState(manager->GetCpuAddr() + offset, bytes); - manager->ApplyProtection(changed, false); + manager->ChangeState(manager->GetCpuAddr() + offset, bytes); }); } void MemoryTracker::MarkRegionAsGpuModified(uint64_t vaddr, uint64_t size) { CheckNotInUploadCallback(); std::lock_guard access(m_access_mutex); - Iterate(vaddr, size, [this](RegionManager* manager, uint64_t offset, uint64_t bytes) { + Iterate(vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) { std::scoped_lock lock(manager->lock); - const auto changed = - manager->ChangeState(manager->GetCpuAddr() + offset, bytes); - manager->ApplyGpuProtection(changed, true, m_gpu_watch_mode); + manager->ChangeState(manager->GetCpuAddr() + offset, bytes); }); } void MemoryTracker::UnmarkRegionAsGpuModified(uint64_t vaddr, uint64_t size) { CheckNotInUploadCallback(); std::lock_guard access(m_access_mutex); - Iterate(vaddr, size, [this](RegionManager* manager, uint64_t offset, uint64_t bytes) { + Iterate(vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) { std::scoped_lock lock(manager->lock); - if (!manager->IsFullyModified(offset, bytes)) { - EXIT("cannot clear partially GPU-dirty tracking range\n"); - } - const auto changed = - manager->ChangeState(manager->GetCpuAddr() + offset, bytes); - manager->ApplyGpuProtection(changed, false, m_gpu_watch_mode); + manager->ChangeState(manager->GetCpuAddr() + offset, bytes); }); } @@ -156,9 +141,7 @@ void MemoryTracker::UntrackMemoryLocked(uint64_t vaddr, uint64_t size) { EXIT("cannot untrack GPU-dirty memory\n"); } Iterate(vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) { - const auto changed = - manager->ChangeState(manager->GetCpuAddr() + offset, bytes); - manager->ApplyProtection(changed, false); + manager->ChangeState(manager->GetCpuAddr() + offset, bytes); }); locks.clear(); } diff --git a/src/graphics/host_gpu/memoryTracker.h b/src/graphics/host_gpu/memoryTracker.h index e313e2c..6a9b539 100644 --- a/src/graphics/host_gpu/memoryTracker.h +++ b/src/graphics/host_gpu/memoryTracker.h @@ -18,8 +18,7 @@ namespace Libs::Graphics { class MemoryTracker final { public: - explicit MemoryTracker(PageManager& page_manager, - PageWatchMode gpu_watch_mode = PageWatchMode::ReadWrite); + explicit MemoryTracker(PageManager& page_manager); ~MemoryTracker(); KYTY_CLASS_NO_COPY(MemoryTracker); @@ -56,9 +55,8 @@ public: } Iterate(vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) { - const auto changed = manager->ChangeState( + manager->ChangeState( manager->GetCpuAddr() + offset, bytes); - manager->ApplyProtection(changed, false); }); return false; }; @@ -109,10 +107,8 @@ public: Iterate(vaddr, size, [&](RegionManager* manager, uint64_t offset, uint64_t bytes) { const auto address = manager->GetCpuAddr() + offset; - const auto changed = - manager->template ForEachModifiedRange( - address, bytes, [](uint64_t, uint64_t) noexcept {}); - manager->ApplyGpuProtection(changed, false, m_gpu_watch_mode); + manager->template ForEachModifiedRange( + address, bytes, [](uint64_t, uint64_t) noexcept {}); }); } } @@ -142,13 +138,12 @@ public: }); upload_func(); if (is_written) { - Iterate( - vaddr, size, [this](RegionManager* manager, uint64_t offset, uint64_t bytes) { - const auto changed = manager->template ChangeState( - manager->GetCpuAddr() + offset, bytes); - manager->ApplyGpuProtection(changed, true, m_gpu_watch_mode); - manager->lock.unlock(); - }); + Iterate(vaddr, size, + [](RegionManager* manager, uint64_t offset, uint64_t bytes) { + manager->template ChangeState( + manager->GetCpuAddr() + offset, bytes); + manager->lock.unlock(); + }); } s_upload_owner = previous_upload_owner; } @@ -202,7 +197,6 @@ private: std::mutex m_region_mutex; std::mutex m_access_mutex; PageManager& m_page_manager; - PageWatchMode m_gpu_watch_mode = PageWatchMode::ReadWrite; }; } // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/pageManager.cpp b/src/graphics/host_gpu/pageManager.cpp index 367abb6..c819b11 100644 --- a/src/graphics/host_gpu/pageManager.cpp +++ b/src/graphics/host_gpu/pageManager.cpp @@ -9,7 +9,6 @@ #include #include #include -#include #include #include #include @@ -22,13 +21,9 @@ #undef min #undef max #elif defined(__APPLE__) -#include -#include #include #else #include -#include -#include #include #endif @@ -41,9 +36,8 @@ constexpr uint64_t ADDRESS_SIZE = TRACKER_ADDRESS_SIZE; constexpr uint64_t REGION_COUNT = ADDRESS_SIZE / REGION_SIZE; #if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS -// The tracker reuses Win32 memory-protection tags as internal page-state values (on -// Windows they come from and are what VirtualQuery returns). Mirror the -// canonical Win32 numeric values so the shared state-machine logic is identical. +// The tracker reuses Win32 memory-protection tags as internal page-state values. +// Mirror their canonical numeric values so the shared state-machine logic is identical. constexpr uint32_t PAGE_NOACCESS = 0x01; constexpr uint32_t PAGE_READONLY = 0x02; constexpr uint32_t PAGE_READWRITE = 0x04; @@ -53,8 +47,6 @@ constexpr uint64_t REGION_PAGES = REGION_SIZE / PAGE_SIZE; constexpr uint32_t NO_ACCESS_PROTECTION = PAGE_NOACCESS; constexpr uint32_t READ_ONLY_PROTECTION = PAGE_READONLY; constexpr uint32_t READ_WRITE_PROTECTION = PAGE_READWRITE; -// Zero is the unknown protection sentinel. -constexpr uint32_t UNKNOWN_PROTECTION = 0; [[noreturn]] void FailFast(const char* reason = nullptr) noexcept { std::fputs("PageManager fail-fast: ", stderr); @@ -102,25 +94,6 @@ Common::VirtualMemory::Mode ToMemoryMode(uint32_t protection) { } } -uint32_t CurrentThread() noexcept { -#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS - return GetCurrentThreadId(); -#elif defined(__APPLE__) - return static_cast(pthread_mach_thread_np(pthread_self())); -#elif defined(__linux__) - static thread_local const uint32_t tid = [] { - const auto raw = static_cast(::syscall(SYS_gettid)); - if (raw == 0) { - FailFast("gettid returned the reserved zero owner token"); - } - return raw; - }(); - return tid; -#else - FailFast("page tracking thread identity is unsupported on this platform"); -#endif -} - class SpinGuard final { public: explicit SpinGuard(std::atomic_flag& lock): m_lock(lock) { @@ -154,38 +127,58 @@ uint64_t PageEnd(uint64_t vaddr, uint64_t size) { struct PageManager::Impl { struct PageState { - std::atomic_flag lock = ATOMIC_FLAG_INIT; - uint32_t write_watchers = 0; - uint32_t access_watchers = 0; - uint32_t original_protection = 0; - uint32_t backing_writer = 0; - // Shadow the protection applied through Protect(). - uint32_t current_protection = UNKNOWN_PROTECTION; - bool resolving = false; - }; + uint8_t write_watchers : 7 = 0; + uint8_t access_watchers : 1 = 0; - struct Region { - std::array pages; - }; + [[nodiscard]] uint32_t Perms() const noexcept { + if (access_watchers != 0) { + return NO_ACCESS_PROTECTION; + } + if (write_watchers != 0) { + return READ_ONLY_PROTECTION; + } + return READ_WRITE_PROTECTION; + } - class PageRangeGuard final { - public: - explicit PageRangeGuard(std::span pages): m_pages(pages) { - for (auto* page: m_pages) { - while (page->lock.test_and_set(std::memory_order_acquire)) { - std::atomic_signal_fence(std::memory_order_seq_cst); + template + uint32_t AddDelta(uint64_t address) { + static_assert(delta >= -1 && delta <= 1); + if constexpr (is_read) { + if constexpr (delta == 1) { + if (access_watchers != 0) { + Fatal("read-watcher overflow at 0x%016" PRIx64, address); + } + return ++access_watchers; + } else if constexpr (delta == -1) { + if (access_watchers == 0) { + Fatal("read-watcher underflow at 0x%016" PRIx64, address); + } + return --access_watchers; + } else { + return access_watchers; + } + } else { + if constexpr (delta == 1) { + if (write_watchers == 0x7f) { + Fatal("write-watcher overflow at 0x%016" PRIx64, address); + } + return ++write_watchers; + } else if constexpr (delta == -1) { + if (write_watchers == 0) { + Fatal("write-watcher underflow at 0x%016" PRIx64, address); + } + return --write_watchers; + } else { + return write_watchers; } } } - ~PageRangeGuard() { - for (auto it = m_pages.rbegin(); it != m_pages.rend(); ++it) { - (*it)->lock.clear(std::memory_order_release); - } - } - KYTY_CLASS_NO_COPY(PageRangeGuard); + }; + static_assert(sizeof(PageState) == 1); - private: - std::span m_pages; + struct Region { + std::atomic_flag lock = ATOMIC_FLAG_INIT; + std::array pages; }; Impl() { @@ -215,10 +208,9 @@ struct PageManager::Impl { ~Impl() { for (const auto& region: region_storage) { + SpinGuard lock(region->lock); for (auto& page: region->pages) { - SpinGuard lock(page.lock); - if (page.write_watchers != 0 || page.access_watchers != 0 || - page.backing_writer != 0 || page.resolving) { + if (page.write_watchers != 0 || page.access_watchers != 0) { FailFast("PageManager destroyed with live page state"); } } @@ -246,56 +238,81 @@ struct PageManager::Impl { return ptr; } - PageState& GetPage(Region& region, uint64_t vaddr) const { - return region.pages[(vaddr % REGION_SIZE) / PAGE_SIZE]; - } - - static uint32_t WatcherProtection(const PageState& page) { - if (page.access_watchers != 0) { - return NO_ACCESS_PROTECTION; - } - if (page.write_watchers != 0) { - return READ_ONLY_PROTECTION; - } - return page.original_protection; - } - - static void InitializeProtection(std::span pages) { - for (auto* page: pages) { - page->original_protection = READ_WRITE_PROTECTION; - page->current_protection = READ_WRITE_PROTECTION; - } - } - - void ProtectRange(std::span pages, uint64_t vaddr, uint32_t protection, - std::span expected_old) noexcept { - const auto size = pages.size() * PAGE_SIZE; - if (pages.size() != expected_old.size()) { - FailFast("protection range state size mismatch"); - } - for (size_t i = 0; i < pages.size(); i++) { - const auto actual = pages[i]->current_protection; - if (actual != UNKNOWN_PROTECTION && actual != expected_old[i]) { - Fatal("invalid protection transition at 0x%016" PRIx64 ", old=0x%08" PRIx32 - ", expected=0x%08" PRIx32 ", new=0x%08" PRIx32, - vaddr + i * PAGE_SIZE, actual, expected_old[i], protection); - } - } + void Protect(uint64_t vaddr, uint64_t size, uint32_t protection) noexcept { if (!Libs::LibKernel::Memory::ProtectGuestHostMemory(vaddr, size, ToMemoryMode(protection))) { Fatal("address-space protection failed at 0x%016" PRIx64 ", new=0x%08" PRIx32, vaddr, protection); } - for (auto* page: pages) { - page->current_protection = protection; - } } - void Protect(PageState& page, uint64_t vaddr, uint32_t protection, - uint32_t expected_old) noexcept { - PageState* pages[] = {&page}; - uint32_t expected[] = {expected_old}; - ProtectRange(pages, vaddr, protection, expected); + template + void UpdateRegionWatchers(Region& region, uint64_t base_addr, size_t first, size_t last, + const RegionBits* mask = nullptr) { + SpinGuard lock(region.lock); + auto perms = region.pages[first].Perms(); + uint64_t range_begin = 0; + uint64_t range_bytes = 0; + uint64_t potential_range_bytes = 0; + + const auto release_pending = [&] { + if (range_bytes != 0) { + Protect(base_addr + range_begin * PAGE_SIZE, range_bytes, perms); + range_bytes = 0; + potential_range_bytes = 0; + } + }; + + for (size_t page_index = first; page_index < last; page_index++) { + auto& page = region.pages[page_index]; + const auto address = base_addr + page_index * PAGE_SIZE; + const bool update = !masked || mask->Get(page_index); + + const auto old_perms = page.Perms(); + const auto new_count = update ? page.AddDelta(address) + : page.AddDelta<0, is_read>(address); + const auto new_perms = page.Perms(); + + if (new_perms != perms) [[unlikely]] { + release_pending(); + perms = new_perms; + } else if (range_bytes != 0) { + potential_range_bytes += PAGE_SIZE; + } + + if (!update) { + continue; + } + + const bool watcher_edge = (track && new_count == 1) || (!track && new_count == 0); + if (watcher_edge && old_perms != new_perms) { + if (range_bytes == 0) { + range_begin = page_index; + potential_range_bytes = PAGE_SIZE; + } + range_bytes = potential_range_bytes; + } + } + + release_pending(); + } + + template + void UpdatePageWatchers(uint64_t vaddr, uint64_t size) { + const auto begin = PageStart(vaddr); + const auto end = PageEnd(vaddr, size); + for (auto chunk_begin = begin; chunk_begin < end;) { + const auto chunk_end = std::min(end, (chunk_begin / REGION_SIZE + 1) * REGION_SIZE); + const auto region_base = chunk_begin / REGION_SIZE * REGION_SIZE; + auto* region = track ? GetOrCreateRegion(chunk_begin) : FindRegion(chunk_begin); + if (region == nullptr) { + Fatal("untracking unknown page 0x%016" PRIx64, chunk_begin); + } + const auto first = static_cast((chunk_begin - region_base) / PAGE_SIZE); + const auto last = static_cast((chunk_end - region_base) / PAGE_SIZE); + UpdateRegionWatchers(*region, region_base, first, last); + chunk_begin = chunk_end; + } } std::unique_ptr[]> regions; @@ -313,212 +330,48 @@ uint64_t PageManager::GetPageSize() const { return PAGE_SIZE; } -void PageManager::UpdatePageWatchers(bool track, uint64_t vaddr, uint64_t size, - PageWatchMode mode) { - if (mode != PageWatchMode::Write && mode != PageWatchMode::ReadWrite) { - Fatal("invalid watcher mode"); - } - const auto begin = PageStart(vaddr); - const auto end = PageEnd(vaddr, size); - for (auto chunk_begin = begin; chunk_begin < end;) { - const auto chunk_end = std::min(end, (chunk_begin / REGION_SIZE + 1) * REGION_SIZE); - auto* region = - track ? m_impl->GetOrCreateRegion(chunk_begin) : m_impl->FindRegion(chunk_begin); - if (region == nullptr) { - Fatal("untracking unknown page 0x%016" PRIx64, chunk_begin); - } - - const auto page_count = static_cast((chunk_end - chunk_begin) / PAGE_SIZE); - std::vector pages; - pages.reserve(page_count); - for (auto address = chunk_begin; address < chunk_end; address += PAGE_SIZE) { - pages.push_back(&m_impl->GetPage(*region, address)); - } - Impl::PageRangeGuard lock(pages); - - std::vector first_watchers(page_count); - for (size_t i = 0; i < page_count; i++) { - auto& page = *pages[i]; - const auto address = chunk_begin + i * PAGE_SIZE; - if (page.resolving && track) { - FailFast("new page watcher raced active fault resolution"); - } - auto& watchers = - (mode == PageWatchMode::ReadWrite ? page.access_watchers : page.write_watchers); - if (track) { - if (watchers == std::numeric_limits::max()) { - Fatal("watcher overflow at 0x%016" PRIx64, address); - } - first_watchers[i] = page.write_watchers == 0 && page.access_watchers == 0; - } else { - if (watchers == 0) { - Fatal("watcher underflow at 0x%016" PRIx64, address); - } - if (page.backing_writer != 0 && page.backing_writer != CurrentThread()) { - Fatal("backing write ownership changed at 0x%016" PRIx64, address); - } - } - } - - if (track) { - for (size_t first = 0; first < page_count;) { - while (first < page_count && first_watchers[first] == 0) { - first++; - } - auto last = first; - while (last < page_count && first_watchers[last] != 0) { - last++; - } - if (first != last) { - Impl::InitializeProtection(std::span {pages}.subspan(first, last - first)); - } - first = last; - } - } - - std::vector old_protections(page_count); - std::vector new_protections(page_count); - std::vector transitions(page_count); - for (size_t i = 0; i < page_count; i++) { - auto& page = *pages[i]; - auto& watchers = - (mode == PageWatchMode::ReadWrite ? page.access_watchers : page.write_watchers); - const auto old_protection = Impl::WatcherProtection(page); - if (track) { - watchers++; - } else { - watchers--; - } - const auto new_protection = Impl::WatcherProtection(page); - old_protections[i] = old_protection; - new_protections[i] = new_protection; - if (new_protection != old_protection && (track || page.backing_writer == 0)) { - transitions[i] = 1; - } - } - - for (size_t first = 0; first < page_count;) { - while (first < page_count && transitions[first] == 0) { - first++; - } - if (first == page_count) { - break; - } - const auto protection = new_protections[first]; - auto current = first + 1; - auto last = current; - for (; current < page_count && new_protections[current] == protection; current++) { - if (old_protections[current] != new_protections[current] && - transitions[current] == 0) { - break; - } - if (transitions[current] != 0) { - last = current + 1; - } - } - m_impl->ProtectRange(std::span {pages}.subspan(first, last - first), - chunk_begin + first * PAGE_SIZE, protection, - std::span {old_protections}.subspan(first, last - first)); - first = current; - } - - for (auto* page: pages) { - if (!track && page->backing_writer == 0 && page->write_watchers == 0 && - page->access_watchers == 0) { - page->original_protection = 0; - } - } - chunk_begin = chunk_end; - } +template +void PageManager::UpdatePageWatchers(uint64_t vaddr, uint64_t size) { + m_impl->UpdatePageWatchers(vaddr, size); } +template void PageManager::UpdatePageWatchers(uint64_t, uint64_t); +template void PageManager::UpdatePageWatchers(uint64_t, uint64_t); + +template +void PageManager::UpdatePageWatchersForRegion(uint64_t base_addr, RegionBits& mask) { + if (base_addr % REGION_SIZE != 0 || base_addr >= ADDRESS_SIZE || + REGION_SIZE > ADDRESS_SIZE - base_addr) { + Fatal("invalid tracking region base 0x%016" PRIx64, base_addr); + } + + const auto start_range = mask.FirstRange(); + const auto end_range = mask.LastRange(); + if (start_range.first == REGION_PAGES) { + FailFast("empty region watcher mask"); + } + const auto first = start_range.first; + const auto last = end_range.second; + if (start_range.second == end_range.second) { + m_impl->UpdatePageWatchers(base_addr + first * PAGE_SIZE, + (last - first) * PAGE_SIZE); + return; + } + + auto* region = track ? m_impl->GetOrCreateRegion(base_addr) : m_impl->FindRegion(base_addr); + if (region == nullptr) { + Fatal("untracking unknown region 0x%016" PRIx64, base_addr); + } + m_impl->UpdateRegionWatchers(*region, base_addr, first, last, &mask); +} + +template void PageManager::UpdatePageWatchersForRegion(uint64_t, RegionBits&); +template void PageManager::UpdatePageWatchersForRegion(uint64_t, RegionBits&); +template void PageManager::UpdatePageWatchersForRegion(uint64_t, RegionBits&); +template void PageManager::UpdatePageWatchersForRegion(uint64_t, RegionBits&); + void PageManager::OnGpuMap(uint64_t, uint64_t) {} void PageManager::OnGpuUnmap(uint64_t, uint64_t) {} -PageManager::BackingWrite::BackingWrite(PageManager& manager, uint64_t vaddr, - uint64_t size) noexcept - : m_manager(manager), m_vaddr(vaddr), m_size(size) { - m_manager.BeginBackingWrite(vaddr, size); -} - -PageManager::BackingWrite::~BackingWrite() { - m_manager.EndBackingWrite(m_vaddr, m_size); -} - -std::vector> -PageManager::ReserveBackingWrites(std::span ranges) { - if (ranges.empty()) { - Fatal("cannot reserve empty backing-write ranges"); - } - std::vector> writes; - writes.reserve(ranges.size()); - uint64_t begin = 0; - uint64_t end = 0; - for (const auto& range: ranges) { - if (range.address == 0 || range.size == 0 || range.size > UINT64_MAX - range.address || - range.address + range.size > UINT64_MAX - (PAGE_SIZE - 1)) { - Fatal("invalid backing-write range"); - } - const auto page_begin = PageStart(range.address); - const auto page_end = PageStart(range.address + range.size + PAGE_SIZE - 1); - if (begin != 0 && page_begin > end) { - writes.push_back(std::make_unique(*this, begin, end - begin)); - begin = 0; - } - if (begin == 0) { - begin = page_begin; - end = page_end; - } else { - end = std::max(end, page_end); - } - } - writes.push_back(std::make_unique(*this, begin, end - begin)); - return writes; -} - -void PageManager::BeginBackingWrite(uint64_t vaddr, uint64_t size) noexcept { - const auto end = PageEnd(vaddr, size); - const auto writer = CurrentThread(); - for (auto address = PageStart(vaddr); address < end; address += PAGE_SIZE) { - auto* region = m_impl->FindRegion(address); - if (region == nullptr) { - Fatal("backing write reserves an unknown page at 0x%016" PRIx64, address); - } - auto& page = m_impl->GetPage(*region, address); - SpinGuard lock(page.lock); - if (page.resolving || page.backing_writer != 0 || page.access_watchers == 0) { - Fatal("backing write races page resolution at 0x%016" PRIx64, address); - } - page.resolving = true; - page.backing_writer = writer; - } -} - -void PageManager::EndBackingWrite(uint64_t vaddr, uint64_t size) noexcept { - const auto end = PageEnd(vaddr, size); - const auto writer = CurrentThread(); - for (auto address = PageStart(vaddr); address < end; address += PAGE_SIZE) { - auto* region = m_impl->FindRegion(address); - if (region == nullptr) { - FailFast("backing write ended for an unknown page"); - } - auto& page = m_impl->GetPage(*region, address); - SpinGuard lock(page.lock); - if (!page.resolving || page.backing_writer != writer) { - FailFast("backing write ended without matching owner and resolving state"); - } - const auto old_protection = NO_ACCESS_PROTECTION; - const auto new_protection = Impl::WatcherProtection(page); - if (new_protection != old_protection) { - m_impl->Protect(page, address, new_protection, old_protection); - } - if (page.write_watchers == 0 && page.access_watchers == 0) { - page.original_protection = 0; - } - page.backing_writer = 0; - page.resolving = false; - } -} - } // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/pageManager.h b/src/graphics/host_gpu/pageManager.h index 619602e..ed304ee 100644 --- a/src/graphics/host_gpu/pageManager.h +++ b/src/graphics/host_gpu/pageManager.h @@ -2,31 +2,16 @@ #define EMULATOR_SRC_GRAPHICS_HOST_GPU_PAGEMANAGER_H_ #include "common/common.h" -#include "graphics/host_gpu/rangeSet.h" +#include "graphics/host_gpu/regionDefinitions.h" #include -#include -#include namespace Libs::Graphics { enum class PageFaultAccess { Read, Write, Execute, Unknown }; -enum class PageWatchMode { Write, ReadWrite }; class PageManager final { public: - class BackingWrite final { - public: - BackingWrite(PageManager& manager, uint64_t vaddr, uint64_t size) noexcept; - ~BackingWrite(); - KYTY_CLASS_NO_COPY(BackingWrite); - - private: - PageManager& m_manager; - uint64_t m_vaddr = 0; - uint64_t m_size = 0; - }; - PageManager(); // The owner must stop all PageManager callers before destruction. ~PageManager(); @@ -35,18 +20,14 @@ public: [[nodiscard]] uint64_t GetPageSize() const; - void UpdatePageWatchers(bool track, uint64_t vaddr, uint64_t size, - PageWatchMode mode = PageWatchMode::Write); + template + void UpdatePageWatchers(uint64_t vaddr, uint64_t size); + template + void UpdatePageWatchersForRegion(uint64_t base_addr, RegionBits& mask); void OnGpuMap(uint64_t vaddr, uint64_t size); void OnGpuUnmap(uint64_t vaddr, uint64_t size); - [[nodiscard]] std::vector> - ReserveBackingWrites(std::span ranges); - private: - void BeginBackingWrite(uint64_t vaddr, uint64_t size) noexcept; - void EndBackingWrite(uint64_t vaddr, uint64_t size) noexcept; - struct Impl; std::unique_ptr m_impl; }; diff --git a/src/graphics/host_gpu/regionDefinitions.h b/src/graphics/host_gpu/regionDefinitions.h index 2b47828..1c5ee11 100644 --- a/src/graphics/host_gpu/regionDefinitions.h +++ b/src/graphics/host_gpu/regionDefinitions.h @@ -1,10 +1,9 @@ #ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_REGIONDEFINITIONS_H_ #define EMULATOR_SRC_GRAPHICS_HOST_GPU_REGIONDEFINITIONS_H_ +#include "common/bitArray.h" #include "common/common.h" -#include - namespace Libs::Graphics { constexpr uint64_t TRACKER_PAGE_SIZE = 4ull * 1024ull; @@ -13,7 +12,8 @@ constexpr uint64_t TRACKER_ADDRESS_SIZE = 1ull << 40u; constexpr size_t TRACKER_REGION_PAGES = TRACKER_REGION_SIZE / TRACKER_PAGE_SIZE; enum class DirtySource { Cpu, Gpu }; -using RegionBits = std::bitset; +using RegionBits = Common::BitArray; +static_assert(sizeof(RegionBits) == TRACKER_REGION_PAGES / 8); } // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/regionManager.h b/src/graphics/host_gpu/regionManager.h index 8b053e5..ee9ced6 100644 --- a/src/graphics/host_gpu/regionManager.h +++ b/src/graphics/host_gpu/regionManager.h @@ -76,8 +76,9 @@ public: if (m_cpu_addr % TRACKER_REGION_SIZE != 0) { EXIT("invalid region tracking manager construction\n"); } - m_cpu_dirty.set(); - m_writable.set(); + m_cpu_dirty.Fill(); + m_writable.Fill(); + m_readable.Fill(); } KYTY_CLASS_NO_COPY(RegionManager); @@ -87,106 +88,81 @@ public: [[nodiscard]] bool IsModified(uint64_t offset, uint64_t size) const { const auto [start, end] = GetPageRange(m_cpu_addr + offset, size); const auto& bits = GetBits(); - for (auto page = start; page < end; page++) { - if (bits.test(page)) { - return true; - } - } - return false; - } - - template - [[nodiscard]] bool IsFullyModified(uint64_t offset, uint64_t size) const { - const auto [start, end] = GetPageRange(m_cpu_addr + offset, size); - const auto& bits = GetBits(); - for (auto page = start; page < end; page++) { - if (!bits.test(page)) { - return false; - } - } - return true; + return RegionBits(bits, start, end).Any(); } template - RegionBits ChangeState(uint64_t vaddr, uint64_t size) { + void ChangeState(uint64_t vaddr, uint64_t size) { const auto [start, end] = GetPageRange(vaddr, size); if constexpr (source == DirtySource::Cpu && enable) { - for (auto page = start; page < end; page++) { - if (m_gpu_dirty.test(page)) { - EXIT("CPU dirty state conflicts with GPU dirty state\n"); - } + if (RegionBits(m_gpu_dirty, start, end).Any()) { + EXIT("CPU dirty state conflicts with GPU dirty state\n"); } } if constexpr (source == DirtySource::Gpu && enable) { - for (auto page = start; page < end; page++) { - if (m_cpu_dirty.test(page)) { - EXIT("GPU dirty state conflicts with CPU dirty state\n"); - } + if (RegionBits(m_cpu_dirty, start, end).Any()) { + EXIT("GPU dirty state conflicts with CPU dirty state\n"); } } - auto& bits = GetBits(); - auto changed = bits; - for (auto page = start; page < end; page++) { - bits.set(page, enable); + auto& bits = GetBits(); + if constexpr (enable) { + bits.SetRange(start, end); + } else { + bits.UnsetRange(start, end); } - changed ^= bits; if constexpr (source == DirtySource::Cpu) { - changed = m_cpu_dirty ^ m_writable; - m_writable = m_cpu_dirty; + UpdateCpuProtection(); + } else { + UpdateGpuProtection(); } - return changed; } template - RegionBits ForEachModifiedRange(uint64_t vaddr, uint64_t size, Func&& func) { + void ForEachModifiedRange(uint64_t vaddr, uint64_t size, Func&& func) { const auto [start, end] = GetPageRange(vaddr, size); - auto mask = GetBits(); - for (auto page = 0u; page < start; page++) { - mask.reset(page); - } - for (auto page = end; page < TRACKER_REGION_PAGES; page++) { - mask.reset(page); - } + RegionBits mask(GetBits(), start, end); if constexpr (clear) { - auto& bits = GetBits(); - for (auto page = start; page < end; page++) { - if (mask.test(page)) { - bits.reset(page); - } - } + GetBits().UnsetRange(start, end); } if constexpr (source == DirtySource::Cpu && clear) { - auto changed = m_cpu_dirty ^ m_writable; - m_writable = m_cpu_dirty; - ApplyProtection(changed, true); + UpdateCpuProtection(); ForEachRange(mask, std::forward(func)); - return changed; + return; + } + if constexpr (source == DirtySource::Gpu && clear) { + UpdateGpuProtection(); } ForEachRange(mask, std::forward(func)); - if constexpr (clear) { - return mask; - } - return {}; - } - - void ApplyProtection(const RegionBits& changed, bool track) { - ForEachRange(changed, [this, track](uint64_t vaddr, uint64_t size) { - m_page_manager.UpdatePageWatchers(track, vaddr, size); - }); - } - - void ApplyGpuProtection(const RegionBits& changed, bool track, PageWatchMode mode) { - if (mode != PageWatchMode::Write && mode != PageWatchMode::ReadWrite) { - EXIT("unsupported GPU page-watch mode\n"); - } - ForEachRange(changed, [this, track, mode](uint64_t vaddr, uint64_t size) { - m_page_manager.UpdatePageWatchers(track, vaddr, size, mode); - }); } TrackingSpinLock lock; private: + template + void UpdateCpuProtection() { + auto mask = m_cpu_dirty ^ m_writable; + m_writable = m_cpu_dirty; + if (mask.None()) { + return; + } + m_page_manager.UpdatePageWatchersForRegion(m_cpu_addr, mask); + } + + template + void UpdateGpuProtection() { + auto readable = ~m_gpu_dirty; + auto mask = readable ^ m_readable; + m_readable = readable; + if (mask.None()) { + return; + } + if constexpr (track) { + m_page_manager.UpdatePageWatchersForRegion(m_cpu_addr, mask); + } else { + m_page_manager.UpdatePageWatchersForRegion(m_cpu_addr, mask); + } + } + template RegionBits& GetBits() { if constexpr (source == DirtySource::Cpu) { @@ -217,18 +193,8 @@ private: template void ForEachRange(const RegionBits& bits, Func&& func) const { - size_t page = 0; - while (page < TRACKER_REGION_PAGES) { - while (page < TRACKER_REGION_PAGES && !bits.test(page)) { - page++; - } - const auto start = page; - while (page < TRACKER_REGION_PAGES && bits.test(page)) { - page++; - } - if (start != page) { - func(m_cpu_addr + start * TRACKER_PAGE_SIZE, (page - start) * TRACKER_PAGE_SIZE); - } + for (const auto [start, end]: bits) { + func(m_cpu_addr + start * TRACKER_PAGE_SIZE, (end - start) * TRACKER_PAGE_SIZE); } } @@ -237,6 +203,7 @@ private: RegionBits m_cpu_dirty; RegionBits m_gpu_dirty; RegionBits m_writable; + RegionBits m_readable; }; } // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/cache/bufferCache.cpp b/src/graphics/host_gpu/renderer/cache/bufferCache.cpp index 62e20e1..0d29be5 100644 --- a/src/graphics/host_gpu/renderer/cache/bufferCache.cpp +++ b/src/graphics/host_gpu/renderer/cache/bufferCache.cpp @@ -123,12 +123,6 @@ struct BufferCache::RetiredBuffer { std::shared_ptr owner; }; -struct BufferCache::PendingBackingPublication { - uint64_t address = 0; - uint64_t size = 0; - uint64_t tick = 0; -}; - std::pair BufferCache::DownloadEnvelope(const DownloadCopy& copy) { if (copy.owner == nullptr || copy.size == 0 || copy.source_offset > copy.owner->Size() || copy.size > copy.owner->Size() - copy.source_offset) { @@ -200,34 +194,26 @@ void BufferCache::QueueGarbageDownload(std::span copies, Ret if (copies.empty()) { return; } - auto downloads = RecordDownloads(copies); - const auto tick = m_scheduler.CurrentTick(); - BeginBackingPublication(retire.address, retire.size, tick); - m_scheduler.DeferOperation([this, downloads = std::move(downloads), retire = std::move(retire), - tick]() mutable { - PublishDownloads(downloads); - { - FaultSafeCacheLock lock(this, m_mutex); - if (m_memory_tracker.IsRegionGpuModified(retire.address, retire.size)) { - m_memory_tracker.ForEachDownloadRange( - retire.address, retire.size, - [&](uint64_t address, uint64_t size) noexcept { - m_memory_tracker.ValidateGpuDirtyPages(m_gpu_modified_ranges, address, size, - "asynchronous garbage retirement"); - }, - [](uint64_t, uint64_t) noexcept {}); - } - for (const auto& range: downloads) { - m_gpu_modified_ranges.Subtract(range.address, range.size); - } - if (m_memory_tracker.IsRegionGpuModified(retire.address, retire.size) || - !m_gpu_modified_ranges.Intersections(retire.address, retire.size).empty()) { - EXIT("BufferCache: asynchronous garbage collection retained GPU ownership\n"); - } - m_memory_tracker.UntrackMemory(retire.address, retire.size); - } - CompleteBackingPublication(retire.address, retire.size, tick); - }); + auto downloads = RecordDownloads(copies); + m_scheduler.DeferOperation( + [this, downloads = std::move(downloads), retire = std::move(retire)]() mutable { + PublishDownloads(downloads); + { + FaultSafeCacheLock lock(this, m_mutex); + for (const auto& range: downloads) { + m_gpu_modified_ranges.Subtract(range.address, range.size); + } + // ForEachDownloadRange reports full tracker pages, and every exact GPU-owned + // interval on those pages was downloaded and removed. Clearing the original + // query therefore cannot orphan a dirty sibling on an edge page. + m_memory_tracker.UnmarkRegionAsGpuModified(retire.address, retire.size); + if (m_memory_tracker.IsRegionGpuModified(retire.address, retire.size) || + !m_gpu_modified_ranges.Intersections(retire.address, retire.size).empty()) { + EXIT("BufferCache: asynchronous garbage collection retained GPU ownership\n"); + } + m_memory_tracker.UntrackMemory(retire.address, retire.size); + } + }); } BufferCache::BufferCache(GraphicContext& graphics, CommandScheduler& scheduler, @@ -240,8 +226,7 @@ BufferCache::BufferCache(GraphicContext& graphics, CommandScheduler& scheduler, m_stream_buffer(graphics, scheduler, MemoryUsage::Stream, 64 * MiB), m_download_buffer(graphics, scheduler, MemoryUsage::Download, 32 * MiB), m_device_buffer(graphics, scheduler, MemoryUsage::DeviceLocal, 128 * MiB), - m_page_manager(page_manager), m_texture_cache(texture_cache), - m_resource_mutex(resource_mutex) { + m_texture_cache(texture_cache), m_resource_mutex(resource_mutex) { std::memset(m_gds_buffer.Mapped().data(), 0, static_cast(m_gds_buffer.Size())); m_gds_buffer.Flush(0, m_gds_buffer.Size()); if (!m_graphics.CanReportMemoryUsage()) { @@ -262,9 +247,6 @@ BufferCache::~BufferCache() { if (!m_gpu_modified_ranges.Empty()) { EXIT("BufferCache: destroyed with pending GPU-modified ranges\n"); } - if (!m_pending_backing_publications.empty()) { - EXIT("BufferCache: destroyed with pending backing publications\n"); - } for (const auto& [vaddr, cached]: m_buffers) { (void)vaddr; if (m_memory_tracker.IsRegionGpuModified(cached->vaddr, cached->size)) { @@ -274,68 +256,6 @@ BufferCache::~BufferCache() { m_buffers.clear(); } -bool BufferCache::SynchronizeBacking(uint64_t vaddr, uint64_t size) { - bool waited = false; - for (;;) { - uint64_t tick = 0; - const auto page_begin = vaddr & ~(TRACKER_PAGE_SIZE - 1); - const auto page_end = (vaddr + size + TRACKER_PAGE_SIZE - 1) & ~(TRACKER_PAGE_SIZE - 1); - CacheRange affected {.address = page_begin, .size = page_end - page_begin}; - { - FaultSafeCacheLock lock(this, m_mutex); - bool changed = true; - while (changed) { - changed = false; - for (const auto& [address, cached]: m_buffers) { - const CacheRange previous = affected; - if (ResolveOverlap(affected, {address, cached->size}) && - (previous.address != affected.address || previous.size != affected.size)) { - changed = true; - } - } - } - } - { - std::lock_guard lock(m_publication_mutex); - for (const auto& publication: m_pending_backing_publications) { - if (publication.address < affected.address + affected.size && - affected.address < publication.address + publication.size) { - tick = std::max(tick, publication.tick); - } - } - } - if (tick == 0) { - return waited; - } - waited = true; - m_scheduler.Wait(tick); - m_scheduler.WaitPriorityOperations(tick); - } -} - -void BufferCache::RefreshInvalidatedRanges(CommandBuffer& command, CachedBuffer& cached, - uint64_t vaddr, uint64_t size, bool upload) { - const auto invalidated = m_image_invalidated_ranges.Intersections(vaddr, size); - if (upload) { - std::array bytes; - for (const auto& range: invalidated) { - for (uint64_t copied = 0; copied < range.size;) { - const auto chunk = std::min(range.size - copied, bytes.size()); - if (!Libs::LibKernel::Memory::TryReadBacking(range.address + copied, bytes.data(), - chunk)) { - EXIT("BufferCache: failed to refresh an invalidated image alias\n"); - } - Upload(command, *cached.buffer, cached.buffer->Offset(range.address + copied), - bytes.data(), chunk); - copied += chunk; - } - } - } - if (!invalidated.empty()) { - m_image_invalidated_ranges.Subtract(vaddr, size); - } -} - StreamBuffer& BufferCache::GetUtilityBuffer(MemoryUsage usage) noexcept { switch (usage) { case MemoryUsage::Upload: return m_staging_buffer; @@ -364,7 +284,6 @@ void BufferCache::InvalidateMemory(uint64_t vaddr, uint64_t size) { size > TRACKER_ADDRESS_SIZE - vaddr) { EXIT("BufferCache: invalid memory-invalidation range\n"); } - (void)SynchronizeBacking(vaddr, size); if (!HasPageOverlap(vaddr, size)) { return; } @@ -373,7 +292,6 @@ void BufferCache::InvalidateMemory(uint64_t vaddr, uint64_t size) { } void BufferCache::ReadMemory(uint64_t vaddr, uint64_t size) { - (void)SynchronizeBacking(vaddr, size); std::vector copies; { FaultSafeCacheLock lock(this, m_mutex); @@ -413,16 +331,11 @@ void BufferCache::ReadMemory(uint64_t vaddr, uint64_t size) { PublishDownloads(downloads); { FaultSafeCacheLock lock(this, m_mutex); - m_memory_tracker.ForEachDownloadRange( - vaddr, size, - [&](uint64_t address, uint64_t bytes) noexcept { - m_memory_tracker.ValidateGpuDirtyPages(m_gpu_modified_ranges, address, bytes, - "memory invalidation completion"); - }, - [](uint64_t, uint64_t) noexcept {}); for (const auto& range: downloads) { m_gpu_modified_ranges.Subtract(range.address, range.size); } + // The enumeration above covered whole dirty pages and every exact interval on them. + m_memory_tracker.UnmarkRegionAsGpuModified(vaddr, size); } } @@ -430,13 +343,9 @@ void BufferCache::UnmapMemory(uint64_t vaddr, uint64_t size) { if (vaddr == 0 || size == 0 || size > UINT64_MAX - vaddr) { EXIT("BufferCache: invalid unmap range\n"); } - (void)SynchronizeBacking(vaddr, size); - - std::vector copies; - std::vector dirty_ranges; - std::vector> modified_buffers; - std::vector> backing_writes; - std::vector> retired_buffers; + std::vector copies; + std::vector> modified_buffers; + std::vector> retired_buffers; { FaultSafeCacheLock lock(this, m_mutex); for (const auto& [begin, cached]: m_buffers) { @@ -453,12 +362,8 @@ void BufferCache::UnmapMemory(uint64_t vaddr, uint64_t size) { if (dirty.empty()) { EXIT("BufferCache: GPU-modified buffer has no dirty ranges\n"); } - dirty_ranges.insert(dirty_ranges.end(), dirty.begin(), dirty.end()); modified_buffers.emplace_back(begin, cached->size); } - if (!dirty_ranges.empty()) { - backing_writes = m_page_manager.ReserveBackingWrites(dirty_ranges); - } for (const auto& [begin, bytes]: modified_buffers) { auto owner = m_buffers.find(begin); if (owner == m_buffers.end() || owner->second->size != bytes) { @@ -488,24 +393,11 @@ void BufferCache::UnmapMemory(uint64_t vaddr, uint64_t size) { // command stream before removing such backing. m_scheduler.FinishCurrent(); } - backing_writes.clear(); - { FaultSafeCacheLock lock(this, m_mutex); - for (const auto& [begin, bytes]: modified_buffers) { - if (!m_memory_tracker.IsRegionGpuModified(begin, bytes)) { - continue; - } - m_memory_tracker.ForEachDownloadRange( - begin, bytes, - [&](uint64_t address, uint64_t download_size) noexcept { - m_memory_tracker.ValidateGpuDirtyPages(m_gpu_modified_ranges, address, - download_size, "unmap retirement"); - }, - [](uint64_t, uint64_t) noexcept {}); - } for (const auto& [begin, bytes]: modified_buffers) { m_gpu_modified_ranges.Subtract(begin, bytes); + m_memory_tracker.UnmarkRegionAsGpuModified(begin, bytes); } for (const auto& [begin, bytes]: retired_buffers) { m_memory_tracker.MarkRegionAsCpuModified(begin, bytes); @@ -513,7 +405,6 @@ void BufferCache::UnmapMemory(uint64_t vaddr, uint64_t size) { if (!m_gpu_modified_ranges.Intersections(vaddr, size).empty()) { EXIT("BufferCache: unmap retained dirty byte ranges\n"); } - m_image_invalidated_ranges.Subtract(vaddr, size); m_memory_tracker.UntrackMemory(vaddr, size); for (auto it = m_buffers.begin(); it != m_buffers.end();) { if (vaddr < it->first + it->second->size && it->first < vaddr + size) { @@ -607,7 +498,6 @@ BufferBinding BufferCache::ObtainBuffer(CommandBuffer& command, uint64_t vaddr, EXIT("BufferCache: buffer request requires a recording command buffer\n"); } std::lock_guard transaction(m_resource_mutex); - (void)SynchronizeBacking(vaddr, size); if (is_read && !is_written && size <= CACHING_PAGE_SIZE && !m_memory_tracker.IsRegionGpuModified(vaddr, size) && @@ -630,9 +520,7 @@ BufferBinding BufferCache::ObtainBuffer(CommandBuffer& command, uint64_t vaddr, } } - if (is_formatted && is_read && !is_written) { - (void)m_texture_cache.SynchronizeImageToBuffer(vaddr, size); - } else if (is_formatted && is_written) { + if (is_formatted && is_written) { (void)m_texture_cache.InvalidateMemoryFromGPU(vaddr, size, true); } @@ -648,10 +536,12 @@ BufferBinding BufferCache::ObtainBuffer(CommandBuffer& command, uint64_t vaddr, reinterpret_cast(address), bytes); } }); - RefreshInvalidatedRanges(command, cached, vaddr, size, is_read); if (is_written) { m_gpu_modified_ranges.Add(vaddr, size); } + if (is_formatted && is_read && !is_written) { + (void)SynchronizeBufferFromImage(*cached.buffer, vaddr, size); + } return {cached.buffer, cached.buffer->Handle(), cached.buffer->Offset(vaddr)}; } @@ -676,7 +566,6 @@ ImageBufferSource BufferCache::ObtainBufferForImage(uint64_t vaddr, uint64_t siz size > TRACKER_ADDRESS_SIZE - vaddr) { EXIT("BufferCache: invalid image source\n"); } - (void)SynchronizeBacking(vaddr, size); auto find_owner = [&]() { auto owner = m_buffers.upper_bound(vaddr); if (owner == m_buffers.begin()) { @@ -691,13 +580,12 @@ ImageBufferSource BufferCache::ObtainBufferForImage(uint64_t vaddr, uint64_t siz const bool cpu_modified = m_memory_tracker.IsRegionCpuModified(vaddr, size); const bool gpu_modified = m_memory_tracker.IsRegionGpuModified(vaddr, size); const auto dirty = m_gpu_modified_ranges.Intersections(vaddr, size); - const bool invalidated = !m_image_invalidated_ranges.Intersections(vaddr, size).empty(); - const bool requested_gpu_owned = !dirty.empty(); + const bool has_dirty_buffer_source = !dirty.empty(); m_memory_tracker.ValidateGpuDirtyOwnership(m_gpu_modified_ranges, vaddr, size, "image source"); auto owner = find_owner(); - if (requested_gpu_owned && owner == m_buffers.end()) { + if (has_dirty_buffer_source && owner == m_buffers.end()) { CacheRange merged {.address = AlignDown(vaddr), .size = AlignUp(vaddr + size) - AlignDown(vaddr)}; using Iterator = decltype(m_buffers.begin()); @@ -747,42 +635,32 @@ ImageBufferSource BufferCache::ObtainBufferForImage(uint64_t vaddr, uint64_t siz EXIT("BufferCache: merged image source does not contain the requested range\n"); } } - if (owner != m_buffers.end() && !cpu_modified && !invalidated && - (!gpu_modified || requested_gpu_owned)) { - DiscardGpuDirtyBytesLocked(vaddr, size, "image source transfer"); + if (owner != m_buffers.end() && !cpu_modified && + (!gpu_modified || has_dirty_buffer_source)) { owner->second->tick_accessed_last = m_gc_tick; - return {owner->second->buffer.get(), owner->second->buffer->Offset(vaddr), - requested_gpu_owned}; + return {owner->second->buffer.get(), owner->second->buffer->Offset(vaddr)}; } - if (requested_gpu_owned && owner == m_buffers.end()) { + if (has_dirty_buffer_source && owner == m_buffers.end()) { EXIT("BufferCache: GPU-dirty image source could not resolve its native owner\n"); } } - // Direct-memory backing remains readable while PageManager protects the guest mapping. The - // fallback exists for plain host mappings used by standalone renderer tests and is deliberately - // performed outside the cache lock so a page fault cannot recurse into BufferCache. - const auto stage_address = vaddr & ~(TRACKER_PAGE_SIZE - 1); - const auto stage_end = (vaddr + size + TRACKER_PAGE_SIZE - 1) & ~(TRACKER_PAGE_SIZE - 1); - const auto stage_size = stage_end - stage_address; - (void)SynchronizeBacking(stage_address, stage_size); - std::vector bytes(stage_size); - if (!Libs::LibKernel::Memory::TryReadBacking(stage_address, bytes.data(), stage_size)) { + auto [staging, stage_offset] = m_staging_buffer.Map(size, 16); + if (staging == nullptr || !Libs::LibKernel::Memory::TryReadBacking(vaddr, staging, size)) { EXIT("BufferCache: failed to read mapped guest image backing\n"); } + m_staging_buffer.Commit(); FaultSafeCacheLock lock(this, m_mutex); - const auto dirty = m_gpu_modified_ranges.Intersections(vaddr, size); - const bool invalidated = !m_image_invalidated_ranges.Intersections(vaddr, size).empty(); - const bool requested_gpu_owned = !dirty.empty(); - auto owner = find_owner(); - if (requested_gpu_owned && owner == m_buffers.end()) { + const auto dirty = m_gpu_modified_ranges.Intersections(vaddr, size); + const bool has_dirty_buffer_source = !dirty.empty(); + auto owner = find_owner(); + if (has_dirty_buffer_source && owner == m_buffers.end()) { EXIT("BufferCache: GPU-dirty image source lost its native owner\n"); } - const auto stage_offset = m_staging_buffer.Copy(bytes.data(), stage_size, 16); - if (owner == m_buffers.end() || invalidated || - (m_memory_tracker.IsRegionGpuModified(vaddr, size) && !requested_gpu_owned)) { - return {&m_staging_buffer, stage_offset + vaddr - stage_address, false}; + if (owner == m_buffers.end() || + (m_memory_tracker.IsRegionGpuModified(vaddr, size) && !has_dirty_buffer_source)) { + return {&m_staging_buffer, stage_offset}; } auto& cached = *owner->second; @@ -796,42 +674,17 @@ ImageBufferSource BufferCache::ObtainBufferForImage(uint64_t vaddr, uint64_t siz [&]() noexcept { for (const auto& [address, upload_size]: uploads) { cached.buffer->CopyFrom( - m_scheduler.Current(), m_staging_buffer, stage_offset + address - stage_address, + m_scheduler.Current(), m_staging_buffer, stage_offset + address - vaddr, cached.buffer->Offset(address), upload_size, vk::AccessFlagBits::eHostWrite); } }); - DiscardGpuDirtyBytesLocked(vaddr, size, "staged image source transfer"); - return {cached.buffer.get(), cached.buffer->Offset(vaddr), requested_gpu_owned}; -} - -void BufferCache::DiscardGpuDirtyBytesLocked(uint64_t vaddr, uint64_t size, const char* operation) { - m_memory_tracker.ValidateGpuDirtyOwnership(m_gpu_modified_ranges, vaddr, size, operation); - m_gpu_modified_ranges.Subtract(vaddr, size); - const auto page_begin = vaddr & ~(TRACKER_PAGE_SIZE - 1); - const auto page_end = (vaddr + size + TRACKER_PAGE_SIZE - 1) & ~(TRACKER_PAGE_SIZE - 1); - for (auto page = page_begin; page < page_end; page += TRACKER_PAGE_SIZE) { - if (m_gpu_modified_ranges.Intersections(page, TRACKER_PAGE_SIZE).empty() && - m_memory_tracker.IsRegionGpuModified(page, TRACKER_PAGE_SIZE)) { - m_memory_tracker.UnmarkRegionAsGpuModified(page, TRACKER_PAGE_SIZE); - } - } - m_memory_tracker.ValidateGpuDirtyOwnership(m_gpu_modified_ranges, vaddr, size, operation); -} - -void BufferCache::DiscardGpuDirtyBytes(uint64_t vaddr, uint64_t size) { - if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE || - size > TRACKER_ADDRESS_SIZE - vaddr) { - EXIT("BufferCache: invalid dirty-byte discard range\n"); - } - FaultSafeCacheLock lock(this, m_mutex); - DiscardGpuDirtyBytesLocked(vaddr, size, "image output supersession"); + return {cached.buffer.get(), cached.buffer->Offset(vaddr)}; } void BufferCache::WriteHostMemory(uint64_t vaddr, std::span data) { if (vaddr == 0 || data.empty() || data.size() > UINT64_MAX - vaddr) { EXIT("BufferCache: invalid host DMA write\n"); } - (void)SynchronizeBacking(vaddr, data.size()); Libs::LibKernel::Memory::WriteBacking(vaddr, data.data(), data.size()); FaultSafeCacheLock lock(this, m_mutex); @@ -847,45 +700,6 @@ void BufferCache::WriteHostMemory(uint64_t vaddr, std::span data) data.data() + begin - vaddr, range_end - begin); cached->tick_accessed_last = m_gc_tick; } - m_image_invalidated_ranges.Subtract(vaddr, data.size()); -} - -std::pair, uint64_t> BufferCache::ObtainBufferForImageWrite(uint64_t vaddr, - uint64_t size) { - if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE || - size > TRACKER_ADDRESS_SIZE - vaddr) { - EXIT("BufferCache: invalid image destination\n"); - } - const auto stage_address = vaddr & ~(TRACKER_PAGE_SIZE - 1); - const auto stage_end = (vaddr + size + TRACKER_PAGE_SIZE - 1) & ~(TRACKER_PAGE_SIZE - 1); - const auto stage_size = stage_end - stage_address; - (void)SynchronizeBacking(stage_address, stage_size); - std::vector bytes(stage_size); - if (!Libs::LibKernel::Memory::TryReadBacking(stage_address, bytes.data(), stage_size)) { - EXIT("BufferCache: failed to preserve guest bytes around an image mirror\n"); - } - FaultSafeCacheLock lock(this, m_mutex); - auto& cached = GetOrCreateBuffer(m_scheduler.Current(), vaddr, size); - m_memory_tracker.ValidateGpuDirtyOwnership(m_gpu_modified_ranges, vaddr, size, - "image destination"); - if (!m_gpu_modified_ranges.Intersections(vaddr, size).empty()) { - EXIT("BufferCache: image destination aliases GPU-owned buffer bytes\n"); - } - const auto stage_offset = m_staging_buffer.Copy(bytes.data(), stage_size, 16); - std::vector> uploads; - m_memory_tracker.ForEachUploadRange( - vaddr, size, false, - [&](uint64_t address, uint64_t upload_size) noexcept { - uploads.emplace_back(address, upload_size); - }, - [&]() noexcept { - for (const auto& [address, upload_size]: uploads) { - cached.buffer->CopyFrom( - m_scheduler.Current(), m_staging_buffer, stage_offset + address - stage_address, - cached.buffer->Offset(address), upload_size, vk::AccessFlagBits::eHostWrite); - } - }); - return {cached.buffer, cached.buffer->Offset(vaddr)}; } void BufferCache::FillBuffer(uint64_t vaddr, uint64_t size, uint32_t value, bool is_gds) { @@ -945,17 +759,10 @@ void BufferCache::CopyBuffer(uint64_t dst_vaddr, uint64_t src_vaddr, uint64_t si } if (src_memory || dst_memory) { std::lock_guard transaction(m_resource_mutex); - if (src_memory) { - (void)SynchronizeBacking(src_vaddr, size); - } - const auto src_region = + const auto src_region = src_memory ? m_texture_cache.QueryRegion(src_vaddr, size) : TextureCache::RegionInfo {}; const auto dst_region = dst_memory ? m_texture_cache.QueryRegion(dst_vaddr, size) : TextureCache::RegionInfo {}; - if (src_memory && src_region.gpu_image_bytes && - !m_texture_cache.SynchronizeImageToBuffer(src_vaddr, size)) { - EXIT("BufferCache: GPU copy source image could not be synchronized\n"); - } if (src_memory && dst_memory && !HasGpuDirtyBytes(src_vaddr, size) && !HasGpuDirtyBytes(dst_vaddr, size) && !src_region.gpu_image_bytes && !dst_region.gpu_image_bytes) { @@ -977,7 +784,7 @@ void BufferCache::CopyBuffer(uint64_t dst_vaddr, uint64_t src_vaddr, uint64_t si } auto& command = m_scheduler.Current(); - auto src = src_memory ? ObtainBuffer(command, src_vaddr, size, false, true) + auto src = src_memory ? ObtainBuffer(command, src_vaddr, size, false, true, true) : BufferBinding {.buffer = m_gds_buffer.Handle(), .offset = src_vaddr}; auto dst = dst_memory ? ObtainBuffer(command, dst_vaddr, size, true, false, true) : BufferBinding {.buffer = m_gds_buffer.Handle(), .offset = dst_vaddr}; @@ -1030,75 +837,6 @@ bool BufferCache::IsRegionCpuModified(uint64_t vaddr, uint64_t size) { return m_memory_tracker.IsRegionCpuModified(vaddr, size); } -void BufferCache::InvalidateImageAliases(uint64_t vaddr, uint64_t size) { - if (vaddr == 0 || size == 0 || vaddr >= TRACKER_ADDRESS_SIZE || - size > TRACKER_ADDRESS_SIZE - vaddr) { - EXIT("BufferCache: invalid image-alias invalidation\n"); - } - FaultSafeCacheLock lock(this, m_mutex); - const auto end = vaddr + size; - for (const auto& [address, cached]: m_buffers) { - const auto cached_end = address + cached->size; - const auto begin = std::max(vaddr, address); - const auto range_end = std::min(end, cached_end); - if (begin >= range_end) { - continue; - } - const auto bytes = range_end - begin; - if (!m_gpu_modified_ranges.Intersections(begin, bytes).empty()) { - EXIT("BufferCache: image ownership overlaps exact dirty buffer bytes\n"); - } - m_image_invalidated_ranges.Add(begin, bytes); - } -} - -void BufferCache::BeginBackingPublication(uint64_t vaddr, uint64_t size, uint64_t tick) { - if (vaddr == 0 || size == 0 || tick == 0 || vaddr >= TRACKER_ADDRESS_SIZE || - size > TRACKER_ADDRESS_SIZE - vaddr) { - EXIT("BufferCache: invalid pending backing publication\n"); - } - std::lock_guard lock(m_publication_mutex); - m_pending_backing_publications.push_back({vaddr, size, tick}); -} - -void BufferCache::CompleteBackingPublication(uint64_t vaddr, uint64_t size, uint64_t tick) { - std::lock_guard lock(m_publication_mutex); - const auto publication = - std::ranges::find_if(m_pending_backing_publications, [&](const auto& pending) { - return pending.address == vaddr && pending.size == size && pending.tick == tick; - }); - if (publication == m_pending_backing_publications.end()) { - EXIT("BufferCache: completed an unknown backing publication\n"); - } - m_pending_backing_publications.erase(publication); -} - -void BufferCache::PublishImageBuffer(uint64_t vaddr, uint64_t size) { - FaultSafeCacheLock lock(this, m_mutex); - auto owner = m_buffers.end(); - for (auto it = m_buffers.begin(); it != m_buffers.end(); ++it) { - if (!PageOverlaps(vaddr, size, it->second->vaddr, it->second->size)) { - continue; - } - if (owner != m_buffers.end() || !it->second->buffer->IsInBounds(vaddr, size)) { - EXIT("BufferCache: image destination aliases a non-containing cached buffer\n"); - } - owner = it; - } - m_memory_tracker.ValidateGpuDirtyOwnership(m_gpu_modified_ranges, vaddr, size, - "image destination publication"); - if (owner == m_buffers.end() || m_memory_tracker.IsRegionCpuModified(vaddr, size) || - !m_gpu_modified_ranges.Intersections(vaddr, size).empty()) { - EXIT("BufferCache: image destination requires clean buffer ownership\n"); - } - m_memory_tracker.MarkRegionAsGpuModified(vaddr, size); - m_gpu_modified_ranges.Add(vaddr, size); - m_image_invalidated_ranges.Subtract(vaddr, size); - m_memory_tracker.ValidateGpuDirtyOwnership(m_gpu_modified_ranges, vaddr, size, - "published image destination"); - owner->second->tick_accessed_last = m_gc_tick; -} - void BufferCache::RunGarbageCollector() { std::lock_guard transaction(m_resource_mutex); const auto tick = m_gc_tick++; @@ -1174,7 +912,6 @@ void BufferCache::RunGarbageCollector() { if (!m_memory_tracker.IsRegionGpuModified(retire.address, retire.size)) { m_memory_tracker.UntrackMemory(retire.address, retire.size); } - m_image_invalidated_ranges.Subtract(retire.address, retire.size); if (retire.size > m_total_used_memory) { EXIT("BufferCache: allocation accounting underflow\n"); } diff --git a/src/graphics/host_gpu/renderer/cache/bufferCache.h b/src/graphics/host_gpu/renderer/cache/bufferCache.h index 4091859..6e74069 100644 --- a/src/graphics/host_gpu/renderer/cache/bufferCache.h +++ b/src/graphics/host_gpu/renderer/cache/bufferCache.h @@ -10,7 +10,6 @@ #include #include -#include #include #include #include @@ -30,9 +29,8 @@ struct BufferBinding { }; struct ImageBufferSource { - Buffer* buffer = nullptr; - uint64_t offset = 0; - bool gpu_owned = false; + Buffer* buffer = nullptr; + uint64_t offset = 0; }; class BufferCache { @@ -60,9 +58,6 @@ public: uint64_t alignment); [[nodiscard]] std::shared_ptr ObtainNullBuffer(); [[nodiscard]] ImageBufferSource ObtainBufferForImage(uint64_t vaddr, uint64_t size); - [[nodiscard]] std::pair, uint64_t> - ObtainBufferForImageWrite(uint64_t vaddr, uint64_t size); - void DiscardGpuDirtyBytes(uint64_t vaddr, uint64_t size); void FillBuffer(uint64_t vaddr, uint64_t size, uint32_t value, bool is_gds = false); void CopyBuffer(uint64_t dst_vaddr, uint64_t src_vaddr, uint64_t size, bool dst_gds = false, bool src_gds = false); @@ -70,11 +65,6 @@ public: [[nodiscard]] bool HasGpuDirtyBytes(uint64_t vaddr, uint64_t size); [[nodiscard]] bool IsRegionCpuModified(uint64_t vaddr, uint64_t size); [[nodiscard]] bool IsRegionGpuModified(uint64_t vaddr, uint64_t size); - void InvalidateImageAliases(uint64_t vaddr, uint64_t size); - void BeginBackingPublication(uint64_t vaddr, uint64_t size, uint64_t tick); - void CompleteBackingPublication(uint64_t vaddr, uint64_t size, uint64_t tick); - [[nodiscard]] bool SynchronizeBacking(uint64_t vaddr, uint64_t size); - void PublishImageBuffer(uint64_t vaddr, uint64_t size); void RunGarbageCollector(); private: @@ -88,7 +78,6 @@ private: struct DownloadCopy; struct DownloadRange; struct RetiredBuffer; - struct PendingBackingPublication; static constexpr uint64_t DOWNLOAD_ALIGNMENT = 64; [[nodiscard]] static uint64_t AlignDown(uint64_t value) noexcept; [[nodiscard]] static uint64_t AlignUp(uint64_t value); @@ -103,12 +92,10 @@ private: const void* source, uint64_t size); [[nodiscard]] CachedBuffer& GetOrCreateBuffer(CommandBuffer& command, uint64_t vaddr, uint64_t size); + [[nodiscard]] bool SynchronizeBufferFromImage(Buffer& buffer, uint64_t vaddr, uint64_t size); [[nodiscard]] std::vector RecordDownloads(std::span copies); void PublishDownloads(std::span downloads); void QueueGarbageDownload(std::span copies, RetiredBuffer retire); - void RefreshInvalidatedRanges(CommandBuffer& command, CachedBuffer& cached, uint64_t vaddr, - uint64_t size, bool upload); - void DiscardGpuDirtyBytesLocked(uint64_t vaddr, uint64_t size, const char* operation); void WriteHostMemory(uint64_t vaddr, std::span data); GraphicContext& m_graphics; @@ -118,15 +105,11 @@ private: std::shared_ptr m_null_buffer; std::map> m_buffers; RangeSet m_gpu_modified_ranges; - RangeSet m_image_invalidated_ranges; - std::mutex m_publication_mutex; - std::vector m_pending_backing_publications; MemoryTracker m_memory_tracker; StreamBuffer m_staging_buffer; StreamBuffer m_stream_buffer; StreamBuffer m_download_buffer; StreamBuffer m_device_buffer; - PageManager& m_page_manager; TextureCache& m_texture_cache; ResourceMutex& m_resource_mutex; uint64_t m_total_used_memory = 0; diff --git a/src/graphics/host_gpu/renderer/cache/textureCache.cpp b/src/graphics/host_gpu/renderer/cache/textureCache.cpp index 9414b38..8ba6e2a 100644 --- a/src/graphics/host_gpu/renderer/cache/textureCache.cpp +++ b/src/graphics/host_gpu/renderer/cache/textureCache.cpp @@ -323,7 +323,7 @@ void TextureCache::TrackImage(ImageId id) { if (!image.IsTracked()) { image.track_addr = image_begin; image.track_addr_end = image_end; - m_page_manager.UpdatePageWatchers(true, image_begin, image.info.data.size); + m_page_manager.UpdatePageWatchers(image_begin, image.info.data.size); return; } if (image_begin < image.track_addr) { @@ -348,7 +348,7 @@ void TextureCache::TrackImageHead(ImageId id) { } const auto size = image.track_addr - image_begin; image.track_addr = image_begin; - m_page_manager.UpdatePageWatchers(true, image_begin, size); + m_page_manager.UpdatePageWatchers(image_begin, size); } void TextureCache::TrackImageTail(ImageId id) { @@ -366,7 +366,7 @@ void TextureCache::TrackImageTail(ImageId id) { const auto address = image.track_addr_end; const auto size = image_end - address; image.track_addr_end = image_end; - m_page_manager.UpdatePageWatchers(true, address, size); + m_page_manager.UpdatePageWatchers(address, size); } void TextureCache::UntrackImage(ImageId id) { @@ -379,7 +379,7 @@ void TextureCache::UntrackImage(ImageId id) { image.track_addr = 0; image.track_addr_end = 0; if (size != 0) { - m_page_manager.UpdatePageWatchers(false, address, size); + m_page_manager.UpdatePageWatchers(address, size); } } @@ -400,7 +400,7 @@ void TextureCache::UntrackImageHead(ImageId id) { UntrackImage(id); } if (size != 0) { - m_page_manager.UpdatePageWatchers(false, begin, size); + m_page_manager.UpdatePageWatchers(begin, size); } } @@ -421,7 +421,7 @@ void TextureCache::UntrackImageTail(ImageId id) { UntrackImage(id); } if (size != 0) { - m_page_manager.UpdatePageWatchers(false, address, size); + m_page_manager.UpdatePageWatchers(address, size); } } @@ -1038,25 +1038,20 @@ void TextureCache::InitializeImage(ImageId id, const ImageDesc& desc) { if (image.info.samples > 1) { return; } - bool data_gpu_owned = false; - bool data_imported = false; - const bool upload = image.IsBufferModified() || image.IsCpuDirty(); + bool data_imported = false; + const bool upload = image.IsBufferModified() || image.IsCpuDirty(); if (upload) { const auto source = m_buffer_cache.ObtainBufferForImage(image.info.data.address, image.info.data.size); if (source.buffer == nullptr) { EXIT("TextureCache: failed to obtain image upload source\n"); } - data_gpu_owned |= source.gpu_owned; data_imported = true; UploadImage(image, desc, *source.buffer, source.offset); } if (data_imported) { image.ClearBufferModified(); } - if (data_gpu_owned) { - image.MarkGpuModified(); - } if (image.IsCpuDirty()) { image.RefreshComplete(); } @@ -1131,8 +1126,7 @@ ImageId TextureCache::FindImage(ImageDesc& desc, bool exact_format) { } ImageId result {}; - bool replacement_buffer = false; - bool inserted_new = false; + bool inserted_new = false; { std::lock_guard transaction(m_resource_mutex); CacheLock lock(*this, m_lock); @@ -1174,23 +1168,15 @@ ImageId TextureCache::FindImage(ImageDesc& desc, bool exact_format) { if (exact_format && resolved.info.pixel_format != desc.info.pixel_format) { result = {}; } else if (resolved.info.resources < desc.info.resources) { - ImageDesc refresh { - .info = resolved.info, .view_info = {}, .type = UploadBinding(resolved)}; - RefreshImage(result, refresh); - if (resolved.IsGpuModified() && !SynchronizeImageToBuffer(result)) { - EXIT("TextureCache: cannot preserve an unsupported replacement image\n"); - } - replacement_buffer = resolved.IsBufferModified(); - DeleteImage(result); - result = {}; + result = ExpandImage(desc.info, result); } } if (!result) { result = InsertImage(desc.info); inserted_new = true; auto& inserted = ResolveImage(result); - if (replacement_buffer || m_buffer_cache.HasGpuDirtyBytes(inserted.info.data.address, - inserted.info.data.size)) { + if (m_buffer_cache.HasGpuDirtyBytes(inserted.info.data.address, + inserted.info.data.size)) { inserted.MarkBufferModified(); } } @@ -1360,11 +1346,6 @@ void TextureCache::CommitGpuWrite(Image& image) { if (image.depth_id || image.backing.image == nullptr) { EXIT("TextureCache: stencil association cannot own image contents\n"); } - const auto range = image.info.data; - if (m_buffer_cache.HasGpuDirtyBytes(range.address, range.size)) { - m_buffer_cache.DiscardGpuDirtyBytes(range.address, range.size); - } - m_buffer_cache.InvalidateImageAliases(range.address, range.size); image.ClearBufferModified(); if (image.IsCpuDirty()) { image.RefreshComplete(); @@ -1429,9 +1410,6 @@ bool TextureCache::ClearImageFromBuffer(CommandBuffer& command, uint64_t address return false; } } - if (m_buffer_cache.HasGpuDirtyBytes(address, size)) { - m_buffer_cache.DiscardGpuDirtyBytes(address, size); - } if (image.IsBufferModified() || image.IsCpuDirty()) { ImageDesc refresh {.info = image.info, .view_info = {}, .type = UploadBinding(image)}; InitializeImage(selected, refresh); @@ -1551,11 +1529,14 @@ void TextureCache::DownloadDepth(Image& image, Buffer& destination, uint64_t des } void TextureCache::DownloadImageData(Image& image, Buffer& destination, uint64_t destination_offset, - DownloadPlan plan) { + uint64_t destination_size, DownloadPlan plan) { if (!plan.valid) { EXIT("TextureCache: invalid image download plan\n"); } if (plan.depth) { + if (destination_size != image.info.data.size) { + EXIT("TextureCache: partial depth image download is unsupported\n"); + } DownloadDepth(image, destination, destination_offset); return; } @@ -1565,22 +1546,118 @@ void TextureCache::DownloadImageData(Image& image, Buffer& destination, uint64_t : TileManager::ColorTransform::None; if (!color.tiled) { if (transform == TileManager::ColorTransform::SwapBgra16) { - auto linear = m_tiler->GetScratchBuffer(image.info.data.size); + auto linear = m_tiler->GetScratchBuffer(destination_size); image.Download(color.regions, linear.buffer, 0, linear.size); m_tiler->SwapBgra16(linear, - {destination.Handle(), destination_offset, image.info.data.size}); + {destination.Handle(), destination_offset, destination_size}); return; } for (auto& copy: color.regions) { copy.bufferOffset += destination_offset; } - image.Download(color.regions, destination.Handle(), destination_offset, - image.info.data.size); + image.Download(color.regions, destination.Handle(), destination_offset, destination_size); return; } m_tiler->TileImage(image, color.regions, destination.Handle(), destination_offset, - image.info.data.size, image.info.data.size, color.tiles, transform); + destination_size, destination_size, color.tiles, transform); +} + +bool BufferCache::SynchronizeBufferFromImage(Buffer& buffer, uint64_t vaddr, uint64_t size) { + CacheLock lock(m_texture_cache, m_texture_cache.m_lock); + std::vector matches; + for (const auto id: m_texture_cache.FindImagesInRegion(vaddr, size, false)) { + auto owner = m_texture_cache.ResolveOwner(id); + if (owner == nullptr || owner->info.data.address != vaddr) { + continue; + } + if (owner->depth_id) { + owner = m_texture_cache.ResolveOwner(owner->depth_id); + } + if (owner != nullptr && owner->SafeToDownload()) { + matches.push_back(id); + } + } + + ImageId selected {}; + if (matches.size() == 1) { + selected = matches.front(); + } else { + for (const auto id: matches) { + const auto& image = m_texture_cache.ResolveImage(id); + if (image.info.data.size == size) { + selected = id; + break; + } + } + } + if (!selected) { + return false; + } + if (const auto owner = m_texture_cache.ResolveOwner(selected); + owner != nullptr && owner->depth_id) { + selected = owner->depth_id; + } + + auto& image = m_texture_cache.ResolveImage(selected); + if (!buffer.IsInBounds(image.info.data.address, 1)) { + return false; + } + const auto buf_offset = buffer.Offset(image.info.data.address); + const auto available = buffer.Size() - buf_offset; + uint32_t levels = 0; + uint64_t copy_size = 0; + if (image.info.IsVolume()) { + // Volume mips contain strided block slices, so a mip's linear span cannot prove that + // every retained slice fits. Keep volume synchronization whole-image only. + if (!buffer.IsInBounds(image.info.data.address, image.info.data.size)) { + return false; + } + levels = image.info.resources.levels; + copy_size = image.info.data.size; + } else { + for (; levels < image.info.resources.levels; ++levels) { + const auto& mip = image.info.mip_layout[levels]; + if (mip.size == 0 || mip.offset > available || mip.size > available - mip.offset) { + break; + } + copy_size = std::max(copy_size, mip.offset + mip.size); + } + } + if (copy_size == 0) { + return false; + } + auto plan = m_texture_cache.BuildDownload(image); + if (!plan.valid) { + return false; + } + if (plan.depth && copy_size != image.info.data.size) { + return false; + } + if (!plan.depth && levels < image.info.resources.levels) { + auto& color = plan.color; + std::erase_if(color.regions, [levels](const vk::BufferImageCopy& region) { + return region.imageSubresource.mipLevel >= levels; + }); + if (color.regions.empty()) { + return false; + } + if (color.tiled) { + const auto binding = m_texture_cache.UploadBinding(image); + const auto format = + binding == TextureCache::BindingType::RenderTarget + ? ImageOps::RenderTargetTransferFormat(image.info.bytes_per_block) + : image.info.guest_format; + color.tiles.clear(); + if (!TextureBuildGpuTileInfos(copy_size, color.regions, color.layout, format, + image.info.TransferLayers(), levels, color.tiles)) { + return false; + } + } + } + m_texture_cache.DownloadImageData(image, buffer, buf_offset, copy_size, std::move(plan)); + m_texture_cache.RetainImage(m_scheduler.Current(), selected); + return true; } std::pair TextureCache::MapDownload(uint64_t size, uint64_t alignment) { @@ -1612,12 +1689,9 @@ void TextureCache::QueueDownload(GuestRange range, StreamBuffer& download, uint8 m_scheduler.Current().Handle().pipelineBarrier(vk::PipelineStageFlagBits::eAllCommands, vk::PipelineStageFlagBits::eHost, {}, 0, nullptr, 1, &barrier, 0, nullptr); - const auto tick = m_scheduler.CurrentTick(); - m_buffer_cache.BeginBackingPublication(range.address, range.size, tick); - m_scheduler.DeferPriorityOperation([this, &download, range, mapped, offset, tick] { + m_scheduler.DeferPriorityOperation([&download, range, mapped, offset] { download.Invalidate(offset, range.size); LibKernel::Memory::WriteBacking(range.address, mapped, range.size); - m_buffer_cache.CompleteBackingPublication(range.address, range.size, tick); }); } @@ -1638,7 +1712,7 @@ bool TextureCache::TryDownloadImage(ImageId id) { } download.Flush(offset, range.size); - DownloadImageData(image, download, offset, std::move(plan)); + DownloadImageData(image, download, offset, range.size, std::move(plan)); QueueDownload(range, download, mapped, offset); return true; @@ -1652,62 +1726,6 @@ void TextureCache::DownloadImage(ImageId id) { m_scheduler.DrainPriorityOperations(); } -bool TextureCache::SynchronizeImageToBuffer(ImageId id) { - auto& image = ResolveImage(id); - if (image.depth_id) { - return true; - } - auto plan = BuildDownload(image); - if (!plan.valid) { - return false; - } - const auto range = image.info.data; - if (image.IsCpuDirty()) { - RefreshImage(id, - ImageDesc {.info = image.info, .view_info = {}, .type = UploadBinding(image)}); - } - if (!image.IsGpuModified()) { - return true; - } - if (image.IsDefinitelyCpuDirty() || image.IsBufferModified()) { - EXIT("TextureCache: image mirror source is not native-current\n"); - } - auto [destination, offset] = - m_buffer_cache.ObtainBufferForImageWrite(range.address, range.size); - if (destination == nullptr) { - EXIT("TextureCache: failed to allocate image mirror\n"); - } - DownloadImageData(image, *destination, offset, std::move(plan)); - m_scheduler.Current().RetainResourceUntilFence(destination); - m_buffer_cache.PublishImageBuffer(range.address, range.size); - image.MarkBufferModified(); - RetainImage(m_scheduler.Current(), id); - ClearGpuModified(id); - return true; -} - -bool TextureCache::SynchronizeImageToBuffer(uint64_t address, uint64_t size) { - if (!GuestRange {address, size}.Valid()) { - return false; - } - CacheLock lock(*this, m_lock); - ImageId selected {}; - for (const auto id: FindImagesInRegion(address, size, true)) { - auto owner = ResolveOwner(id); - if (owner == nullptr || !owner->GpuOverlaps(address, size)) { - continue; - } - if (selected) { - EXIT("TextureCache: ambiguous image-to-buffer synchronization\n"); - } - selected = id; - } - if (!selected) { - return false; - } - return SynchronizeImageToBuffer(selected); -} - bool TextureCache::InvalidateMemoryFromGPU(uint64_t address, uint64_t size, bool formatted_buffer_write) { if (!GuestRange {address, size}.Valid()) { diff --git a/src/graphics/host_gpu/renderer/cache/textureCache.h b/src/graphics/host_gpu/renderer/cache/textureCache.h index e9964a7..622da33 100644 --- a/src/graphics/host_gpu/renderer/cache/textureCache.h +++ b/src/graphics/host_gpu/renderer/cache/textureCache.h @@ -66,7 +66,6 @@ public: [[nodiscard]] bool ClearImageFromBuffer(CommandBuffer& command, uint64_t address, uint64_t size, uint32_t packed_clear); void InvalidateMemory(uint64_t address, uint64_t size); - [[nodiscard]] bool SynchronizeImageToBuffer(uint64_t address, uint64_t size); [[nodiscard]] bool InvalidateMemoryFromGPU(uint64_t address, uint64_t size, bool formatted_buffer_write = false); [[nodiscard]] RegionInfo QueryRegion(uint64_t address, uint64_t size); @@ -140,7 +139,7 @@ private: [[nodiscard]] DownloadPlan BuildDownload(const Image& image) const; void UploadImage(Image& image, const ImageDesc& desc, Buffer& source, uint64_t source_offset); void DownloadImageData(Image& image, Buffer& destination, uint64_t destination_offset, - DownloadPlan plan); + uint64_t destination_size, DownloadPlan plan); void DownloadDepth(Image& image, Buffer& destination, uint64_t destination_offset); void CommitGpuWrite(Image& image); void PrepareImageCopy(Image& image); @@ -155,7 +154,6 @@ private: void InvalidateCpuAliases(uint64_t address, uint64_t size); void ClearGpuModified(ImageId id); - [[nodiscard]] bool SynchronizeImageToBuffer(ImageId id); void DownloadImage(ImageId id); [[nodiscard]] bool TryDownloadImage(ImageId id); [[nodiscard]] std::pair MapDownload(uint64_t size, uint64_t alignment); @@ -184,6 +182,7 @@ private: bool m_readback_linear_images = false; friend struct TextureCacheTestAccess; + friend class BufferCache; friend class RenderExecutor; }; diff --git a/tests/BitArrayTests.cpp b/tests/BitArrayTests.cpp new file mode 100644 index 0000000..8dc53b4 --- /dev/null +++ b/tests/BitArrayTests.cpp @@ -0,0 +1,310 @@ +#include "common/bitArray.h" + +#include +#include +#include +#include +#include + +namespace { + +using Bits = Common::BitArray<128>; + +static_assert(sizeof(Common::BitArray<1024>) == 128); + +void Check(bool value, const char *message) { + if (!value) { + std::fprintf(stderr, "BitArrayTests: failed: %s\n", message); + std::abort(); + } +} + +void TestPointAndRangeOperations() { + Bits bits; + Check(bits.None() && !bits.Any(), "default state is not empty"); + + for (const auto index : {size_t{0}, size_t{63}, size_t{64}, size_t{127}}) { + bits.Set(index); + Check(bits.Get(index), "Set did not set a boundary bit"); + bits.Unset(index); + Check(!bits.Get(index), "Unset did not clear a boundary bit"); + } + + bits.SetRange(60, 68); + for (size_t index = 0; index < 128; index++) { + Check(bits.Get(index) == (index >= 60 && index < 68), + "cross-word SetRange changed the wrong bits"); + } + bits.Fill(); + bits.UnsetRange(60, 68); + for (size_t index = 0; index < 128; index++) { + Check(bits.Get(index) == !(index >= 60 && index < 68), + "cross-word UnsetRange changed the wrong bits"); + } + bits.Clear(); + bits.SetRange(0, 128); + Check(!bits.None(), "full SetRange left the array empty"); + bits.UnsetRange(0, 128); + Check(bits.None(), "full UnsetRange left set bits"); + + bits.Set(7); + bits.SetRange(9, 9); + bits.SetRange(0, 129); + bits.UnsetRange(9, 9); + bits.UnsetRange(0, 129); + Check(bits.Get(7), "invalid or empty range modified the array"); +} + +void TestMaskedConstructionAndBitwiseOperations() { + Bits source; + source.Fill(); + const Bits masked(source, 31, 97); + for (size_t index = 0; index < 128; index++) { + Check(masked.Get(index) == (index >= 31 && index < 97), + "masked constructor retained a bit outside its range"); + } + Check(Bits(source, 12, 12).None(), "empty masked constructor produced bits"); + Check(Bits(source, 0, 129).None(), + "invalid masked constructor produced bits"); + + Bits left; + left.SetRange(0, 80); + Bits right; + right.SetRange(40, 120); + const auto exclusive = left ^ right; + for (size_t index = 0; index < 128; index++) { + const bool expected = (index < 80) != (index >= 40 && index < 120); + Check(exclusive.Get(index) == expected, "XOR produced the wrong bit"); + } + const auto inverted = ~left; + for (size_t index = 0; index < 128; index++) { + Check(inverted.Get(index) == (index >= 80), "NOT produced the wrong bit"); + } +} + +void TestRangeDiscoveryAndIteration() { + Bits bits; + Check(bits.FirstRange() == Bits::Range{128, 128}, + "empty FirstRange is wrong"); + Check(bits.LastRange() == Bits::Range{0, 0}, "empty LastRange is wrong"); + + bits.SetRange(3, 8); + bits.SetRange(63, 70); + bits.Set(127); + Check(bits.FirstRange() == Bits::Range{3, 8}, "FirstRange is wrong"); + Check(bits.FirstRangeFrom(5) == Bits::Range{5, 8}, + "FirstRangeFrom inside a run is wrong"); + Check(bits.FirstRangeFrom(8) == Bits::Range{63, 70}, + "FirstRangeFrom gap is wrong"); + Check(bits.LastRange() == Bits::Range{127, 128}, "LastRange is wrong"); + Check(bits.LastRangeFrom(69) == Bits::Range{63, 69}, + "LastRangeFrom inside a run is wrong"); + Check(bits.LastRangeFrom(63) == Bits::Range{3, 8}, + "LastRangeFrom gap is wrong"); + + constexpr std::array expected{Bits::Range{3, 8}, Bits::Range{63, 70}, + Bits::Range{127, 128}}; + size_t range_index = 0; + for (const auto range : bits) { + Check(range_index < expected.size() && range == expected[range_index], + "range iterator produced the wrong run"); + range_index++; + } + Check(range_index == expected.size(), "range iterator omitted a run"); +} + +void TestRandomizedDifferential() { + Bits bits; + std::array reference{}; + uint64_t random = 0x53a9'7f11'ced4'29b5ull; + const auto next_random = [&random] { + random ^= random << 13; + random ^= random >> 7; + random ^= random << 17; + return random; + }; + + for (size_t operation = 0; operation < 10000; operation++) { + const auto first = static_cast(next_random() % 128); + const auto last = + first + 1 + static_cast(next_random() % (128 - first)); + if ((next_random() & 1) != 0) { + bits.SetRange(first, last); + for (auto index = first; index < last; index++) { + reference[index] = true; + } + } else { + bits.UnsetRange(first, last); + for (auto index = first; index < last; index++) { + reference[index] = false; + } + } + + bool any = false; + for (size_t index = 0; index < reference.size(); index++) { + Check(bits.Get(index) == reference[index], + "randomized bit state diverged"); + any |= reference[index]; + } + Check(bits.Any() == any && bits.None() == !any, + "randomized Any/None diverged"); + + const auto range_start = static_cast(next_random() % 129); + auto expected_first_begin = range_start; + while (expected_first_begin < reference.size() && + !reference[expected_first_begin]) { + expected_first_begin++; + } + if (expected_first_begin == reference.size()) { + Check(bits.FirstRangeFrom(range_start) == Bits::Range{128, 128}, + "randomized FirstRangeFrom empty suffix diverged"); + } else { + auto expected_first_end = expected_first_begin; + while (expected_first_end < reference.size() && + reference[expected_first_end]) { + expected_first_end++; + } + Check(bits.FirstRangeFrom(range_start) == + Bits::Range{expected_first_begin, expected_first_end}, + "randomized FirstRangeFrom diverged"); + } + + const auto range_end = static_cast(next_random() % 129); + auto expected_last_end = range_end; + while (expected_last_end != 0 && !reference[expected_last_end - 1]) { + expected_last_end--; + } + if (expected_last_end == 0) { + Check(bits.LastRangeFrom(range_end) == Bits::Range{0, 0}, + "randomized LastRangeFrom empty prefix diverged"); + } else { + auto expected_last_begin = expected_last_end; + while (expected_last_begin != 0 && reference[expected_last_begin - 1]) { + expected_last_begin--; + } + Check(bits.LastRangeFrom(range_end) == + Bits::Range{expected_last_begin, expected_last_end}, + "randomized LastRangeFrom diverged"); + } + + const auto masked_start = static_cast(next_random() % 129); + const auto masked_end = + masked_start + + static_cast(next_random() % (129 - masked_start)); + const Bits masked(bits, masked_start, masked_end); + for (size_t index = 0; index < reference.size(); index++) { + Check(masked.Get(index) == (index >= masked_start && index < masked_end && + reference[index]), + "randomized masked constructor diverged"); + } + + size_t first_begin = 0; + while (first_begin < reference.size() && !reference[first_begin]) { + first_begin++; + } + if (first_begin == reference.size()) { + Check(bits.FirstRange() == Bits::Range{128, 128}, + "randomized empty FirstRange diverged"); + Check(bits.LastRange() == Bits::Range{0, 0}, + "randomized empty LastRange diverged"); + } else { + auto first_end = first_begin; + while (first_end < reference.size() && reference[first_end]) { + first_end++; + } + Check(bits.FirstRange() == Bits::Range{first_begin, first_end}, + "randomized FirstRange diverged"); + + auto last_end = reference.size(); + while (!reference[last_end - 1]) { + last_end--; + } + auto last_begin = last_end; + while (last_begin != 0 && reference[last_begin - 1]) { + last_begin--; + } + Check(bits.LastRange() == Bits::Range{last_begin, last_end}, + "randomized LastRange diverged"); + } + + size_t expected_begin = 0; + for (const auto [begin, end] : bits) { + while (expected_begin < reference.size() && !reference[expected_begin]) { + expected_begin++; + } + Check(begin == expected_begin, "randomized iterator run start diverged"); + while (expected_begin < reference.size() && reference[expected_begin]) { + expected_begin++; + } + Check(end == expected_begin, "randomized iterator run end diverged"); + } + while (expected_begin < reference.size() && !reference[expected_begin]) { + expected_begin++; + } + Check(expected_begin == reference.size(), + "randomized iterator omitted a run"); + } +} + +void TestTrackerSizedRandomizedDifferential() { + using TrackerBits = Common::BitArray<1024>; + TrackerBits bits; + std::array reference{}; + uint64_t random = 0x9e37'79b9'7f4a'7c15ull; + const auto next_random = [&random] { + random ^= random << 13; + random ^= random >> 7; + random ^= random << 17; + return random; + }; + + for (size_t operation = 0; operation < 4096; operation++) { + const auto first = static_cast(next_random() % reference.size()); + const auto last = + first + 1 + + static_cast(next_random() % (reference.size() - first)); + const bool set = (next_random() & 1) != 0; + if (set) { + bits.SetRange(first, last); + } else { + bits.UnsetRange(first, last); + } + for (auto index = first; index < last; index++) { + reference[index] = set; + } + + for (size_t index = 0; index < reference.size(); index++) { + Check(bits.Get(index) == reference[index], + "tracker-sized randomized bit state diverged"); + } + + size_t expected = 0; + for (const auto [begin, end] : bits) { + while (expected < reference.size() && !reference[expected]) { + expected++; + } + Check(begin == expected, "tracker-sized randomized range start diverged"); + while (expected < reference.size() && reference[expected]) { + expected++; + } + Check(end == expected, "tracker-sized randomized range end diverged"); + } + while (expected < reference.size() && !reference[expected]) { + expected++; + } + Check(expected == reference.size(), + "tracker-sized randomized iterator omitted a run"); + } +} + +} // namespace + +int main() { + TestPointAndRangeOperations(); + TestMaskedConstructionAndBitwiseOperations(); + TestRangeDiscoveryAndIteration(); + TestRandomizedDifferential(); + TestTrackerSizedRandomizedDifferential(); + std::puts("BitArrayTests: all cases passed"); + return 0; +} diff --git a/tests/MemoryTrackerTests.cpp b/tests/MemoryTrackerTests.cpp index 5177a66..0ce6645 100644 --- a/tests/MemoryTrackerTests.cpp +++ b/tests/MemoryTrackerTests.cpp @@ -27,7 +27,6 @@ namespace { using Libs::Graphics::MemoryTracker; using Libs::Graphics::PageManager; -using Libs::Graphics::PageWatchMode; using Libs::Graphics::RangeSet; void Check(bool value, const char *text) { @@ -127,6 +126,21 @@ bool IsWritable(const void *address) { return Protection(address) == PAGE_READWRITE; } +uint64_t g_protection_calls = 0; + +struct ProtectionCall { + uint64_t address; + uint64_t size; + Common::VirtualMemory::Mode mode; +}; + +std::vector g_protection_log; + +void ResetProtectionLog() { + g_protection_calls = 0; + g_protection_log.clear(); +} + bool ProtectAddressSpace(uint64_t vaddr, uint64_t size, Common::VirtualMemory::Mode mode) { uint32_t protection = PAGE_NOACCESS; @@ -136,14 +150,14 @@ bool ProtectAddressSpace(uint64_t vaddr, uint64_t size, protection = PAGE_READWRITE; } DWORD old_protection = 0; + g_protection_calls++; + g_protection_log.push_back({vaddr, size, mode}); return VirtualProtect(reinterpret_cast(vaddr), size, protection, &old_protection) != 0; } struct TrackerHarness { - explicit TrackerHarness( - PageWatchMode gpu_watch_mode = PageWatchMode::ReadWrite) - : tracker(page_manager, gpu_watch_mode) {} + TrackerHarness() : tracker(page_manager) {} PageManager page_manager; MemoryTracker tracker; @@ -287,6 +301,124 @@ void TestGpuDirtyBits() { Release(page_manager, memory, page_size * 2); } +void TestExactDirtyIntervalsSharingTrackerPage() { + TrackerHarness harness; + auto &tracker = harness.tracker; + auto &page_manager = harness.page_manager; + const auto page_size = page_manager.GetPageSize(); + auto *memory = Allocate(page_manager, 1); + const auto address = reinterpret_cast(memory); + + tracker.ForEachUploadRange( + address, page_size, false, [](uint64_t, uint64_t) noexcept {}, + []() noexcept {}); + RangeSet exact_dirty; + exact_dirty.Add(address + 64, 16); + exact_dirty.Add(address + 192, 32); + + ResetProtectionLog(); + tracker.MarkRegionAsGpuModified(address + 64, 16); + tracker.MarkRegionAsGpuModified(address + 192, 32); + Check(g_protection_calls == 1 && + tracker.IsRegionGpuModified(address, page_size) && + Protection(memory) == PAGE_NOACCESS, + "disjoint byte dirtiness duplicated the page watcher"); + + exact_dirty.Subtract(address + 64, 16); + if (exact_dirty.Intersections(address, page_size).empty()) { + tracker.UnmarkRegionAsGpuModified(address, page_size); + } + Check(g_protection_calls == 1 && + tracker.IsRegionGpuModified(address, page_size) && + Protection(memory) == PAGE_NOACCESS, + "draining one exact interval prematurely released its shared page"); + + exact_dirty.Subtract(address + 192, 32); + if (exact_dirty.Intersections(address, page_size).empty()) { + tracker.UnmarkRegionAsGpuModified(address, page_size); + } + Check(g_protection_calls == 2 && + !tracker.IsRegionGpuModified(address, page_size) && + Protection(memory) == PAGE_READONLY, + "draining the final exact interval did not release its tracker page"); + + tracker.UntrackMemory(address, page_size); + Release(page_manager, memory, page_size); +} + +void TestGpuDownloadProtectionMirrors() { + TrackerHarness harness; + auto &tracker = harness.tracker; + auto &page_manager = harness.page_manager; + const auto page_size = page_manager.GetPageSize(); + auto *memory = Allocate(page_manager, 4); + const auto address = reinterpret_cast(memory); + + tracker.ForEachUploadRange( + address, page_size * 4, false, [](uint64_t, uint64_t) noexcept {}, + []() noexcept {}); + tracker.MarkRegionAsGpuModified(address + 16, 32); + tracker.MarkRegionAsGpuModified(address + page_size * 2 + 16, 32); + + std::vector visited; + ResetProtectionLog(); + tracker.ForEachDownloadRange( + address, page_size * 3, + [&](uint64_t range_address, uint64_t range_size) noexcept { + visited.push_back({range_address, range_size}); + }); + Check(visited.size() == 2 && visited[0].address == address && + visited[0].size == page_size && + visited[1].address == address + page_size * 2 && + visited[1].size == page_size && g_protection_calls == 0 && + tracker.IsRegionGpuModified(address, page_size * 3), + "non-clearing download changed protection or lost sparse ranges"); + + visited.clear(); + tracker.ForEachDownloadRange( + address + 16, 32, + [&](uint64_t range_address, uint64_t range_size) noexcept { + visited.push_back({range_address, range_size}); + }); + Check(visited.size() == 1 && visited[0].address == address && + visited[0].size == page_size && g_protection_log.size() == 1 && + g_protection_log[0].address == address && + g_protection_log[0].size == page_size && + g_protection_log[0].mode == Common::VirtualMemory::Mode::Read && + !tracker.IsRegionGpuModified(address, page_size) && + tracker.IsRegionGpuModified(address + page_size * 2, page_size) && + Protection(memory) == PAGE_READONLY && + Protection(memory + page_size * 2) == PAGE_NOACCESS, + "partial download did not preserve the CPU/GPU protection mirrors"); + + visited.clear(); + ResetProtectionLog(); + tracker.ForEachDownloadRange( + address + 16, 32, + [&](uint64_t range_address, uint64_t range_size) noexcept { + visited.push_back({range_address, range_size}); + }); + Check(visited.empty() && g_protection_calls == 0 && + tracker.IsRegionGpuModified(address + page_size * 2, page_size), + "idempotent partial download disturbed another GPU-owned page"); + + tracker.UnmarkRegionAsGpuModified(address, page_size * 3); + Check(!tracker.IsRegionGpuModified(address, page_size * 3) && + Protection(memory + page_size * 2) == PAGE_READONLY, + "broad final unmark did not restore write-only tracking"); + ResetProtectionLog(); + tracker.MarkRegionAsCpuModified(address + 16, 32); + Check( + g_protection_log.size() == 1 && g_protection_log[0].address == address && + g_protection_log[0].size == page_size && + g_protection_log[0].mode == Common::VirtualMemory::Mode::ReadWrite && + IsWritable(memory) && !IsWritable(memory + page_size), + "CPU-dirty transition did not release only its write watcher"); + + tracker.UntrackMemory(address, page_size * 4); + Release(page_manager, memory, page_size * 4); +} + void TestCrossRegionUpload() { constexpr uintptr_t base = 0x0000000200010000ull; constexpr uint64_t region_size = 4ull * 1024ull * 1024ull; @@ -315,35 +447,106 @@ void TestCrossRegionUpload() { Release(page_manager, memory, region_size * 2); } -void TestBackingWritePublication() { +void TestGpuUnmarkUsesRegionMask() { + constexpr auto region_size = Libs::Graphics::TRACKER_REGION_SIZE; + constexpr auto page_size = Libs::Graphics::TRACKER_PAGE_SIZE; TrackerHarness harness; auto &tracker = harness.tracker; auto &page_manager = harness.page_manager; - const auto page_size = page_manager.GetPageSize(); - auto *memory = Allocate(page_manager, 1); - const auto address = reinterpret_cast(memory); + auto *memory = Allocate(page_manager, region_size * 2 / page_size); + const auto allocation_base = reinterpret_cast(memory); + const auto region_base = + (allocation_base + region_size - 1) & ~(region_size - 1); + Check(region_base + region_size + page_size <= + allocation_base + region_size * 2, + "test allocation does not span two complete tracker regions"); + + const auto sparse_begin = region_base + page_size; + tracker.ForEachUploadRange( + sparse_begin, page_size * 3, false, [](uint64_t, uint64_t) noexcept {}, + []() noexcept {}); + tracker.MarkRegionAsGpuModified(sparse_begin, page_size); + tracker.MarkRegionAsGpuModified(sparse_begin + page_size * 2, page_size); + ResetProtectionLog(); + tracker.UnmarkRegionAsGpuModified(sparse_begin, page_size * 3); + Check( + g_protection_calls == 1 && g_protection_log.size() == 1 && + g_protection_log[0].address == sparse_begin && + g_protection_log[0].size == page_size * 3 && + g_protection_log[0].mode == Common::VirtualMemory::Mode::Read && + !tracker.IsRegionGpuModified(sparse_begin, page_size * 3) && + Protection(reinterpret_cast(sparse_begin)) == PAGE_READONLY && + Protection(reinterpret_cast(sparse_begin + page_size)) == + PAGE_READONLY && + Protection(reinterpret_cast(sparse_begin + page_size * 2)) == + PAGE_READONLY, + "GPU unmark did not coalesce a sparse 4 MiB region mask"); + ResetProtectionLog(); + tracker.UnmarkRegionAsGpuModified(sparse_begin, page_size * 3); + Check(g_protection_calls == 0, + "idempotent GPU unmark performed a protection call"); + + const auto boundary = region_base + region_size; + const auto cross_begin = boundary - page_size; + tracker.ForEachUploadRange( + cross_begin, page_size * 2, false, [](uint64_t, uint64_t) noexcept {}, + []() noexcept {}); + tracker.MarkRegionAsGpuModified(cross_begin, page_size * 2); + ResetProtectionLog(); + tracker.UnmarkRegionAsGpuModified(cross_begin, page_size * 2); + Check(g_protection_calls == 2 && g_protection_log.size() == 2 && + g_protection_log[0].address == cross_begin && + g_protection_log[0].size == page_size && + g_protection_log[0].mode == Common::VirtualMemory::Mode::Read && + g_protection_log[1].address == boundary && + g_protection_log[1].size == page_size && + g_protection_log[1].mode == Common::VirtualMemory::Mode::Read && + !tracker.IsRegionGpuModified(cross_begin, page_size * 2), + "cross-region GPU unmark did not use one update per 4 MiB region"); + + tracker.UntrackMemory(allocation_base, region_size * 2); + Release(page_manager, memory, region_size * 2); +} + +void TestFullRegionGpuUnmarkBatching() { + constexpr auto region_size = Libs::Graphics::TRACKER_REGION_SIZE; + constexpr auto page_size = Libs::Graphics::TRACKER_PAGE_SIZE; + TrackerHarness harness; + auto &tracker = harness.tracker; + auto &page_manager = harness.page_manager; + auto *memory = Allocate(page_manager, region_size * 2 / page_size); + const auto allocation_base = reinterpret_cast(memory); + const auto region_base = + (allocation_base + region_size - 1) & ~(region_size - 1); + Check(region_base + region_size <= allocation_base + region_size * 2, + "test allocation does not contain a complete tracker region"); tracker.ForEachUploadRange( - address, page_size, true, [](uint64_t, uint64_t) noexcept {}, + region_base, region_size, false, [](uint64_t, uint64_t) noexcept {}, []() noexcept {}); - Check(tracker.IsRegionGpuModified(address, page_size) && - Protection(memory) == PAGE_NOACCESS, - "backing publication setup did not establish GPU ownership"); - std::vector dirty{{address, page_size}}; - auto writes = page_manager.ReserveBackingWrites(dirty); - Check(writes.size() == 1 && Protection(memory) == PAGE_NOACCESS, - "backing reservation exposed protected guest memory"); - uint32_t downloads = 0; - tracker.ForEachDownloadRange( - address, page_size, [&](uint64_t, uint64_t) noexcept { downloads++; }); - tracker.MarkRegionAsCpuModified(address, page_size); - writes.clear(); - Check(downloads == 1 && !tracker.IsRegionGpuModified(address, page_size) && - tracker.IsRegionCpuModified(address, page_size) && - IsWritable(memory), - "backing publication did not restore CPU ownership"); - tracker.UntrackMemory(address, page_size); - Release(page_manager, memory, page_size); + tracker.MarkRegionAsGpuModified(region_base, region_size); + Check(tracker.IsRegionGpuModified(region_base, region_size) && + Protection(reinterpret_cast(region_base)) == + PAGE_NOACCESS && + Protection(reinterpret_cast(region_base + region_size - + page_size)) == PAGE_NOACCESS, + "full-region setup did not establish GPU read protection"); + + ResetProtectionLog(); + tracker.UnmarkRegionAsGpuModified(region_base, region_size); + Check( + g_protection_log.size() == 1 && + g_protection_log[0].address == region_base && + g_protection_log[0].size == region_size && + g_protection_log[0].mode == Common::VirtualMemory::Mode::Read && + !tracker.IsRegionGpuModified(region_base, region_size) && + Protection(reinterpret_cast(region_base)) == PAGE_READONLY && + Protection(reinterpret_cast(region_base + region_size - + page_size)) == PAGE_READONLY, + "full-region GPU unmark did not use one exact 4 MiB protection request"); + + tracker.UntrackMemory(allocation_base, region_size * 2); + Release(page_manager, memory, region_size * 2); } [[noreturn]] void RunDeathCase(const char *name) { @@ -433,8 +636,11 @@ int main(int argc, char **argv) { TestCpuDirtyUpload(); TestRangeInvalidation(); TestGpuDirtyBits(); + TestExactDirtyIntervalsSharingTrackerPage(); + TestGpuDownloadProtectionMirrors(); TestCrossRegionUpload(); - TestBackingWritePublication(); + TestGpuUnmarkUsesRegionMask(); + TestFullRegionGpuUnmarkBatching(); TestFatalPaths(); std::puts("MemoryTrackerTests: all cases passed"); return 0; diff --git a/tests/PageManagerTests.cpp b/tests/PageManagerTests.cpp index 7c00a11..9a1a0e7 100644 --- a/tests/PageManagerTests.cpp +++ b/tests/PageManagerTests.cpp @@ -26,7 +26,9 @@ namespace { using Libs::Graphics::PageManager; -using Libs::Graphics::PageWatchMode; +using Libs::Graphics::RegionBits; +using Libs::Graphics::TRACKER_PAGE_SIZE; +using Libs::Graphics::TRACKER_REGION_SIZE; void Check(bool value, const char *text) { if (!value) { @@ -113,6 +115,11 @@ bool IsWritable(const void *address) { } uint64_t g_protection_calls = 0; +struct ProtectionCall { + uint64_t address; + uint64_t size; +}; +std::vector g_protection_ranges; bool ProtectAddressSpace(uint64_t vaddr, uint64_t size, Common::VirtualMemory::Mode mode) { @@ -124,6 +131,7 @@ bool ProtectAddressSpace(uint64_t vaddr, uint64_t size, } DWORD old_protection = 0; g_protection_calls++; + g_protection_ranges.push_back({vaddr, size}); return VirtualProtect(reinterpret_cast(vaddr), size, protection, &old_protection) != 0; } @@ -155,12 +163,12 @@ void TestWatchAndUnwatch() { const auto address = reinterpret_cast(memory); manager.OnGpuMap(address, page_size * 2); - manager.UpdatePageWatchers(true, address, page_size); + manager.UpdatePageWatchers(address, page_size); Check(Protection(memory) == PAGE_READONLY && IsWritable(memory + page_size), "write watch installed incorrect protections"); Check(g_protection_calls != 0, "watch protection bypassed the address-space owner callback"); - manager.UpdatePageWatchers(false, address, page_size); + manager.UpdatePageWatchers(address, page_size); Check(IsWritable(memory), "write unwatch did not restore access"); manager.OnGpuUnmap(address, page_size * 2); Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); @@ -173,39 +181,17 @@ void TestSharedWatcherCounts() { const auto address = reinterpret_cast(memory); manager.OnGpuMap(address, page_size); - manager.UpdatePageWatchers(true, address + 8, 32); - manager.UpdatePageWatchers(true, address + 128, 64); - manager.UpdatePageWatchers(false, address + 8, 32); + manager.UpdatePageWatchers(address + 8, 32); + manager.UpdatePageWatchers(address + 128, 64); + manager.UpdatePageWatchers(address + 8, 32); Check(Protection(memory) == PAGE_READONLY, "first unwatch released a shared watcher"); - manager.UpdatePageWatchers(false, address + 128, 64); + manager.UpdatePageWatchers(address + 128, 64); Check(IsWritable(memory), "last unwatch did not restore access"); manager.OnGpuUnmap(address, page_size); Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); } -void TestMixedWatcherModes() { - PageManager manager; - const auto page_size = manager.GetPageSize(); - auto *memory = Allocate(page_size); - const auto address = reinterpret_cast(memory); - - manager.OnGpuMap(address, page_size); - manager.UpdatePageWatchers(true, address, page_size, PageWatchMode::Write); - manager.UpdatePageWatchers(true, address, page_size, - PageWatchMode::ReadWrite); - Check(Protection(memory) == PAGE_NOACCESS, - "read/write watcher did not deny access"); - manager.UpdatePageWatchers(false, address, page_size, PageWatchMode::Write); - Check(Protection(memory) == PAGE_NOACCESS, - "write unwatch released a read/write watcher"); - manager.UpdatePageWatchers(false, address, page_size, - PageWatchMode::ReadWrite); - Check(IsWritable(memory), "read/write unwatch did not restore access"); - manager.OnGpuUnmap(address, page_size); - Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); -} - void TestCrossRegionRange() { PageManager manager; const auto page_size = manager.GetPageSize(); @@ -218,11 +204,27 @@ void TestCrossRegionRange() { "test allocation does not contain a region boundary"); manager.OnGpuMap(base, region_size * 2); - manager.UpdatePageWatchers(true, boundary - page_size, page_size * 2); + g_protection_calls = 0; + g_protection_ranges.clear(); + manager.UpdatePageWatchers(boundary - page_size, page_size * 2); + Check(g_protection_calls == 2 && g_protection_ranges.size() == 2 && + g_protection_ranges[0].address == boundary - page_size && + g_protection_ranges[0].size == page_size && + g_protection_ranges[1].address == boundary && + g_protection_ranges[1].size == page_size, + "cross-region watch was not split only at the region boundary"); Check(!IsWritable(reinterpret_cast(boundary - page_size)) && !IsWritable(reinterpret_cast(boundary)), "cross-region watch did not protect both pages"); - manager.UpdatePageWatchers(false, boundary - page_size, page_size * 2); + g_protection_calls = 0; + g_protection_ranges.clear(); + manager.UpdatePageWatchers(boundary - page_size, page_size * 2); + Check(g_protection_calls == 2 && g_protection_ranges.size() == 2 && + g_protection_ranges[0].address == boundary - page_size && + g_protection_ranges[0].size == page_size && + g_protection_ranges[1].address == boundary && + g_protection_ranges[1].size == page_size, + "cross-region unwatch was not split only at the region boundary"); Check(IsWritable(reinterpret_cast(boundary - page_size)) && IsWritable(reinterpret_cast(boundary)), "cross-region unwatch did not restore both pages"); @@ -239,26 +241,26 @@ void TestBatchedWatcherRanges() { const auto address = reinterpret_cast(memory); manager.OnGpuMap(address, allocation_size); - manager.UpdatePageWatchers(true, address + page_size, page_size); - manager.UpdatePageWatchers(true, address + page_size * 3, page_size); - manager.UpdatePageWatchers(true, address, page_size * 5); - manager.UpdatePageWatchers(false, address, page_size * 5); + manager.UpdatePageWatchers(address + page_size, page_size); + manager.UpdatePageWatchers(address + page_size * 3, page_size); + manager.UpdatePageWatchers(address, page_size * 5); + manager.UpdatePageWatchers(address, page_size * 5); Check(IsWritable(memory) && Protection(memory + page_size) == PAGE_READONLY && IsWritable(memory + page_size * 2) && Protection(memory + page_size * 3) == PAGE_READONLY && IsWritable(memory + page_size * 4), "fragmented unwatch lost overlapping watcher counts"); - manager.UpdatePageWatchers(false, address + page_size, page_size); - manager.UpdatePageWatchers(false, address + page_size * 3, page_size); + manager.UpdatePageWatchers(address + page_size, page_size); + manager.UpdatePageWatchers(address + page_size * 3, page_size); g_protection_calls = 0; - manager.UpdatePageWatchers(true, address, allocation_size); + manager.UpdatePageWatchers(address, allocation_size); Check(g_protection_calls == 4 && Protection(memory) == PAGE_READONLY && Protection(memory + region_size) == PAGE_READONLY && Protection(memory + region_size * 2) == PAGE_READONLY && Protection(memory + allocation_size - page_size) == PAGE_READONLY, "large watch was not batched and protected by tracking region"); - manager.UpdatePageWatchers(false, address, allocation_size); + manager.UpdatePageWatchers(address, allocation_size); Check(IsWritable(memory) && IsWritable(memory + region_size) && IsWritable(memory + region_size * 2) && IsWritable(memory + allocation_size - page_size), @@ -268,20 +270,266 @@ void TestBatchedWatcherRanges() { Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); } +void TestRegionMaskWatcherRanges() { + PageManager manager; + constexpr auto page_size = TRACKER_PAGE_SIZE; + constexpr auto region_size = TRACKER_REGION_SIZE; + auto *memory = Allocate(region_size * 2); + const auto allocation_base = reinterpret_cast(memory); + const auto region_base = + (allocation_base + region_size - 1) & ~(region_size - 1); + Check(region_base + region_size <= allocation_base + region_size * 2, + "test allocation does not contain a complete tracking region"); + manager.OnGpuMap(allocation_base, region_size * 2); + + RegionBits full_mask; + full_mask.Fill(); + g_protection_calls = 0; + g_protection_ranges.clear(); + manager.UpdatePageWatchersForRegion(region_base, full_mask); + Check(g_protection_calls == 1 && g_protection_ranges.size() == 1 && + g_protection_ranges[0].address == region_base && + g_protection_ranges[0].size == region_size, + "full region mask did not use one protection span"); + g_protection_calls = 0; + g_protection_ranges.clear(); + manager.UpdatePageWatchersForRegion(region_base, full_mask); + Check(g_protection_calls == 1 && g_protection_ranges.size() == 1 && + g_protection_ranges[0].address == region_base && + g_protection_ranges[0].size == region_size, + "full region unmask did not use one protection span"); + + RegionBits sparse_mask; + sparse_mask.Set(1); + sparse_mask.Set(3); + g_protection_calls = 0; + g_protection_ranges.clear(); + manager.UpdatePageWatchersForRegion(region_base, sparse_mask); + Check(g_protection_calls == 2 && + Protection(reinterpret_cast(region_base + page_size)) == + PAGE_READONLY && + IsWritable(reinterpret_cast(region_base + page_size * 2)) && + Protection(reinterpret_cast(region_base + page_size * 3)) == + PAGE_READONLY, + "sparse region mask installed incorrect write watchers"); + g_protection_calls = 0; + g_protection_ranges.clear(); + manager.UpdatePageWatchersForRegion(region_base, sparse_mask); + Check(g_protection_calls == 1 && g_protection_ranges.size() == 1 && + g_protection_ranges[0].address == region_base + page_size && + g_protection_ranges[0].size == page_size * 3, + "sparse unmask did not bridge a compatible gap"); + + manager.UpdatePageWatchersForRegion(region_base, sparse_mask); + g_protection_calls = 0; + manager.UpdatePageWatchersForRegion(region_base, sparse_mask); + Check(g_protection_calls == 0, + "duplicate sparse watch changed an already protected range"); + manager.UpdatePageWatchersForRegion(region_base, sparse_mask); + Check(g_protection_calls == 0, + "first sparse unwatch released a duplicate watcher"); + manager.UpdatePageWatchersForRegion(region_base, sparse_mask); + + manager.UpdatePageWatchers(region_base + page_size * 2, page_size); + g_protection_calls = 0; + g_protection_ranges.clear(); + manager.UpdatePageWatchersForRegion(region_base, sparse_mask); + Check(g_protection_calls == 1 && g_protection_ranges.size() == 1 && + g_protection_ranges[0].address == region_base + page_size && + g_protection_ranges[0].size == page_size * 3, + "sparse mask did not bridge a compatible protected gap"); + g_protection_calls = 0; + g_protection_ranges.clear(); + manager.UpdatePageWatchersForRegion(region_base, sparse_mask); + Check(g_protection_calls == 2 && g_protection_ranges.size() == 2 && + g_protection_ranges[0].address == region_base + page_size && + g_protection_ranges[0].size == page_size && + g_protection_ranges[1].address == region_base + page_size * 3 && + g_protection_ranges[1].size == page_size, + "sparse unmask crossed an incompatible protected gap"); + manager.UpdatePageWatchers(region_base + page_size * 2, page_size); + + g_protection_calls = 0; + manager.UpdatePageWatchersForRegion(region_base, sparse_mask); + Check(g_protection_calls == 2 && + Protection(reinterpret_cast(region_base + page_size)) == + PAGE_NOACCESS && + Protection(reinterpret_cast(region_base + page_size * 3)) == + PAGE_NOACCESS, + "sparse read mask did not deny access"); + g_protection_calls = 0; + g_protection_ranges.clear(); + manager.UpdatePageWatchersForRegion(region_base, sparse_mask); + Check(g_protection_calls == 1 && g_protection_ranges.size() == 1 && + g_protection_ranges[0].size == page_size * 3, + "sparse read unmask did not bridge a compatible gap"); + + manager.OnGpuUnmap(allocation_base, region_size * 2); + Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); +} + +void TestRegionEndpointBatching() { + PageManager manager; + constexpr auto page_size = TRACKER_PAGE_SIZE; + constexpr auto region_size = TRACKER_REGION_SIZE; + constexpr auto last_page = region_size / page_size - 1; + auto *memory = Allocate(region_size * 2); + const auto allocation_base = reinterpret_cast(memory); + const auto region_base = + (allocation_base + region_size - 1) & ~(region_size - 1); + Check(region_base + region_size <= allocation_base + region_size * 2, + "test allocation does not contain a complete tracking region"); + manager.OnGpuMap(allocation_base, region_size * 2); + + RegionBits endpoints; + endpoints.Set(0); + endpoints.Set(last_page); + g_protection_calls = 0; + g_protection_ranges.clear(); + manager.UpdatePageWatchersForRegion(region_base, endpoints); + Check(g_protection_calls == 2 && g_protection_ranges.size() == 2 && + g_protection_ranges[0].address == region_base && + g_protection_ranges[0].size == page_size && + g_protection_ranges[1].address == + region_base + region_size - page_size && + g_protection_ranges[1].size == page_size, + "endpoint watch did not protect only the selected pages"); + + g_protection_calls = 0; + g_protection_ranges.clear(); + manager.UpdatePageWatchersForRegion(region_base, endpoints); + Check(g_protection_calls == 1 && g_protection_ranges.size() == 1 && + g_protection_ranges[0].address == region_base && + g_protection_ranges[0].size == region_size, + "endpoint unwatch did not coalesce the compatible 4 MiB span"); + + RegionBits full_mask; + full_mask.Fill(); + manager.UpdatePageWatchersForRegion(region_base, full_mask); + g_protection_calls = 0; + g_protection_ranges.clear(); + manager.UpdatePageWatchersForRegion(region_base, full_mask); + Check(g_protection_calls == 0, + "duplicate full-region watch issued a redundant protection call"); + manager.UpdatePageWatchersForRegion(region_base, full_mask); + Check(g_protection_calls == 0, + "first full-region unwatch released overlapping watcher counts"); + manager.UpdatePageWatchersForRegion(region_base, full_mask); + Check(g_protection_calls == 1 && g_protection_ranges.size() == 1 && + g_protection_ranges[0].address == region_base && + g_protection_ranges[0].size == region_size, + "last full-region unwatch did not use one 4 MiB protection call"); + + manager.OnGpuUnmap(allocation_base, region_size * 2); + Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); +} + +void TestReadWriteWatcherInteractions() { + PageManager manager; + constexpr auto page_size = TRACKER_PAGE_SIZE; + constexpr auto region_size = TRACKER_REGION_SIZE; + auto *memory = Allocate(region_size * 2); + const auto allocation_base = reinterpret_cast(memory); + const auto region_base = + (allocation_base + region_size - 1) & ~(region_size - 1); + Check(region_base + region_size <= allocation_base + region_size * 2, + "test allocation does not contain a complete tracking region"); + manager.OnGpuMap(allocation_base, region_size * 2); + + RegionBits write_mask; + write_mask.SetRange(10, 15); + RegionBits read_mask; + read_mask.Set(11); + read_mask.Set(13); + + g_protection_calls = 0; + g_protection_ranges.clear(); + manager.UpdatePageWatchersForRegion(region_base, write_mask); + Check(g_protection_calls == 1 && g_protection_ranges.size() == 1 && + g_protection_ranges[0].address == region_base + page_size * 10 && + g_protection_ranges[0].size == page_size * 5, + "contiguous write watch was not batched"); + + g_protection_calls = 0; + g_protection_ranges.clear(); + manager.UpdatePageWatchersForRegion(region_base, read_mask); + Check( + g_protection_calls == 2 && + Protection(reinterpret_cast(region_base + page_size * 11)) == + PAGE_NOACCESS && + Protection(reinterpret_cast(region_base + page_size * 12)) == + PAGE_READONLY && + Protection(reinterpret_cast(region_base + page_size * 13)) == + PAGE_NOACCESS, + "read watchers did not compose with write-only watchers"); + + g_protection_calls = 0; + g_protection_ranges.clear(); + manager.UpdatePageWatchersForRegion(region_base, write_mask); + Check( + g_protection_calls == 3 && + IsWritable(reinterpret_cast(region_base + page_size * 10)) && + Protection(reinterpret_cast(region_base + page_size * 11)) == + PAGE_NOACCESS && + IsWritable(reinterpret_cast(region_base + page_size * 12)) && + Protection(reinterpret_cast(region_base + page_size * 13)) == + PAGE_NOACCESS && + IsWritable(reinterpret_cast(region_base + page_size * 14)), + "write unwatch changed pages still owned by read watchers"); + + g_protection_calls = 0; + g_protection_ranges.clear(); + manager.UpdatePageWatchersForRegion(region_base, read_mask); + Check( + g_protection_calls == 1 && g_protection_ranges.size() == 1 && + g_protection_ranges[0].address == region_base + page_size * 11 && + g_protection_ranges[0].size == page_size * 3 && + IsWritable(reinterpret_cast(region_base + page_size * 11)) && + IsWritable(reinterpret_cast(region_base + page_size * 13)), + "read unwatch did not coalesce through compatible writable state"); + + manager.OnGpuUnmap(allocation_base, region_size * 2); + Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); +} + [[noreturn]] void RunDeathCase(const char *name) { PageManager manager; const auto page_size = manager.GetPageSize(); if (std::strcmp(name, "invalid-range") == 0) { - manager.UpdatePageWatchers(true, (1ull << 40u) - 1, 2); + manager.UpdatePageWatchers((1ull << 40u) - 1, 2); } else if (std::strcmp(name, "unknown-untrack") == 0) { - manager.UpdatePageWatchers(false, 0x1000, page_size); + manager.UpdatePageWatchers(0x1000, page_size); } else if (std::strcmp(name, "destructor-watch") == 0) { auto doomed = std::make_unique(); auto *memory = Allocate(page_size); const auto address = reinterpret_cast(memory); doomed->OnGpuMap(address, page_size); - doomed->UpdatePageWatchers(true, address, page_size); + doomed->UpdatePageWatchers(address, page_size); doomed.reset(); + } else if (std::strcmp(name, "known-write-underflow") == 0) { + auto *memory = Allocate(page_size); + const auto address = reinterpret_cast(memory); + manager.OnGpuMap(address, page_size); + manager.UpdatePageWatchers(address, page_size); + manager.UpdatePageWatchers(address, page_size); + manager.UpdatePageWatchers(address, page_size); + } else if (std::strcmp(name, "read-overflow") == 0) { + auto *memory = Allocate(page_size); + const auto address = reinterpret_cast(memory); + manager.OnGpuMap(address, page_size); + RegionBits mask; + const auto region_base = address & ~(TRACKER_REGION_SIZE - 1); + const auto page = static_cast((address - region_base) / page_size); + mask.Set(page); + manager.UpdatePageWatchersForRegion(region_base, mask); + manager.UpdatePageWatchersForRegion(region_base, mask); + } else if (std::strcmp(name, "write-overflow") == 0) { + auto *memory = Allocate(page_size); + const auto address = reinterpret_cast(memory); + manager.OnGpuMap(address, page_size); + for (uint32_t count = 0; count < 128; count++) { + manager.UpdatePageWatchers(address, page_size); + } } std::_Exit(0x7f); } @@ -328,7 +576,8 @@ void CheckDeathCase(const char *name) { void TestFatalPaths() { for (const char *name : - {"invalid-range", "unknown-untrack", "destructor-watch"}) { + {"invalid-range", "unknown-untrack", "destructor-watch", + "known-write-underflow", "read-overflow", "write-overflow"}) { CheckDeathCase(name); } } @@ -350,9 +599,11 @@ int main(int argc, char **argv) { } TestWatchAndUnwatch(); TestSharedWatcherCounts(); - TestMixedWatcherModes(); TestCrossRegionRange(); TestBatchedWatcherRanges(); + TestRegionMaskWatcherRanges(); + TestRegionEndpointBatching(); + TestReadWriteWatcherInteractions(); TestFatalPaths(); std::puts("PageManagerTests: all cases passed"); return 0; diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index d0f27a3..680fdbe 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -91,6 +91,26 @@ template concept HasGetDownloadBuffer = requires(Cache& cache) { cache.GetDownloadBuffer(uint64_t {1}); }; static_assert(!HasGetDownloadBuffer); +template +concept HasSynchronizeImageToBuffer = + requires(Cache& cache) { cache.SynchronizeImageToBuffer(uint64_t {1}, uint64_t {1}); }; +template +concept HasObtainBufferForImageCopy = + requires(Cache& cache) { cache.ObtainBufferForImageCopy(uint64_t {1}, uint64_t {1}); }; +template +concept HasObtainBufferForImageWrite = + requires(Cache& cache) { cache.ObtainBufferForImageWrite(uint64_t {1}, uint64_t {1}); }; +template +concept HasDiscardGpuDirtyBytes = + requires(Cache& cache) { cache.DiscardGpuDirtyBytes(uint64_t {1}, uint64_t {1}); }; +template +concept HasGpuOwnedImageSource = requires(Source& source) { source.gpu_owned; }; +static_assert(!HasSynchronizeImageToBuffer); +static_assert(!HasObtainBufferForImageCopy); +static_assert(!HasObtainBufferForImageWrite); +static_assert(!HasDiscardGpuDirtyBytes); +static_assert(!HasGpuOwnedImageSource); + template concept HasLegacyImageLayout = requires(Backing& backing) { backing.layout; }; static_assert(!HasLegacyImageLayout); @@ -111,6 +131,11 @@ struct BufferCacheTestAccess { } static StreamBuffer& DownloadBuffer(BufferCache& cache) { return cache.m_download_buffer; } + + static bool SynchronizeBufferFromImage(BufferCache& cache, Buffer& buffer, uint64_t address, + uint64_t size) { + return cache.SynchronizeBufferFromImage(buffer, address, size); + } }; struct StreamBufferTestAccess { @@ -1858,21 +1883,6 @@ public: MarkGpuWrite(base + second_offset, sizeof(second_value)); cache.FillBuffer(base + first_offset, sizeof(first_value), first_value); cache.FillBuffer(base + second_offset, sizeof(second_value), second_value); - cache.DiscardGpuDirtyBytes(base + first_offset, sizeof(first_value)); - Require(name, "exact dirty discard", - !cache.HasGpuDirtyBytes(base + first_offset, sizeof(first_value)) && - cache.HasGpuDirtyBytes(base + second_offset, sizeof(second_value)) && - cache.IsRegionGpuModified(base, TRACKER_PAGE_SIZE), - "discarding one exact range released its dirty page sibling"); - cache.DiscardGpuDirtyBytes(base + second_offset, sizeof(second_value)); - Require(name, "last dirty discard", - !cache.HasGpuDirtyBytes(base + second_offset, sizeof(second_value)) && - !cache.IsRegionGpuModified(base, TRACKER_PAGE_SIZE), - "discarding the last exact range retained page ownership"); - MarkGpuWrite(base + first_offset, sizeof(first_value)); - MarkGpuWrite(base + second_offset, sizeof(second_value)); - cache.FillBuffer(base + first_offset, sizeof(first_value), first_value); - cache.FillBuffer(base + second_offset, sizeof(second_value), second_value); auto& download = BufferCacheTestAccess::DownloadBuffer(cache); auto* fixed_download = &download; const auto fixed_download_handle = download.Handle(); @@ -1930,7 +1940,8 @@ public: const auto gc_submission_tick = scheduler.CurrentTick(); cache.RunGarbageCollector(); Require(name, "dirty retirement", !cache.HasPageOverlap(base, allocation_size), - "aged GPU-dirty buffer survived pressured collection"); + "aged GPU-dirty buffer survived pressured " + "collection"); uint32_t first_before_completion = 0; uint32_t second_before_completion = 0; @@ -1941,11 +1952,14 @@ public: Require(name, "deferred dirty retirement", first_before_completion == first_stale && second_before_completion == second_stale && + cache.IsRegionGpuModified(base + first_offset, sizeof(first_value)) && + cache.IsRegionGpuModified(base + second_offset, sizeof(second_value)) && + cache.HasGpuDirtyBytes(base + first_offset, sizeof(first_value)) && + cache.HasGpuDirtyBytes(base + second_offset, sizeof(second_value)) && scheduler.CurrentTick() == gc_submission_tick, - "buffer GC synchronously submitted or published its download"); - Require(name, "publication-gated CPU fault", - resources.HandleFault(PageFaultAccess::Read, base + first_offset), - "CPU fault did not wait for the deferred buffer publication"); + "buffer GC published bytes or cleared dirty " + "ownership before GPU completion"); + scheduler.FinishCurrent(); uint32_t first_backing = 0; uint32_t second_backing = 0; @@ -1955,8 +1969,13 @@ public: std::memcpy(&clean_backing, memory + clean_offset, sizeof(clean_backing)); Require(name, "downloaded contents", first_backing == first_value && second_backing == second_value && - clean_backing == clean_value, - "dirty GC did not publish exact buffer ranges"); + clean_backing == clean_value && + !cache.IsRegionGpuModified(base + first_offset, sizeof(first_value)) && + !cache.IsRegionGpuModified(base + second_offset, sizeof(second_value)) && + !cache.HasGpuDirtyBytes(base + first_offset, sizeof(first_value)) && + !cache.HasGpuDirtyBytes(base + second_offset, sizeof(second_value)), + "dirty GC did not publish exact ranges before " + "clearing broad page ownership"); auto unmap_allocation = cache.ObtainBuffer(scheduler.Current(), base + unmap_offset, sizeof(unmap_value), true, false); @@ -1969,7 +1988,8 @@ public: std::memcpy(&unmap_backing, memory + unmap_offset, sizeof(unmap_backing)); Require(name, "direct-unmap contents", unmap_backing == unmap_value && !cache.HasPageOverlap(base + 0x4000, 0x4000), - "direct dirty unmap cleared tracking before publishing bytes"); + "direct dirty unmap cleared tracking before " + "publishing bytes"); auto partial_unmap_allocation = cache.ObtainBuffer(scheduler.Current(), base + 0x8000, 0x8000, true, false); @@ -1981,18 +2001,21 @@ public: cache.FillBuffer(base + partial_unmap_survivor_offset, sizeof(partial_unmap_survivor_value), partial_unmap_survivor_value); cache.UnmapMemory(base + 0x8000, 0x4000); - auto [survivor, survivor_offset] = cache.ObtainBufferForImageWrite( - base + partial_unmap_survivor_offset, sizeof(partial_unmap_survivor_value)); - Require(name, "partial-unmap survivor", survivor != nullptr, + auto survivor = + cache.ObtainBuffer(scheduler.Current(), base + partial_unmap_survivor_offset, + sizeof(partial_unmap_survivor_value), false, true); + Require(name, "partial-unmap survivor", survivor.buffer != nullptr, "still-mapped cached-buffer remainder could not be recreated"); - scheduler.Current().RetainResourceUntilFence(survivor); + if (survivor.owner != nullptr) { + scheduler.Current().RetainResourceUntilFence(survivor.owner); + } auto partial_unmap_readback = CreateHostBuffer(name, sizeof(partial_unmap_survivor_value), vk::BufferUsageFlagBits::eTransferDst, {0}); - const vk::BufferCopy survivor_copy {survivor_offset, 0, + const vk::BufferCopy survivor_copy {survivor.offset, 0, sizeof(partial_unmap_survivor_value)}; - scheduler.Current().Handle().copyBuffer( - survivor->Handle(), partial_unmap_readback.buffer, 1, &survivor_copy); + scheduler.Current().Handle().copyBuffer(survivor.buffer, partial_unmap_readback.buffer, + 1, &survivor_copy); vk::BufferMemoryBarrier survivor_barrier {}; survivor_barrier.sType = vk::StructureType::eBufferMemoryBarrier; survivor_barrier.srcAccessMask = vk::AccessFlagBits::eTransferWrite; @@ -2037,9 +2060,10 @@ public: Require(name, "near-capacity backing remained deferred", large_before_completion == large_stale, "near-capacity Buffer GC published before its scheduler tick"); + scheduler.FinishCurrent(); const auto large_image_source = cache.ObtainBufferForImage(base + large_offset, sizeof(large_value)); - Require(name, "fixed download during Buffer publication", + Require(name, "fixed download after Buffer retirement", large_image_source.buffer != nullptr && &BufferCacheTestAccess::DownloadBuffer(cache) == fixed_download && fixed_download->Handle() == fixed_download_handle && @@ -2134,23 +2158,22 @@ public: Require(name, "disjoint retirement remained deferred", disjoint_before_unmap == disjoint_stale, "whole-owner Buffer publication completed before synchronization"); + scheduler.FinishCurrent(); cache.UnmapMemory(base + disjoint_owner_offset + 0x4000, 0x4000); uint32_t disjoint_after_unmap = 0; Libs::LibKernel::Memory::TryReadBacking( base + disjoint_dirty_offset, &disjoint_after_unmap, sizeof(disjoint_after_unmap)); Require( - name, "disjoint publication-gated unmap", + name, "disjoint synchronized unmap", disjoint_after_unmap == disjoint_value && !cache.HasGpuDirtyBytes(base + disjoint_dirty_offset, sizeof(disjoint_value)), - "disjoint unmap did not wait for the pending whole-owner " - "publication"); + "disjoint unmap lost the completed whole-owner publication"); auto disjoint_new_owner = cache.ObtainBuffer(scheduler.Current(), base + disjoint_new_owner_offset, sizeof(disjoint_value), true, false); - Require(name, "disjoint publication-gated reacquire", + Require(name, "disjoint post-publication reacquire", disjoint_new_owner.owner != nullptr, - "disjoint acquisition did not wait for whole-owner " - "publication"); + "disjoint acquisition failed after whole-owner publication"); scheduler.Current().RetainResourceUntilFence(disjoint_new_owner.owner); uint32_t disjoint_backing = 0; Libs::LibKernel::Memory::TryReadBacking(base + disjoint_dirty_offset, &disjoint_backing, @@ -2158,7 +2181,7 @@ public: Require(name, "disjoint retirement contents", disjoint_backing == disjoint_value, "old retirement callback lost dirty bytes or retained ownership " "after a disjoint reacquire"); - cache.DiscardGpuDirtyBytes(base + disjoint_new_owner_offset, sizeof(disjoint_value)); + cache.ReadMemory(base + disjoint_new_owner_offset, sizeof(disjoint_value)); constexpr uint64_t reacquire_owner_offset = 0x2200000; constexpr uint64_t reacquire_owner_size = 0x8000; @@ -2193,13 +2216,13 @@ public: Require(name, "reacquire publication remained deferred", reacquire_before == reacquire_stale, "fresh whole-owner publication completed before reacquisition"); + scheduler.FinishCurrent(); auto reacquired = cache.ObtainBuffer(scheduler.Current(), base + reacquire_disjoint_offset, sizeof(reacquire_value), true, false); - Require(name, "independent disjoint publication-gated reacquire", + Require(name, "independent disjoint post-publication reacquire", reacquired.owner != nullptr, - "clean disjoint-half acquisition did not synchronize the " - "pending whole-owner publication"); + "clean disjoint-half acquisition failed after publication"); scheduler.Current().RetainResourceUntilFence(reacquired.owner); uint32_t reacquire_after = 0; Libs::LibKernel::Memory::TryReadBacking(base + reacquire_dirty_offset, &reacquire_after, @@ -2210,7 +2233,7 @@ public: !cache.HasGpuDirtyBytes(base + reacquire_dirty_offset, sizeof(reacquire_value)), "disjoint-half acquisition failed to publish the retired " "owner's dirty prefix"); - cache.DiscardGpuDirtyBytes(base + reacquire_disjoint_offset, sizeof(reacquire_value)); + cache.ReadMemory(base + reacquire_disjoint_offset, sizeof(reacquire_value)); resources.SetGpu(nullptr); resources.UnmapMemory(base, allocation_size); @@ -2421,6 +2444,111 @@ public: vk::PipelineStageFlagBits::eHost, {}, 0, nullptr, 1, &barrier, 0, nullptr); }; + const auto TransferReadBarrier = [&](vk::Buffer buffer, uint64_t size) { + vk::BufferMemoryBarrier barrier {}; + barrier.sType = vk::StructureType::eBufferMemoryBarrier; + barrier.srcAccessMask = vk::AccessFlagBits::eTransferWrite; + barrier.dstAccessMask = vk::AccessFlagBits::eTransferRead; + barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.buffer = buffer; + barrier.offset = 0; + barrier.size = size; + scheduler.Current().Handle().pipelineBarrier( + vk::PipelineStageFlagBits::eTransfer, vk::PipelineStageFlagBits::eTransfer, {}, + 0, nullptr, 1, &barrier, 0, nullptr); + }; + + // A formatted Buffer read must use the private + // shadPS4-shaped image-copy path. Use a request larger + // than the stream shortcut and poison guest backing + // after upload so stale CPU staging cannot accidentally + // satisfy the content check. + constexpr uint64_t mip_prefix_offset = 0x2740000; + constexpr uint32_t mip_format = Prospero::GpuEnumValue(Prospero::BufferFormat::k32UInt); + constexpr uint32_t mip_tile = Prospero::GpuEnumValue(Prospero::TileMode::kLinear); + constexpr uint32_t mip_width = 4097; + constexpr uint32_t mip_height = 1; + constexpr uint32_t mip_levels = 1; + const uint32_t mip_pitch = + TileGetTexturePitch(mip_format, mip_width, mip_levels, mip_tile); + TileSizeAlign mip_total {}; + std::array mip_sizes {}; + std::array mip_padded {}; + TileGetTextureSize(mip_format, mip_width, mip_height, mip_pitch, mip_levels, mip_tile, + &mip_total, mip_sizes.data(), mip_padded.data()); + const uint64_t mip_prefix_size = mip_sizes[0].offset + mip_sizes[0].size; + const uint64_t mip_guest_size = mip_total.size + 256; + Require(name, "formatted mip fixture", + mip_sizes[0].offset == 0 && mip_prefix_size > BufferCache::CACHING_PAGE_SIZE && + mip_prefix_size < mip_guest_size && mip_guest_size % sizeof(uint32_t) == 0, + "single-mip fixture did not expose a cache-sized fitting backing prefix"); + std::vector mip_native(mip_guest_size / sizeof(uint32_t)); + std::iota(mip_native.begin(), mip_native.end(), 0x61000000u); + std::memcpy(memory + mip_prefix_offset, mip_native.data(), mip_guest_size); + auto mip_desc = MakeLinearDesc( + base + mip_prefix_offset, mip_guest_size, vk::Format::eR32Uint, mip_format, + Prospero::ImageType::kColor2D, {mip_width, mip_height, 1}, 1, sizeof(uint32_t), 1); + mip_desc.info.resources.levels = mip_levels; + mip_desc.info.pitch = mip_pitch; + mip_desc.view_info.level_count = mip_levels; + for (uint32_t level = 0; level < mip_levels; level++) { + mip_desc.info.mip_layout[level] = {mip_sizes[level].offset, mip_sizes[level].size, + mip_padded[level].width, + mip_padded[level].height}; + } + const auto mip_image = texture_cache.FindImage(mip_desc); + texture_cache.MarkGpuWritten(mip_image); + std::vector mip_stale(mip_native.size(), 0xdeadbeefu); + Libs::LibKernel::Memory::WriteBacking(base + mip_prefix_offset, mip_stale.data(), + mip_guest_size); + + Libs::Graphics::Buffer mip_insufficient( + m_runtime_context, scheduler, MemoryUsage::DeviceLocal, mip_desc.info.data.address, + AllFlags, mip_prefix_size - 1); + Libs::Graphics::Buffer mip_prefix(m_runtime_context, scheduler, + MemoryUsage::DeviceLocal, mip_desc.info.data.address, + AllFlags, mip_prefix_size); + Require(name, "formatted mip containment", + !BufferCacheTestAccess::SynchronizeBufferFromImage( + resources.GetBufferCache(), mip_insufficient, mip_desc.info.data.address, + mip_prefix_size - 1) && + BufferCacheTestAccess::SynchronizeBufferFromImage( + resources.GetBufferCache(), mip_prefix, mip_desc.info.data.address, + mip_prefix_size) && + texture_cache.GetImage(mip_image).IsGpuModified(), + "image synchronization accepted a partial first " + "mip, rejected a fitting mip " + "prefix, or transferred ownership"); + + auto mip_formatted = resources.GetBufferCache().ObtainBuffer( + command, mip_desc.info.data.address, mip_desc.info.data.size, false, true, true); + Require(name, "formatted Buffer path", + mip_formatted.owner != nullptr && mip_formatted.buffer != nullptr && + texture_cache.GetImage(mip_image).IsGpuModified(), + "formatted read bypassed the cached image-copy " + "path or transferred ownership"); + command.RetainResourceUntilFence(mip_formatted.owner); + auto mip_prefix_readback = + CreateHostBuffer(name, mip_prefix_size, vk::BufferUsageFlagBits::eTransferDst, + std::vector(mip_prefix_size / sizeof(uint32_t), 0)); + auto mip_formatted_readback = + CreateHostBuffer(name, mip_guest_size, vk::BufferUsageFlagBits::eTransferDst, + std::vector(mip_guest_size / sizeof(uint32_t), 0)); + TransferReadBarrier(mip_prefix.Handle(), mip_prefix_size); + const vk::BufferCopy mip_prefix_copy {0, 0, mip_prefix_size}; + command.Handle().copyBuffer(mip_prefix.Handle(), mip_prefix_readback.buffer, 1, + &mip_prefix_copy); + TransferReadBarrier(mip_formatted.buffer, mip_guest_size); + const vk::BufferCopy mip_formatted_copy {mip_formatted.offset, 0, mip_guest_size}; + command.Handle().copyBuffer(mip_formatted.buffer, mip_formatted_readback.buffer, 1, + &mip_formatted_copy); + HostReadBarrier(mip_prefix_readback.buffer, mip_prefix_readback.size, + vk::PipelineStageFlagBits::eTransfer, + vk::AccessFlagBits::eTransferWrite); + HostReadBarrier(mip_formatted_readback.buffer, mip_formatted_readback.size, + vk::PipelineStageFlagBits::eTransfer, + vk::AccessFlagBits::eTransferWrite); const std::array volume_values {0x10203040u, 0x50607080u}; std::memcpy(memory + 0x1000, volume_values.data(), sizeof(volume_values)); auto array_desc = @@ -2441,7 +2569,68 @@ public: texture_cache.GetImage(volume_image).backing.image_type == vk::ImageType::e3D && texture_cache.GetImage(volume_image).IsGpuModified(), - "array-to-volume expansion lost slices or GPU ownership"); + "array-to-volume expansion lost slices or GPU " + "ownership"); + + constexpr uint64_t unique_volume_offset = 0x27b0000; + std::memcpy(memory + unique_volume_offset, volume_values.data(), sizeof(volume_values)); + auto unique_volume_desc = MakeLinearDesc( + base + unique_volume_offset, sizeof(volume_values), vk::Format::eR32Uint, + Prospero::GpuEnumValue(Prospero::BufferFormat::k32UInt), + Prospero::ImageType::kColor3D, {1, 1, 2}, 1, sizeof(uint32_t), 1); + const auto unique_volume_image = texture_cache.FindImage(unique_volume_desc); + texture_cache.MarkGpuWritten(unique_volume_image); + Libs::Graphics::Buffer partial_volume( + m_runtime_context, scheduler, MemoryUsage::DeviceLocal, + unique_volume_desc.info.data.address, AllFlags, + unique_volume_desc.info.data.size - sizeof(uint32_t)); + Libs::Graphics::Buffer full_volume( + m_runtime_context, scheduler, MemoryUsage::DeviceLocal, + unique_volume_desc.info.data.address, AllFlags, unique_volume_desc.info.data.size); + Require(name, "formatted volume containment", + !BufferCacheTestAccess::SynchronizeBufferFromImage( + resources.GetBufferCache(), partial_volume, + unique_volume_desc.info.data.address, + unique_volume_desc.info.data.size - sizeof(uint32_t)) && + BufferCacheTestAccess::SynchronizeBufferFromImage( + resources.GetBufferCache(), full_volume, + unique_volume_desc.info.data.address, + unique_volume_desc.info.data.size) && + texture_cache.GetImage(unique_volume_image).IsGpuModified(), + "partial 3D synchronization was accepted, " + "full-volume synchronization was " + "rejected, or ownership changed"); + + constexpr uint64_t depth_containment_offset = 0x27c0000; + constexpr std::array depth_containment_values {0.25f, 0.75f, 0.5f}; + std::memcpy(memory + depth_containment_offset, depth_containment_values.data(), + sizeof(depth_containment_values)); + auto depth_containment = MakeLinearDesc( + base + depth_containment_offset, sizeof(depth_containment_values), + vk::Format::eD32Sfloat, Prospero::GpuEnumValue(Prospero::BufferFormat::k32Float), + Prospero::ImageType::kColor2D, {2, 1, 1}, 1, sizeof(float), 1); + depth_containment.type = BindingType::DepthTarget; + depth_containment.info.resources.levels = 2; + depth_containment.info.mip_layout[0] = {0, 2 * sizeof(float), 2, 1}; + depth_containment.info.mip_layout[1] = {2 * sizeof(float), sizeof(float), 1, 1}; + depth_containment.view_info.format = vk::Format::eD32Sfloat; + depth_containment.view_info.aspect = vk::ImageAspectFlagBits::eDepth; + depth_containment.view_info.usage = vk::ImageUsageFlagBits::eDepthStencilAttachment; + depth_containment.view_info.level_count = 2; + const auto depth_containment_image = texture_cache.FindImage(depth_containment); + texture_cache.MarkGpuWritten(depth_containment_image); + Libs::Graphics::Buffer partial_depth(m_runtime_context, scheduler, + MemoryUsage::DeviceLocal, + depth_containment.info.data.address, AllFlags, + depth_containment.info.mip_layout[0].size); + Require(name, "formatted depth containment", + !BufferCacheTestAccess::SynchronizeBufferFromImage( + resources.GetBufferCache(), partial_depth, + depth_containment.info.data.address, + depth_containment.info.mip_layout[0].size) && + texture_cache.GetImage(depth_containment_image).IsGpuModified(), + "partial depth synchronization was accepted or " + "transferred ownership"); auto native_array_info = array_desc.info; native_array_info.data = {}; @@ -2458,7 +2647,8 @@ public: native_array.backing.state.layout == vk::ImageLayout::eGeneral && native_array.backing.state.access_mask == (vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eTransferRead), - "Image::CopyImage did not retain pinned source/destination states"); + "Image::CopyImage did not retain pinned " + "source/destination states"); constexpr uint64_t block_alias_offset = 0x23000; constexpr std::array block_alias_data {0x01234567u, 0x89abcdefu, @@ -2540,13 +2730,11 @@ public: sizeof(refreshed_multisample_source)); constexpr uint64_t ms_stencil_offset = 0x80000; constexpr uint64_t ms_stencil_size = 0x10000; - auto [ms_stencil_owner, ms_stencil_owner_offset] = - resources.GetBufferCache().ObtainBufferForImageWrite(base + ms_stencil_offset, - ms_stencil_size); - (void)ms_stencil_owner_offset; - Require(name, "MS stencil buffer allocation", ms_stencil_owner != nullptr, + auto ms_stencil_owner = resources.GetBufferCache().ObtainBuffer( + command, base + ms_stencil_offset, ms_stencil_size, false, true); + Require(name, "MS stencil buffer allocation", ms_stencil_owner.owner != nullptr, "failed to create an unequal-sample stencil source"); - command.RetainResourceUntilFence(ms_stencil_owner); + command.RetainResourceUntilFence(ms_stencil_owner.owner); resources.GetBufferCache().FillBuffer(base + ms_stencil_offset, ms_stencil_size, 0x41414141u); auto ms_depth_desc = color_desc; @@ -2599,11 +2787,9 @@ public: oversized_ms.info.data.size = (32ull << 20) + 4; const bool oversized_ms_readback = !TextureCacheTestAccess::TryDownload(texture_cache, ms_depth_image); - const bool oversized_ms_mirror = !texture_cache.SynchronizeImageToBuffer( - ms_depth_desc.info.data.address, ms_data_size); oversized_ms.info.data.size = ms_data_size; Require(name, "oversized multisample download rejection", - oversized_ms_readback && oversized_ms_mirror && oversized_ms.IsGpuModified() && + oversized_ms_readback && oversized_ms.IsGpuModified() && !oversized_ms.IsBufferModified() && !resources.GetBufferCache().HasGpuDirtyBytes( ms_depth_desc.info.data.address, ms_data_size), @@ -2742,17 +2928,15 @@ public: resources.HandleFault(PageFaultAccess::Write, mirror_desc.info.data.address), "GPU image did not accept a CPU write before Buffer mirroring"); std::memcpy(memory + 0x5000, &mirror_cpu_value, sizeof(mirror_cpu_value)); - Require(name, "image-to-buffer mirror", - texture_cache.SynchronizeImageToBuffer(base + 0x5000, sizeof(mirror_value)) && - texture_cache.GetImage(mirror_image).IsBufferModified() && - !texture_cache.GetImage(mirror_image).IsGpuModified(), - "image-to-buffer synchronization did not transfer ownership"); auto mirror_binding = resources.GetBufferCache().ObtainBuffer( - command, base + 0x5000, sizeof(mirror_value), false, true); + command, base + 0x5000, sizeof(mirror_value), false, true, true); Require(name, "CPU-dirty formatted mirror source", - mirror_binding.buffer != nullptr && mirror_binding.owner != nullptr, - "formatted mirror did not expose its native Buffer source"); - command.RetainResourceUntilFence(mirror_binding.owner); + mirror_binding.buffer != nullptr && + !texture_cache.GetImage(mirror_image).IsBufferModified(), + "formatted mirror did not expose a readable Buffer source"); + if (mirror_binding.owner != nullptr) { + command.RetainResourceUntilFence(mirror_binding.owner); + } auto mirror_cpu_readback = CreateHostBuffer(name, sizeof(mirror_cpu_value), vk::BufferUsageFlagBits::eTransferDst, std::vector {0}); @@ -2767,10 +2951,9 @@ public: const auto mirror_refresh = texture_cache.FindImage(mirror_refresh_desc); Require(name, "buffer-to-image ownership", mirror_refresh == mirror_image && - !texture_cache.GetImage(mirror_refresh).IsBufferModified() && - texture_cache.GetImage(mirror_refresh).IsGpuModified(), - "buffer-backed refresh did not atomically reclaim image " - "ownership"); + !texture_cache.GetImage(mirror_refresh).IsBufferModified(), + "buffer-backed refresh incorrectly transferred dirty ownership " + "to the image"); texture_cache.MarkGpuWritten(mirror_refresh); constexpr uint64_t exact_buffer_offset = 0x90000; @@ -2783,11 +2966,17 @@ public: Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1); const auto exact_buffer_image = texture_cache.FindImage(exact_buffer_desc); texture_cache.MarkGpuWritten(exact_buffer_image); - Require(name, "exact replacement Buffer publication", - texture_cache.SynchronizeImageToBuffer(exact_buffer_desc.info.data.address, - exact_buffer_desc.info.data.size) && - texture_cache.GetImage(exact_buffer_image).IsBufferModified(), - "exact replacement source did not transfer to Buffer ownership"); + auto exact_buffer_binding = resources.GetBufferCache().ObtainBuffer( + command, exact_buffer_desc.info.data.address, exact_buffer_desc.info.data.size, + false, true, true); + Require(name, "exact replacement Buffer synchronization", + exact_buffer_binding.buffer != nullptr && + !texture_cache.GetImage(exact_buffer_image).IsBufferModified() && + texture_cache.GetImage(exact_buffer_image).IsGpuModified(), + "exact replacement copy transferred cache ownership"); + if (exact_buffer_binding.owner != nullptr) { + command.RetainResourceUntilFence(exact_buffer_binding.owner); + } auto exact_float_desc = exact_buffer_desc; exact_float_desc.info.pixel_format = vk::Format::eR32Sfloat; exact_float_desc.info.guest_format = @@ -2797,11 +2986,11 @@ public: Require(name, "Buffer-superseded exact coexistence", exact_float_image != exact_buffer_image && TextureCacheTestAccess::Contains(texture_cache, exact_buffer_image) && - texture_cache.GetImage(exact_buffer_image).IsBufferModified() && + texture_cache.GetImage(exact_buffer_image).IsGpuModified() && !texture_cache.GetImage(exact_float_image).IsBufferModified() && - texture_cache.GetImage(exact_float_image).IsGpuModified(), - "exact-format lookup retired its old record or failed to import " - "the shared Buffer source"); + !texture_cache.GetImage(exact_float_image).IsGpuModified(), + "exact-format lookup retired its old record or transferred Buffer " + "ownership"); auto exact_buffer_readback = CreateHostBuffer(name, sizeof(exact_buffer_value), vk::BufferUsageFlagBits::eTransferDst, std::vector {0}); @@ -2870,23 +3059,21 @@ public: Prospero::GpuEnumValue(Prospero::BufferFormat::k32UInt), Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1); const auto partial_image = texture_cache.FindImage(partial_desc); - Require(name, "partial-page image ownership", - texture_cache.GetImage(partial_image).IsGpuModified(), - "image did not consume its exact GPU-owned buffer bytes"); + Require(name, "partial-page image upload", + !texture_cache.GetImage(partial_image).IsGpuModified(), + "image upload incorrectly consumed buffer dirty ownership"); auto partial_image_mirror = resources.GetBufferCache().ObtainBuffer( command, base + partial_image_offset, sizeof(partial_image_value), false, true, true); Require(name, "partial-page image mirror", partial_image_mirror.buffer != nullptr && partial_image_mirror.owner != nullptr && - texture_cache.GetImage(partial_image).IsBufferModified() && !texture_cache.GetImage(partial_image).IsGpuModified(), - "same-page image bytes did not transfer back to their buffer"); + "same-page image upload lost its buffer source"); command.RetainResourceUntilFence(partial_image_mirror.owner); const auto partial_clean_source = resources.GetBufferCache().ObtainBufferForImage( base + partial_clean_offset, sizeof(partial_clean_value)); - Require(name, "partial-page clean source", - partial_clean_source.buffer != nullptr && !partial_clean_source.gpu_owned, + Require(name, "partial-page clean source", partial_clean_source.buffer != nullptr, "clean same-page bytes inherited unrelated buffer ownership"); Require(name, "partial-page remaining fault", resources.HandleFault(PageFaultAccess::Read, base + partial_buffer_offset), @@ -2910,8 +3097,7 @@ public: const auto partial_cpu_refresh_source = resources.GetBufferCache().ObtainBufferForImage( base + partial_clean_offset, sizeof(partial_cpu_refresh_value)); Require(name, "partial-page CPU refresh source", - partial_cpu_refresh_source.buffer != nullptr && - !partial_cpu_refresh_source.gpu_owned, + partial_cpu_refresh_source.buffer != nullptr, "CPU-dirty same-page bytes did not resolve through the cached " "buffer"); auto partial_cpu_refresh_readback = @@ -2972,20 +3158,25 @@ public: texture_cache.GetImage(fault_a_image).IsMaybeCpuDirty() && texture_cache.GetImage(fault_b_image).IsMaybeCpuDirty(), "a byte-disjoint CPU write discarded authoritative images"); - const auto retracked_a = texture_cache.FindImage(fault_a_desc); - const auto retracked_b = texture_cache.FindImage(fault_b_desc); + const auto retracked_a = texture_cache.FindImage(fault_a_desc); + const auto retracked_b = texture_cache.FindImage(fault_b_desc); + auto fault_a_mirror = resources.GetBufferCache().ObtainBuffer( + command, base + 0x8000, sizeof(fault_a), false, true, true); + if (fault_a_mirror.owner != nullptr) { + command.RetainResourceUntilFence(fault_a_mirror.owner); + } Require(name, "same-page image re-track", retracked_a == fault_a_image && retracked_b == fault_b_image && texture_cache.GetImage(fault_a_image).IsTracked() && texture_cache.GetImage(fault_b_image).IsTracked() && !texture_cache.GetImage(fault_a_image).IsCpuDirty() && !texture_cache.GetImage(fault_b_image).IsCpuDirty() && - texture_cache.SynchronizeImageToBuffer(base + 0x8000, sizeof(fault_a)) && + fault_a_mirror.buffer != nullptr && texture_cache.GetImage(fault_a_image).IsTracked() && texture_cache.GetImage(fault_b_image).IsTracked() && - !texture_cache.GetImage(fault_a_image).IsGpuModified() && + texture_cache.GetImage(fault_a_image).IsGpuModified() && texture_cache.GetImage(fault_b_image).IsGpuModified(), - "retiring one same-page image lost the surviving owner"); + "copying one same-page image lost an authoritative owner"); Require(name, "same-page survivor write fault", resources.HandleFault(PageFaultAccess::Write, base + 0x8010) && texture_cache.GetImage(fault_b_image).IsGpuModified() && @@ -3011,13 +3202,6 @@ public: Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1); const auto publish_image = texture_cache.FindImage(publish_image_desc); texture_cache.MarkGpuWritten(publish_image); - auto [publish_buffer_owner, publish_buffer_native_offset] = - resources.GetBufferCache().ObtainBufferForImageWrite(base + publish_buffer_offset, - sizeof(publish_buffer_value)); - (void)publish_buffer_native_offset; - Require(name, "exact-disjoint buffer allocation", publish_buffer_owner != nullptr, - "failed to create the neighboring native buffer owner"); - command.RetainResourceUntilFence(publish_buffer_owner); resources.GetBufferCache().FillBuffer( base + publish_buffer_offset, sizeof(publish_buffer_value), publish_buffer_value); auto publish_replacement_desc = publish_image_desc; @@ -3286,8 +3470,6 @@ public: "compressed image incorrectly claimed a CPU read fault"); Require(name, "compressed download rejection", !TextureCacheTestAccess::TryDownload(texture_cache, compressed_image) && - !texture_cache.SynchronizeImageToBuffer(compressed_desc.info.data.address, - compressed_desc.info.data.size) && texture_cache.GetImage(compressed_image).IsGpuModified() && !texture_cache.GetImage(compressed_image).IsBufferModified(), "a compressed image escaped the unified download guard"); @@ -3309,13 +3491,11 @@ public: constexpr uint32_t mixed_source_width = 1025; constexpr uint32_t mixed_cpu_value = 0x1234abcdu; constexpr uint32_t mixed_gpu_value = 0x9876fedcu; - auto [mixed_owner, mixed_owner_offset] = - resources.GetBufferCache().ObtainBufferForImageWrite(base + mixed_source_offset, - mixed_source_size); - (void)mixed_owner_offset; - Require(name, "mixed-page source allocation", mixed_owner != nullptr, + auto mixed_owner = resources.GetBufferCache().ObtainBuffer( + command, base + mixed_source_offset, mixed_source_size, true, true); + Require(name, "mixed-page source allocation", mixed_owner.owner != nullptr, "mixed CPU/GPU image source did not create a containing buffer"); - command.RetainResourceUntilFence(mixed_owner); + command.RetainResourceUntilFence(mixed_owner.owner); Require(name, "mixed-page CPU write fault", resources.HandleFault(PageFaultAccess::Write, base + mixed_source_offset), "mixed image source could not dirty its first page"); @@ -3362,18 +3542,13 @@ public: auto byte_mirror = resources.GetBufferCache().ObtainBuffer( command, base + byte_mirror_offset, 1, false, true, true); Require(name, "byte image mirror", - byte_mirror.owner != nullptr && byte_mirror.buffer != nullptr && - texture_cache.GetImage(byte_mirror_image).IsBufferModified(), - "one-byte image did not transfer exact ownership to BufferCache"); - command.RetainResourceUntilFence(byte_mirror.owner); - Require(name, "byte image fault", - resources.HandleFault(PageFaultAccess::Read, base + byte_mirror_offset), - "one-byte image mirror could not be downloaded"); - std::array byte_mirror_backing {}; - std::memcpy(byte_mirror_backing.data(), memory + byte_mirror_page_offset, - byte_mirror_backing.size()); - Require(name, "byte image fault contents", byte_mirror_backing == byte_mirror_guest, - "aligned one-byte download changed neighboring guest sentinels"); + byte_mirror.buffer != nullptr && + !texture_cache.GetImage(byte_mirror_image).IsBufferModified() && + texture_cache.GetImage(byte_mirror_image).IsGpuModified(), + "one-byte image copy transferred cache ownership"); + if (byte_mirror.owner != nullptr) { + command.RetainResourceUntilFence(byte_mirror.owner); + } const std::array bgra16_guest {0x3c00u, 0x4000u, 0x4200u, 0x4400u}; std::memcpy(memory + 0xb000, bgra16_guest.data(), sizeof(bgra16_guest)); @@ -3557,23 +3732,42 @@ public: scheduler.Finish(); + const auto mip_prefix_words = + ReadBuffer(name, mip_prefix_readback, mip_prefix_size / sizeof(uint32_t)); + const auto mip_formatted_words = + ReadBuffer(name, mip_formatted_readback, mip_guest_size / sizeof(uint32_t)); + const auto mip0_word = mip_sizes[0].offset / sizeof(uint32_t); + Require(name, "formatted mip prefix content", + mip0_word < mip_prefix_words.size() && mip0_word < mip_native.size() && + mip_prefix_words[mip0_word] == mip_native[mip0_word], + "fitting mip-prefix synchronization copied " + "stale guest backing"); + Require(name, "formatted Buffer content", + mip0_word < mip_formatted_words.size() && mip0_word < mip_native.size() && + mip_formatted_words[mip0_word] == mip_native[mip0_word], + "formatted Buffer read bypassed authoritative " + "native image mip data"); Require(name, "CPU-dirty formatted mirror content", ReadBuffer(name, mirror_cpu_readback, 1) == std::vector {mirror_cpu_value}, - "formatted Buffer mirror published stale native image bytes"); + "formatted Buffer mirror published stale " + "native image bytes"); Require(name, "Buffer-superseded exact content", ReadBuffer(name, exact_buffer_readback, 1) == std::vector {exact_buffer_value}, - "exact-format recreation initialized from stale guest bytes"); + "exact-format recreation initialized from " + "stale guest bytes"); Require(name, "partial-page CPU refresh content", ReadBuffer(name, partial_cpu_refresh_readback, 1) == std::vector {partial_cpu_refresh_value}, - "cached buffer uploaded bytes outside the exact staged guest range"); + "cached buffer uploaded bytes outside the exact " + "staged guest range"); const auto mixed_source_words = ReadBuffer(name, mixed_source_readback, mixed_source_size / sizeof(u32)); Require(name, "mixed-page image content", mixed_source_words.front() == mixed_cpu_value && mixed_source_words[0x1000 / sizeof(u32)] == mixed_gpu_value, - "mixed CPU/GPU source upload lost one ownership domain"); + "mixed CPU/GPU source upload lost one " + "ownership domain"); Require(name, "BGRA16 content", ReadBuffer(name, bgra16_readback, 2) == std::vector {0x40004200u, 0x44003c00u}, @@ -3666,8 +3860,8 @@ public: texture_cache.GetImage(exact_image).IsGpuModified(), "image ownership discarded or conflicted with disjoint dirty " "Buffer bytes on the same tracker page"); - resources.GetBufferCache().DiscardGpuDirtyBytes(base + dirty_sibling_offset, - sizeof(dirty_sibling_value)); + resources.GetBufferCache().ReadMemory(base + dirty_sibling_offset, + sizeof(dirty_sibling_value)); constexpr std::array gc_image_offsets {0x330000, 0x332000}; constexpr std::array gc_image_values {0x76543210u, 0x89abcdefu}; @@ -3689,8 +3883,8 @@ public: clean_buffer_alias.owner != nullptr && clean_buffer_alias.buffer != nullptr, "failed to create the clean cached Buffer alias"); scheduler.Current().RetainResourceUntilFence(clean_buffer_alias.owner); - resources.GetBufferCache().DiscardGpuDirtyBytes(gc_image_desc_a.info.data.address, - gc_image_desc_a.info.data.size); + resources.GetBufferCache().ReadMemory(gc_image_desc_a.info.data.address, + gc_image_desc_a.info.data.size); const std::array gc_images {texture_cache.FindImage(gc_image_desc_a), texture_cache.FindImage(gc_image_desc_b)}; Require(name, "non-GPU image range validity", @@ -3746,12 +3940,14 @@ public: gc_before_completion == gc_stale_values, "GC submitted per image or published a readback before GPU " "completion"); + scheduler.FinishCurrent(); + scheduler.DrainPriorityOperations(); auto refreshed_buffer_alias = resources.GetBufferCache().ObtainBuffer( scheduler.Current(), gc_image_desc_a.info.data.address, gc_image_desc_a.info.data.size, false, true); - Require(name, "publication-gated Buffer reacquire", + Require(name, "post-publication Buffer reacquire", refreshed_buffer_alias.buffer != nullptr, - "Buffer lookup did not wait for the retired image publication"); + "Buffer lookup failed after the retired image publication"); if (refreshed_buffer_alias.owner != nullptr) { scheduler.Current().RetainResourceUntilFence(refreshed_buffer_alias.owner); } @@ -4373,32 +4569,47 @@ public: auto repeated_standard_4kb_alias = standard_4kb_alias; const auto repeated_standard_4kb_alias_image = texture_cache.FindImage(repeated_standard_4kb_alias); - Require(name, "equal-size tile-mode alias", - render_target_alias_image && standard_4kb_alias_image && - standard_4kb_alias_image != render_target_alias_image && - repeated_standard_4kb_alias_image == standard_4kb_alias_image && - texture_cache.GetImage(render_target_alias_image).info.tile_mode == - Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget) && - texture_cache.GetImage(standard_4kb_alias_image).info.tile_mode == - Prospero::GpuEnumValue(Prospero::TileMode::kStandard4KB), - "equal address/size lookup reused an incompatible tiled backing"); + Require( + name, "equal-size tile-mode alias", + render_target_alias_image && + standard_4kb_alias_image && + standard_4kb_alias_image != + render_target_alias_image && + repeated_standard_4kb_alias_image == + standard_4kb_alias_image && + texture_cache + .GetImage(render_target_alias_image) + .info.tile_mode == + Prospero::GpuEnumValue( + Prospero::TileMode::kRenderTarget) && + texture_cache.GetImage(standard_4kb_alias_image) + .info.tile_mode == + Prospero::GpuEnumValue( + Prospero::TileMode::kStandard4KB), + "equal address/size lookup reused an incompatible " + "tiled backing"); - for (auto& output: ms_observer_outputs) { - DestroyBuffer(&output); - } - DestroyBuffer(&layered_readback); - DestroyBuffer(&bgra16_readback); - DestroyBuffer(&mixed_source_readback); - DestroyBuffer(&partial_cpu_refresh_readback); - DestroyBuffer(&mirror_cpu_readback); - DestroyBuffer(&exact_buffer_readback); - m_device.destroyDescriptorPool(observer_pool, nullptr); - m_device.destroyPipeline(observer_depth_pipeline, nullptr); - m_device.destroyPipelineLayout(observer_pipeline_layout, nullptr); - m_device.destroyDescriptorSetLayout(observer_set_layout, nullptr); - m_device.destroyShaderModule(ms_depth_module, nullptr); + for (auto &output : ms_observer_outputs) { + DestroyBuffer(&output); + } + DestroyBuffer(&layered_readback); + DestroyBuffer(&bgra16_readback); + DestroyBuffer(&mixed_source_readback); + DestroyBuffer(&mip_formatted_readback); + DestroyBuffer(&mip_prefix_readback); + DestroyBuffer(&partial_cpu_refresh_readback); + DestroyBuffer(&mirror_cpu_readback); + DestroyBuffer(&exact_buffer_readback); + m_device.destroyDescriptorPool(observer_pool, nullptr); + m_device.destroyPipeline(observer_depth_pipeline, + nullptr); + m_device.destroyPipelineLayout(observer_pipeline_layout, + nullptr); + m_device.destroyDescriptorSetLayout(observer_set_layout, + nullptr); + m_device.destroyShaderModule(ms_depth_module, nullptr); - resources.SetGpu(nullptr); + resources.SetGpu(nullptr); resources.UnmapMemory(base, allocation_size); scheduler.Finish(); } @@ -4487,11 +4698,8 @@ public: cache.MarkGpuWritten(id); Require(name, "guest readback queue", TextureCacheTestAccess::TryDownload(cache, id), "tiled BGRA16 guest readback was rejected"); - Require(name, "Buffer mirror", cache.SynchronizeImageToBuffer(base, total.size), - "tiled BGRA16 Buffer mirror was rejected"); - - auto mirror = - resources.GetBufferCache().ObtainBuffer(scheduler.Current(), base, 8, false, true); + auto mirror = resources.GetBufferCache().ObtainBuffer(scheduler.Current(), base, + total.size, false, true, true); Require(name, "mirror owner", mirror.buffer != nullptr && mirror.owner != nullptr, "tiled BGRA16 mirror has no BufferCache owner"); scheduler.Current().RetainResourceUntilFence(mirror.owner); @@ -4730,9 +4938,9 @@ public: "storage descriptor did not preserve its sRGB backing and " "select an UNORM Vulkan view"); - ShaderTextureResource sint_storage {{0x01514b00u, 0xc1500000u, 0x000bc00bu, - 0x91b00204u, 0x00000000u, 0x00700000u, - 0x102b0000u, 0x0001514au}}; + ShaderTextureResource sint_storage {{0x01514b00u, 0xc1500000u, 0x000bc00bu, 0x91b00204u, + 0x00000000u, 0x00700000u, 0x102b0000u, + 0x0001514au}}; Require(name, "PPSA06888 R32 SINT descriptor", sint_storage.Base40() == 0x1514b0000ull && sint_storage.Width5() + 1u == 48 && sint_storage.Height5() + 1u == 48 && sint_storage.Depth() + 1u == 1 && @@ -4743,20 +4951,20 @@ public: sint_storage.TileMode() == Prospero::GpuEnumValue(Prospero::TileMode::kRenderTarget) && sint_storage.DstSelXYZW() == DstSel(4, 0, 0, 1), - "captured write-only signed storage descriptor was decoded incorrectly"); - const uint64_t mapped_sint_address = base + 0xe0000; + "captured write-only signed storage descriptor was decoded " + "incorrectly"); + const uint64_t mapped_sint_address = base + 0xe0000; const auto encoded_sint_address = mapped_sint_address >> 8u; - sint_storage.fields[0] = static_cast(encoded_sint_address); - sint_storage.fields[1] = - (sint_storage.fields[1] & ~0xffu) | - static_cast(encoded_sint_address >> 32u); + sint_storage.fields[0] = static_cast(encoded_sint_address); + sint_storage.fields[1] = (sint_storage.fields[1] & ~0xffu) | + static_cast(encoded_sint_address >> 32u); ShaderRecompiler::IR::DescriptorValue sint_storage_descriptor {}; std::copy(std::begin(sint_storage.fields), std::end(sint_storage.fields), sint_storage_descriptor.dwords.begin()); sint_storage_descriptor.dword_count = 8; - auto sint_storage_resource = srgb_storage_resource; - sint_storage_resource.kind = ShaderRecompiler::IR::ResourceKind::StorageImageUint; - const auto sint_storage_binding = RenderExecutorTestAccess::ResolveTexture( + auto sint_storage_resource = srgb_storage_resource; + sint_storage_resource.kind = ShaderRecompiler::IR::ResourceKind::StorageImageUint; + const auto sint_storage_binding = RenderExecutorTestAccess::ResolveTexture( executor, sint_storage_resource, sint_storage_descriptor); const auto sint_storage_view = texture_cache.FindTexture(sint_storage_binding.image_id, sint_storage_binding.desc); @@ -4767,7 +4975,8 @@ public: sint_storage_binding.desc.view_info.format == vk::Format::eR32Uint && texture_cache.GetImage(sint_storage_binding.image_id).backing.format == vk::Format::eR32Sint, - "write-only R32 SINT storage did not select a bit-compatible uint view"); + "write-only R32 SINT storage did not select a bit-compatible " + "uint view"); auto narrowed_storage = storage; constexpr uint64_t narrowed_storage_address = base + 0xd0000; @@ -5929,7 +6138,8 @@ public: Count(Kind::Storage2DArray) == 0 && Count(Kind::Storage3D) == 0 && Count(Kind::StorageUint1D) == 0 && Count(Kind::StorageUint1DArray) == 0 && Count(Kind::StorageUint2DArray) == 0 && Count(Kind::StorageUint3D) == 0, - "unsupported array/3D image cases must provide matching Vulkan test views " + "unsupported array/3D image cases must provide matching Vulkan test " + "views " "before dispatch"); vk::ShaderModuleCreateInfo module_info {}; @@ -13388,7 +13598,8 @@ TestCase ImageSampleA16OffsetKeepsTexelOffset32BitOnGpu() { using O = ShaderOpcode; std::vector code; - AppendVMovU32(&code, 20, 1); // Non-constant +1 X offset is not a SPIR-V ConstOffset. + AppendVMovU32(&code, 20, + 1); // Non-constant +1 X offset is not a SPIR-V ConstOffset. AppendVMovLiteral(&code, 21, 0x36003900u); // x=0.625, y=0.375 packed as f16. AppendVMovU32(&code, 22, 0); code.push_back(EncodeMimg0(0x30, 0xf)); @@ -14465,8 +14676,8 @@ void CheckEmbeddedFetchVertexOffset() { void CheckRenderTargetFormatContract() { const auto rgb565 = TextureGetRenderTargetFormat(16u, 0u, 0u); Require("RenderTargetFormat", "RGB565 UNorm", - rgb565.format == vk::Format::eB5G6R5UnormPack16 && - rgb565.bytes_per_element == 2u && rgb565.export_mapping.IsIdentity(), + rgb565.format == vk::Format::eB5G6R5UnormPack16 && rgb565.bytes_per_element == 2u && + rgb565.export_mapping.IsIdentity(), "RGB565 UNorm render-target tuple was rejected"); const auto uint_format = TextureGetRenderTargetFormat(12u, 4u, 0u); @@ -15195,10 +15406,16 @@ void CheckSampledDepthDescriptor(RenderContext& renderer) { "normalized depth image rejected a valid padded descriptor"); const ShaderTextureResource uncompressed_msaa {{ - 0x00705d00u, 0xc1600000u, 0x010dc1dfu, 0xe1810924u, - 0x00000000u, 0x00700010u, 0x00000000u, 0x00000000u, + 0x00705d00u, + 0xc1600000u, + 0x010dc1dfu, + 0xe1810924u, + 0x00000000u, + 0x00700010u, + 0x00000000u, + 0x00000000u, }}; - auto msaa_info = + auto msaa_info = make_info(1920, 1080, 1920, 1, vk::Format::eD32Sfloat, Prospero::ImageType::kColor2D, 2); msaa_info.mip_layout[0] = {0, 0x010e0000, 1920, 1152}; Image msaa_image(context, scheduler, msaa_info); @@ -15206,7 +15423,8 @@ void CheckSampledDepthDescriptor(RenderContext& renderer) { Require("SampledDepthDescriptor", "uncompressed 2x MSAA depth", IsSupportedDepthTargetDescriptor(uncompressed_msaa, msaa_image) && IsSupportedDepthTextureEncoding(uncompressed_msaa, msaa_image), - "valid uncompressed MSAA depth descriptor required an HTILE compatibility flag"); + "valid uncompressed MSAA depth descriptor required an HTILE " + "compatibility flag"); descriptor.fields[3] = (descriptor.fields[3] & ~(0xfu << 28u)) | (Prospero::GpuEnumValue(Prospero::ImageType::kColor2DArray) << 28u);